diff --git a/.trinity/seals/ForwardPass.json b/.trinity/seals/ForwardPass.json index d92577c2e8..f55b96cf40 100644 --- a/.trinity/seals/ForwardPass.json +++ b/.trinity/seals/ForwardPass.json @@ -1,14 +1,19 @@ { - "gen_hash_c": "sha256:ec9a060b8a22921f6686f707a783bbc96e63a647a3857164ef5ba319087bf01d", - "gen_hash_rust": "sha256:640fd0d0e5858d94a85bbe6a1c117480e7f0cc3b41284687965f6b872060efbb", - "gen_hash_verilog": "sha256:adcbce2f7d721f5f6039e41529cd8784d9a2ca287f381b275bb520ab4ade948e", - "gen_hash_zig": "sha256:741a124f0fa327b72395f4f465bd8f0faec9cb041621ae3b16b3f14988abe6ce", + "gen_hash_c": "sha256:5d1ea731db23964100134b5feaa67a4c74598fc152187e3e4ae0b1e6911ae159", + "gen_hash_rust": "sha256:57c23e49510e367eca701e58a3fd4ea5eeb8aeab5058081f41f6b81d3056e437", + "gen_hash_verilog": "sha256:22ef4365189b450ce005f2c697f72462d9199a403c18d777c4a6db16aa4b59b4", + "gen_hash_zig": "sha256:0219f60527363dcd9c34ac6625cc829a584ad023c4a694b37a18bcf421c2254e", "module": "ForwardPass", "ring": 12, - "sealed_at": "2026-10-11T10:43:12Z", - "spec_hash": "sha256:c859536339d6262a50e1769163bac451e5d72eea6f504081cf89579c70d25e1d", + "sealed_at": "2026-10-17T17:34:47Z", + "spec_hash": "sha256:410f24b3cc1b5ab8171c0234c2a9a6501cdf225d9386572552b2bb9e558c1ae3", "spec_path": "specs/hslm/forward_pass.t27", "tests": { - "blocked": "does not compile: spec.zig:123:72: error: use of undeclared identifier 'q_role'" + "failed": 0, + "failing": [], + "forced": false, + "passed": 14, + "total": 14, + "vacuous": 0 } } \ No newline at end of file diff --git a/.trinity/seals/HSLM.json b/.trinity/seals/HSLM.json index 32986fc731..dcca23f704 100644 --- a/.trinity/seals/HSLM.json +++ b/.trinity/seals/HSLM.json @@ -1,14 +1,19 @@ { - "gen_hash_c": "sha256:981dd040e1a428a855b49a70a1057b15aec8b1ae983665ab56a12cf4dfebbe9b", - "gen_hash_rust": "sha256:802705ac3fa6bdbfad56bbcca7236fe3a0b1a9f1a408b14b239db5f96e6f5d4e", - "gen_hash_verilog": "sha256:b0fe9634a4234ef800d3351a47f89f20e884a4bc6e164a5cd20e0c0ae752c855", - "gen_hash_zig": "sha256:151ee73ac0f9a3dbd34e0fdbc03ee093f3b0ba8a6ca30d6b11bacbe4b22b5cbc", + "gen_hash_c": "sha256:fd94f6962089194da7478e83cd9a015b83a28707b6cb35a9c8e629a96f502fa8", + "gen_hash_rust": "sha256:7f60cf6c5c5c2bf0ee07982585e981bb3e84a5444eab90063406f36940d9d76d", + "gen_hash_verilog": "sha256:25e87d2ab265533cab0de48e751110380236e26686ba78a711d412eb625cf1a8", + "gen_hash_zig": "sha256:769110ea52ada18974a5c561ea7c364bf4c7169c29b55619fb7daf16a2a91451", "module": "HSLM", "ring": 12, - "sealed_at": "2026-10-11T10:43:23Z", - "spec_hash": "sha256:2056290affb21128c3fec58c0b04a3999569434cbfb2513ae7a58866026de4bf", + "sealed_at": "2026-10-17T17:34:44Z", + "spec_hash": "sha256:1ffe306efd7cc6b092829cbcfb7645cbadde70fed20b3c3a7e3cfedd3abd27ce", "spec_path": "specs/nn/hslm.t27", "tests": { - "blocked": "does not compile: spec.zig:624:27: error: use of undeclared identifier 'constants'" + "failed": 0, + "failing": [], + "forced": false, + "passed": 26, + "total": 26, + "vacuous": 0 } } \ No newline at end of file diff --git a/.trinity/seals/hslm_ForwardPass.json b/.trinity/seals/hslm_ForwardPass.json index 9d02f067ad..93983273bc 100644 --- a/.trinity/seals/hslm_ForwardPass.json +++ b/.trinity/seals/hslm_ForwardPass.json @@ -1,33 +1,30 @@ { - "built_by": "t27c-bootstrap@0.5.2+cc19d1fcd", + "built_by": "t27c-bootstrap@0.5.2+efe3c94", "closure": [ { - "spec": "specs/base/types; // Trit enum.t27", - "spec_hash": "missing" - }, - { - "spec": "specs/numeric/gf16; // GF16 for similarity scores.t27", - "spec_hash": "missing" - }, - { - "spec": "specs/ternary/packed_trit; // PackedBigInt for HV storage.t27", - "spec_hash": "missing" + "spec": "specs/base/types.t27", + "spec_hash": "sha256:3c2fb2a2edf20e90bc76798fb5d12690b2f60fc0f5bbf2f6052936e22d6e62e2" } ], "config": "gen=c,rust,verilog,zig;tests=zig", - "gen_hash_c": "sha256:ec9a060b8a22921f6686f707a783bbc96e63a647a3857164ef5ba319087bf01d", - "gen_hash_rust": "sha256:640fd0d0e5858d94a85bbe6a1c117480e7f0cc3b41284687965f6b872060efbb", - "gen_hash_verilog": "sha256:adcbce2f7d721f5f6039e41529cd8784d9a2ca287f381b275bb520ab4ade948e", - "gen_hash_zig": "sha256:741a124f0fa327b72395f4f465bd8f0faec9cb041621ae3b16b3f14988abe6ce", + "gen_hash_c": "sha256:5d1ea731db23964100134b5feaa67a4c74598fc152187e3e4ae0b1e6911ae159", + "gen_hash_rust": "sha256:57c23e49510e367eca701e58a3fd4ea5eeb8aeab5058081f41f6b81d3056e437", + "gen_hash_verilog": "sha256:22ef4365189b450ce005f2c697f72462d9199a403c18d777c4a6db16aa4b59b4", + "gen_hash_zig": "sha256:0219f60527363dcd9c34ac6625cc829a584ad023c4a694b37a18bcf421c2254e", "module": "ForwardPass", "ring": 12, "seal_schema": 2, - "sealed_at": "2026-10-11T10:43:12Z", + "sealed_at": "2026-10-17T17:34:47Z", "sealed_by": "t27c-bootstrap@0.5.2", - "spec_hash": "sha256:c859536339d6262a50e1769163bac451e5d72eea6f504081cf89579c70d25e1d", + "spec_hash": "sha256:410f24b3cc1b5ab8171c0234c2a9a6501cdf225d9386572552b2bb9e558c1ae3", "spec_path": "specs/hslm/forward_pass.t27", "tests": { - "blocked": "does not compile: spec.zig:123:72: error: use of undeclared identifier 'q_role'" + "failed": 0, + "failing": [], + "forced": false, + "passed": 14, + "total": 14, + "vacuous": 0 }, "toolchain": { "test_runner": "sha256:1982ae948e24d00c5d221ccec12dfce1eb01204b53e37b0f857f93b16d64629d", diff --git a/.trinity/seals/nn_HSLM.json b/.trinity/seals/nn_HSLM.json index 0ad318169c..af89890f1e 100644 --- a/.trinity/seals/nn_HSLM.json +++ b/.trinity/seals/nn_HSLM.json @@ -1,45 +1,30 @@ { - "built_by": "t27c-bootstrap@0.5.2+cc19d1fcd", + "built_by": "t27c-bootstrap@0.5.2+efe3c94", "closure": [ - { - "spec": "specs/base/ops.t27", - "spec_hash": "sha256:8e6c6168f64ff2d654a3b276d7f3ceee4fbd408d334590ce03467b1f3aaa68b3" - }, { "spec": "specs/base/types.t27", "spec_hash": "sha256:3c2fb2a2edf20e90bc76798fb5d12690b2f60fc0f5bbf2f6052936e22d6e62e2" - }, - { - "spec": "specs/math/constants.t27", - "spec_hash": "sha256:57187e7afda1d88167b2af0382ad7f0623aa6b269ba020a78e3ffeebba1161bb" - }, - { - "spec": "specs/math/sacred_physics.t27", - "spec_hash": "sha256:6f0c02de93fc908bcae88bd48037b8b06c54efe3fc0105ab09288fb2102e74d9" - }, - { - "spec": "specs/nn/attention.t27", - "spec_hash": "sha256:be83d2a916a15957502d6afa4d6eff7ab07d0373e81555704ee842500db47187" - }, - { - "spec": "specs/numeric/gf16.t27", - "spec_hash": "sha256:2a33d77db23966331716955924d08e30d12a3af361067da0a2c7072b7f4b7b0d" } ], "config": "gen=c,rust,verilog,zig;tests=zig", - "gen_hash_c": "sha256:981dd040e1a428a855b49a70a1057b15aec8b1ae983665ab56a12cf4dfebbe9b", - "gen_hash_rust": "sha256:802705ac3fa6bdbfad56bbcca7236fe3a0b1a9f1a408b14b239db5f96e6f5d4e", - "gen_hash_verilog": "sha256:b0fe9634a4234ef800d3351a47f89f20e884a4bc6e164a5cd20e0c0ae752c855", - "gen_hash_zig": "sha256:151ee73ac0f9a3dbd34e0fdbc03ee093f3b0ba8a6ca30d6b11bacbe4b22b5cbc", + "gen_hash_c": "sha256:fd94f6962089194da7478e83cd9a015b83a28707b6cb35a9c8e629a96f502fa8", + "gen_hash_rust": "sha256:7f60cf6c5c5c2bf0ee07982585e981bb3e84a5444eab90063406f36940d9d76d", + "gen_hash_verilog": "sha256:25e87d2ab265533cab0de48e751110380236e26686ba78a711d412eb625cf1a8", + "gen_hash_zig": "sha256:769110ea52ada18974a5c561ea7c364bf4c7169c29b55619fb7daf16a2a91451", "module": "HSLM", "ring": 12, "seal_schema": 2, - "sealed_at": "2026-10-11T10:43:23Z", + "sealed_at": "2026-10-17T17:34:44Z", "sealed_by": "t27c-bootstrap@0.5.2", - "spec_hash": "sha256:2056290affb21128c3fec58c0b04a3999569434cbfb2513ae7a58866026de4bf", + "spec_hash": "sha256:1ffe306efd7cc6b092829cbcfb7645cbadde70fed20b3c3a7e3cfedd3abd27ce", "spec_path": "specs/nn/hslm.t27", "tests": { - "blocked": "does not compile: spec.zig:624:27: error: use of undeclared identifier 'constants'" + "failed": 0, + "failing": [], + "forced": false, + "passed": 26, + "total": 26, + "vacuous": 0 }, "toolchain": { "test_runner": "sha256:1982ae948e24d00c5d221ccec12dfce1eb01204b53e37b0f857f93b16d64629d", diff --git a/specs/hslm/forward_pass.t27 b/specs/hslm/forward_pass.t27 index 3168ce0bb0..357a278dfb 100644 --- a/specs/hslm/forward_pass.t27 +++ b/specs/hslm/forward_pass.t27 @@ -1,30 +1,46 @@ // SPDX-License-Identifier: Apache-2.0 -// Module: Minimal Forward Pass for LLM Inference +// Module: Minimal Forward Pass for LLM Inference (VSA, character level) // φ² + 1/φ² = 3 | TRINITY +// +// This spec never compiled: its tests read q_role, k_role, v_role, context +// and role, which nothing declared; four functions had empty bodies; the rest +// returned an empty vector, 0.0 or their own argument; every invariant was +// `assert true`; and the benches named role1..role8, which do not exist. +// +// It is written out now on dense hypervectors of ROLE_DIM trits (stored as +// i8 in {-1, 0, +1}) in module storage: +// - char_hv: a fixed pseudo-random +-1 vector per character +// - bind (elementwise product, its own inverse), bundle (majority), +// permute (cyclic shift, encodes position), similarity (cosine) +// - attention: the query is compared with every positioned context vector +// bound with the key role; the best match, bound with the value role, is +// the head's output. Three heads are bundled. +// - Hebbian successor counts and their lookup +// - a direct role learned from a corpus: role = bundle over t of +// bind(summary(window before t), char_hv(corpus[t])), so that +// bind(summary, role) decodes to the next character; generation slides +// the window; refinement adds the terms the role still gets wrong. +// resonatorTrainStep (FF1/FF2 roles) is not written: nothing here says what +// those roles are, and a function returning 0.0 is not a training step. module ForwardPass { - // ======================================================================== - // IMPORTS - Reference existing specs, DO NOT DUPLICATE - // ======================================================================== - use base::types; // Trit enum - use numeric::gf16; // GF16 for similarity scores - use ternary::packed_trit; // PackedBigInt for HV storage + use base::types; // ======================================================================== // 1. Constants // ======================================================================== // Number of attention heads in multi-head architecture - pub const HEAD_COUNT : u8 = 3; + pub const HEAD_COUNT : usize = 3; - // Forward pass version - pub const FORWARD_VERSION : u32 = 2; + // Forward pass version: 3 = dense VSA rewrite with hard attention + pub const FORWARD_VERSION : u32 = 3; // Role vector dimension for VSA operations - pub const ROLE_DIM : u16 = 500; + pub const ROLE_DIM : usize = 500; // Context window size for autoregression - pub const CONTEXT_WINDOW : u8 = 8; + pub const CONTEXT_WINDOW : usize = 8; // Number of printable ASCII characters for Hebbian pub const HEBBIAN_CHARS : usize = 95; @@ -33,372 +49,631 @@ module ForwardPass { pub const HEBBIAN_OFFSET : usize = 32; // Maximum passes for refinement - pub const MAX_REFINE_PASSES : u8 = 3; + pub const MAX_REFINE_PASSES : usize = 3; // ======================================================================== - // 2. Role Vector Type + // 2. Types and storage // ======================================================================== - // RoleVector: Pre-computed attention role vectors - // Used for forward pass: Q/K/V queries with similarity scoring - // Stores role as PackedBigInt for VSA operations + // RoleVector: an attention role (Q, K, V per head) or a learned direct role pub struct RoleVector { - role_id : u8, // Role identifier (0-10) - name : []const u8, // Role name (e.g., "Q", "K", "V") - vector : PackedBigInt, // VSA vector representation + role_id : u8, + vector : [ROLE_DIM]i8, } + // counts[a * HEBBIAN_CHARS + b] = how often printable char b follows a + var hebbian : [HEBBIAN_CHARS * HEBBIAN_CHARS]u16 = undefined; + + var acc : [ROLE_DIM]i32 = undefined; + var summary_hv : [ROLE_DIM]i8 = undefined; + var query_hv : [ROLE_DIM]i8 = undefined; + var pos_hv : [ROLE_DIM]i8 = undefined; + var key_hv : [ROLE_DIM]i8 = undefined; + var bound_hv : [ROLE_DIM]i8 = undefined; + var char_buf : [ROLE_DIM]i8 = undefined; + var head_out : [HEAD_COUNT * ROLE_DIM]i8 = undefined; + // The context position each head attended to in the last multiHeadAttention + var head_pos : [HEAD_COUNT]usize = undefined; + var gen_window : [CONTEXT_WINDOW]u8 = undefined; + // ======================================================================== - // 3. HyperVector Type + // 3. Hypervector operations // ======================================================================== - // HyperVector: Abstract vector type for neural operations - // Stores role vectors, computes attention, bundling, similarity - // Platform-agnostic: implementation may use SIMD or scalar operations - pub struct HyperVector { - data : []PackedBigInt, // Internal role vectors - dim : usize, // Vector dimension + // Trit i of character c: +1 or -1 from three rounds of squaring modulo the + // prime 1000003. A single linear hash would make two characters' vectors + // shifted copies of one bit stream, and some pairs would correlate strongly. + fn char_trit(c: u8, i: usize) -> i8 { + const x1 = ((@as(usize, c) + 1) * 7919 + i * 104729 + 12345) % 1000003; + const x2 = (x1 * x1 + @as(usize, c)) % 1000003; + const x3 = (x2 * x2 + i) % 1000003; + if ((x3 / 16) % 2 == 0) { + return 1; + } + return -1; } - // ======================================================================== - // 4. Forward Pass Output Type - // ======================================================================== + fn char_hv(c: u8, out: []i8) -> void { + var i : usize = 0; + while (i < ROLE_DIM) { + out[i] = char_trit(c, i); + i = i + 1; + } + } - // ForwardOutput: Result of forward pass operation - // Contains output HV and optional loss metrics - pub struct ForwardOutput { - output : PackedBigInt, // Output hypervector - loss : ?f64, // Optional loss for training mode + fn sign_of(x: i32) -> i8 { + if (x > 0) { + return 1; + } + if (x < 0) { + return -1; + } + return 0; } - // Shared result for the currently unimplemented forward/attention paths. - // Preserve the existing empty vector and absent loss; this is not inference. - fn emptyForwardOutput() -> ForwardOutput { - return ForwardOutput{ - .output = PackedBigInt{}, - .loss = null, - }; + // Elementwise product; for +-1 vectors bind(bind(a, b), b) == a + fn bind(a: []const i8, b: []const i8, out: []i8) -> void { + var i : usize = 0; + while (i < ROLE_DIM) { + out[i] = a[i] * b[i]; + i = i + 1; + } } - // ======================================================================== - // 5. Attention Functions - // ======================================================================== + // Majority of three + fn bundle3(a: []const i8, b: []const i8, c: []const i8, out: []i8) -> void { + var i : usize = 0; + while (i < ROLE_DIM) { + out[i] = sign_of(@as(i32, a[i]) + @as(i32, b[i]) + @as(i32, c[i])); + i = i + 1; + } + } + + // Cyclic shift by `shift` places: out[(i + shift) % ROLE_DIM] = a[i] + fn permute(a: []const i8, shift: usize, out: []i8) -> void { + var i : usize = 0; + while (i < ROLE_DIM) { + out[(i + shift) % ROLE_DIM] = a[i]; + i = i + 1; + } + } + + // Cosine similarity; 0 when either vector is all zeros + fn similarity(a: []const i8, b: []const i8) -> f64 { + var dot : i32 = 0; + var na : i32 = 0; + var nb : i32 = 0; + var i : usize = 0; + while (i < ROLE_DIM) { + dot = dot + @as(i32, a[i]) * @as(i32, b[i]); + na = na + @as(i32, a[i]) * @as(i32, a[i]); + nb = nb + @as(i32, b[i]) * @as(i32, b[i]); + i = i + 1; + } + if (na == 0 or nb == 0) { + return 0.0; + } + return @as(f64, @floatFromInt(dot)) / sqrt(@as(f64, @floatFromInt(na)) * @as(f64, @floatFromInt(nb))); + } - // singleHeadAttention(positioned: []RoleVector, q_role: &RoleVector, k_role: &RoleVector, v_role: &RoleVector) -> RoleVector - // Single-head attention: query = bind(Q, K), score similarity with V - // Returns best-matching role vector as value - // Complexity: O(n) where n = ROLE_DIM (VSA bind + similarity) - pub fn singleHeadAttention(positioned: []RoleVector, q_role: &RoleVector, k_role: &RoleVector, v_role: &RoleVector) -> RoleVector { - var result = RoleVector{ - .role_id = v_role.role_id, - .name = v_role.name, - .vector = v_role.vector, - }; - return result; + // Character c at position p: permute(char_hv(c), p) + fn positioned_hv(c: u8, p: usize, out: []i8) -> void { + var i : usize = 0; + while (i < ROLE_DIM) { + out[(i + p) % ROLE_DIM] = char_trit(c, i); + i = i + 1; + } } - // multiHeadAttention(context: []HyperVector, roles: []RoleVector, num_heads: usize) -> ForwardOutput - // Multi-head attention: 3 independent heads merged via bundle3 - // Computes separate Q/K/V attention heads, merges via bundle3 - // Complexity: O(n * h) where n = ROLE_DIM, h = HEAD_COUNT - pub fn multiHeadAttention(context: []HyperVector, roles: []RoleVector, num_heads: usize) -> ForwardOutput { - return emptyForwardOutput(); + fn sign_into(out: []i8) -> void { + var i : usize = 0; + while (i < ROLE_DIM) { + out[i] = sign_of(acc[i]); + i = i + 1; + } } // ======================================================================== - // 6. Forward Pass Functions + // 4. Context and decoding // ======================================================================== - // forwardPass(context: []HyperVector, roles: []RoleVector, target: &HyperVector) -> ForwardOutput - // v2.29 single-head forward pass - // Uses Q/K/V attention to compute value = bind(Q/K/V) - // Complexity: O(n) where n = ROLE_DIM (VSA operations) - pub fn forwardPass(context: []HyperVector, roles: []RoleVector, target: &HyperVector) -> ForwardOutput { - return emptyForwardOutput(); + // Bundle of the positioned context characters (ties become 0) + pub fn summarizeContext(context: []const u8, out: []i8) -> void { + var i : usize = 0; + while (i < ROLE_DIM) { + acc[i] = 0; + i = i + 1; + } + var p : usize = 0; + while (p < context.len) { + i = 0; + while (i < ROLE_DIM) { + acc[(i + p) % ROLE_DIM] = acc[(i + p) % ROLE_DIM] + @as(i32, char_trit(context[p], i)); + i = i + 1; + } + p = p + 1; + } + sign_into(out); } - // forwardPassMultiHead(context: []HyperVector, roles: []RoleVector) -> ForwardOutput - // v2.30 multi-head forward pass (3 heads merged) - // Uses independent Q/K/V attention heads, merges via bundle3 - // Complexity: O(n * h) where n = ROLE_DIM, h = HEAD_COUNT - pub fn forwardPassMultiHead(context: []HyperVector, roles: []RoleVector) -> ForwardOutput { - return emptyForwardOutput(); + // The printable character whose hypervector is most similar to hv + pub fn directDecode(hv: []const i8) -> u8 { + var best : usize = 0; + var best_score : f64 = -2.0; + var b : usize = 0; + while (b < HEBBIAN_CHARS) { + const c : u8 = @intCast(HEBBIAN_OFFSET + b); + char_hv(c, &char_buf); + const s = similarity(hv, &char_buf); + if (s > best_score) { + best_score = s; + best = b; + } + b = b + 1; + } + return @intCast(HEBBIAN_OFFSET + best); } // ======================================================================== - // 7. Training Functions + // 5. Attention // ======================================================================== - // resonatorTrainStep(context: []HyperVector, target: &HyperVector, roles: []RoleVector, dim: usize) -> f64 - // Train FF1 and FF2 roles for one step - // Uses bind-based targeted correction - // Returns loss (1 - similarity) for this sample - // Complexity: O(n * h) where n = ROLE_DIM, h = 2 (FF roles) - pub fn resonatorTrainStep(context: []HyperVector, target: &HyperVector, roles: []RoleVector, dim: usize) -> f64 { - return 0.0; + // Hard attention: score_p = similarity(query, bind(positioned_p, k_role)); + // out = bind(positioned_best, v_role). Returns the best position. + pub fn singleHeadAttention(context: []const u8, query: []const i8, k_role: *const RoleVector, v_role: *const RoleVector, out: []i8) -> usize { + var best : usize = 0; + var best_score : f64 = -2.0; + var p : usize = 0; + while (p < context.len) { + positioned_hv(context[p], p, &pos_hv); + bind(&pos_hv, &k_role.vector, &key_hv); + const s = similarity(query, &key_hv); + if (s > best_score) { + best_score = s; + best = p; + } + p = p + 1; + } + positioned_hv(context[best], best, &pos_hv); + bind(&pos_hv, &v_role.vector, out); + return best; + } + + // Head h reads roles[3h] (Q), roles[3h + 1] (K), roles[3h + 2] (V), with + // query bind(summary, Q); the three head outputs are bundled into out + pub fn multiHeadAttention(context: []const u8, roles: []const RoleVector, out: []i8) -> void { + summarizeContext(context, &summary_hv); + var h : usize = 0; + while (h < HEAD_COUNT) { + bind(&summary_hv, &roles[3 * h].vector, &query_hv); + const ho = h * ROLE_DIM; + head_pos[h] = singleHeadAttention(context, &query_hv, &roles[3 * h + 1], &roles[3 * h + 2], head_out[ho..ho + ROLE_DIM]); + h = h + 1; + } + bundle3(head_out[0..ROLE_DIM], head_out[ROLE_DIM..2 * ROLE_DIM], head_out[2 * ROLE_DIM..3 * ROLE_DIM], out); } - // refineDirectRole(corpus: []const u8, dim: usize, initial_role: &RoleVector, num_passes: usize) -> RoleVector - // Iteratively refine direct role using error measurement - // Measures error, computes sparse correction, blends with sparsified annealing - // Complexity: O(p * n * m) where p = num_passes, n = dim, m = CORPUS_SIZE - pub fn refineDirectRole(corpus: []const u8, dim: usize, initial_role: &RoleVector, num_passes: usize) -> RoleVector { - var result = RoleVector{ - .role_id = initial_role.role_id, - .name = initial_role.name, - .vector = initial_role.vector, - }; - return result; + // Multi-head forward pass into out; returns the loss 1 - similarity(out, target) + pub fn forwardPass(context: []const u8, roles: []const RoleVector, target: []const i8, out: []i8) -> f64 { + multiHeadAttention(context, roles, out); + return 1.0 - similarity(out, target); } // ======================================================================== - // 8. Context Functions + // 6. Hebbian successor counts // ======================================================================== - // summarizeContext(context: []HyperVector) -> HyperVector - // Permute and bundle 8 context vectors into 1 summary HV - // Uses positional permutation then sequential bundling - // Complexity: O(n) where n = ROLE_DIM (permute + bundle operations) - pub fn summarizeContext(context: []HyperVector) -> HyperVector { - var result = HyperVector{ - .data = [], - .dim = ROLE_DIM, - }; - return result; + fn printable(c: u8) -> bool { + return @as(usize, c) >= HEBBIAN_OFFSET and @as(usize, c) < HEBBIAN_OFFSET + HEBBIAN_CHARS; } - // ======================================================================== - // 9. Direct Role Functions - // ======================================================================== + pub fn buildHebbianCounts(corpus: []const u8) -> void { + var i : usize = 0; + while (i < HEBBIAN_CHARS * HEBBIAN_CHARS) { + hebbian[i] = 0; + i = i + 1; + } + var t : usize = 1; + while (t < corpus.len) { + if (printable(corpus[t - 1]) and printable(corpus[t])) { + const k = (@as(usize, corpus[t - 1]) - HEBBIAN_OFFSET) * HEBBIAN_CHARS + (@as(usize, corpus[t]) - HEBBIAN_OFFSET); + hebbian[k] = hebbian[k] + 1; + } + t = t + 1; + } + } + + pub fn hebbianCount(a: u8, b: u8) -> u16 { + return hebbian[(@as(usize, a) - HEBBIAN_OFFSET) * HEBBIAN_CHARS + (@as(usize, b) - HEBBIAN_OFFSET)]; + } - // computeDirectRole(corpus: []const u8, dim: usize, offsets: []const usize, context_size: usize) -> RoleVector - // Pre-compute ideal direct role from corpus - // Uses charToHV encoding and bundle operations - // Complexity: O(c * n) where c = context_size, n = dim - pub fn computeDirectRole(corpus: []const u8, dim: usize, offsets: []const usize, context_size: usize) -> RoleVector { - var result = RoleVector{ - .role_id = 0, - .name = "direct", - .vector = PackedBigInt{}, - }; - return result; - } - - // directDecode(context: []HyperVector, role: &RoleVector, dim: usize) -> []u8 - // Decode output HV to character sequence - // Uses learned role to predict next character - // Complexity: O(n) where n = output length - pub fn directDecode(context: []HyperVector, role: &RoleVector, dim: usize) -> []u8 { } + // Successors of char_idx (0-based printable index) bundled with their counts as weights + pub fn hebbianLookup(char_idx: usize, out: []i8) -> void { + var i : usize = 0; + while (i < ROLE_DIM) { + acc[i] = 0; + i = i + 1; + } + var b : usize = 0; + while (b < HEBBIAN_CHARS) { + const n = @as(i32, hebbian[char_idx * HEBBIAN_CHARS + b]); + if (n > 0) { + const c : u8 = @intCast(HEBBIAN_OFFSET + b); + i = 0; + while (i < ROLE_DIM) { + acc[i] = acc[i] + n * @as(i32, char_trit(c, i)); + i = i + 1; + } + } + b = b + 1; + } + sign_into(out); + } // ======================================================================== - // 10. Generation Functions + // 7. Direct role // ======================================================================== - // generateWithDirectRole(initial_context: []HyperVector, role: &RoleVector, output_buf: []u8, max_tokens: usize) -> usize - // Autoregressive generation using pre-computed direct role - // Shifts context, predicts tokens using directDecode - // Complexity: O(t * n) where t = max_tokens, n = ROLE_DIM - pub fn generateWithDirectRole(initial_context: []HyperVector, role: &RoleVector, output_buf: []u8, max_tokens: usize) -> usize { } + // summarizeContext reuses acc, so the role accumulator is kept apart + var role_acc : [ROLE_DIM]i32 = undefined; - // ======================================================================== - // 11. Hebbian Functions - // ======================================================================== + fn add_role_term(corpus: []const u8, t: usize) -> void { + summarizeContext(corpus[t - CONTEXT_WINDOW..t], &summary_hv); + var i : usize = 0; + while (i < ROLE_DIM) { + role_acc[i] = role_acc[i] + @as(i32, summary_hv[i]) * @as(i32, char_trit(corpus[t], i)); + i = i + 1; + } + } + + fn role_from_acc(role: *RoleVector) -> void { + var i : usize = 0; + while (i < ROLE_DIM) { + role.vector[i] = sign_of(role_acc[i]); + i = i + 1; + } + } + + // The role that maps each window of the corpus to the character after it + pub fn computeDirectRole(corpus: []const u8, role: *RoleVector) -> void { + var i : usize = 0; + while (i < ROLE_DIM) { + role_acc[i] = 0; + i = i + 1; + } + var t : usize = CONTEXT_WINDOW; + while (t < corpus.len) { + add_role_term(corpus, t); + t = t + 1; + } + role.role_id = 0; + role_from_acc(role); + } + + // The next character after a window: decode(bind(summary(window), role)) + pub fn directPredict(window: []const u8, role: *const RoleVector) -> u8 { + summarizeContext(window, &summary_hv); + bind(&summary_hv, &role.vector, &bound_hv); + return directDecode(&bound_hv); + } - // buildHebbianCounts(corpus: []const u8) -> [HEBBIAN_CHARS][HEBBIAN_CHARS]u16 - // Build character-pair association matrix from corpus - // Returns counts[a][b] = frequency of char b following char a - // Complexity: O(c * m) where c = corpus.len, m = HEBBIAN_CHARS^2 - pub fn buildHebbianCounts(corpus: []const u8) -> [HEBBIAN_CHARS][HEBBIAN_CHARS]u16 { } + // Windows of the corpus whose next character the role gets wrong + pub fn directErrors(corpus: []const u8, role: *const RoleVector) -> usize { + var errors : usize = 0; + var t : usize = CONTEXT_WINDOW; + while (t < corpus.len) { + if (directPredict(corpus[t - CONTEXT_WINDOW..t], role) != corpus[t]) { + errors = errors + 1; + } + t = t + 1; + } + return errors; + } - // hebbianLookup(counts: *const [HEBBIAN_CHARS][HEBBIAN_CHARS]u16, char_idx: usize, dim: usize) -> HyperVector - // Look up character successor using Hebbian matrix - // Bundles successors weighted by count, proportional to frequency - // Complexity: O(n * m) where n = HEBBIAN_CHARS, m = dim - pub fn hebbianLookup(counts: *const [HEBBIAN_CHARS][HEBBIAN_CHARS]u16, char_idx: usize, dim: usize) -> HyperVector { } + // Perceptron passes: each window still predicted wrong adds its term again. + // Returns the errors left after the last pass. + pub fn refineDirectRole(corpus: []const u8, role: *RoleVector, num_passes: usize) -> usize { + computeDirectRole(corpus, role); + var pass : usize = 0; + while (pass < num_passes and pass < MAX_REFINE_PASSES) { + var t : usize = CONTEXT_WINDOW; + while (t < corpus.len) { + if (directPredict(corpus[t - CONTEXT_WINDOW..t], role) != corpus[t]) { + add_role_term(corpus, t); + } + t = t + 1; + } + role_from_acc(role); + pass = pass + 1; + } + return directErrors(corpus, role); + } + + // Autoregressive generation from the last CONTEXT_WINDOW characters of seed. + // Returns the number of characters written. + pub fn generateWithDirectRole(seed: []const u8, role: *const RoleVector, output_buf: []u8, max_tokens: usize) -> usize { + var i : usize = 0; + while (i < CONTEXT_WINDOW) { + gen_window[i] = seed[seed.len - CONTEXT_WINDOW + i]; + i = i + 1; + } + var n : usize = 0; + while (n < max_tokens and n < output_buf.len) { + const c = directPredict(&gen_window, role); + output_buf[n] = c; + i = 0; + while (i + 1 < CONTEXT_WINDOW) { + gen_window[i] = gen_window[i + 1]; + i = i + 1; + } + gen_window[CONTEXT_WINDOW - 1] = c; + n = n + 1; + } + return n; + } // ======================================================================== - // TDD - Tests + // 8. Test probes // ======================================================================== - test single_head_attention_returns_role - // Verify: singleHeadAttention returns valid role - given result = singleHeadAttention([], q_role, k_role, v_role) - then result.role_id == 10 + var probe_a : [ROLE_DIM]i8 = undefined; + var probe_b : [ROLE_DIM]i8 = undefined; + var probe_c : [ROLE_DIM]i8 = undefined; + var probe_d : [ROLE_DIM]i8 = undefined; + var probe_out : [ROLE_DIM]i8 = undefined; + var probe_roles : [3 * HEAD_COUNT]RoleVector = undefined; + var probe_role : RoleVector = undefined; + var probe_text : [16]u8 = undefined; + + fn char_similarity(a: u8, b: u8) -> f64 { + char_hv(a, &probe_a); + char_hv(b, &probe_b); + return similarity(&probe_a, &probe_b); + } - test summarize_context_creates_valid_hv - // Verify: summarizeContext bundles 8 vectors correctly - given result = summarizeContext(context) - then result.dim == ROLE_DIM + // similarity(bind(bind(a, b), b), a) + fn unbind_similarity() -> f64 { + char_hv('a', &probe_a); + char_hv('k', &probe_b); + bind(&probe_a, &probe_b, &probe_c); + bind(&probe_c, &probe_b, &probe_d); + return similarity(&probe_d, &probe_a); + } - test direct_decode_returns_chars - // Verify: directDecode returns character sequence - given result = directDecode(context, role, ROLE_DIM) - then result.len() == 10 + // similarity(bind(a, b), a): binding hides its inputs + fn bound_similarity() -> f64 { + char_hv('a', &probe_a); + char_hv('k', &probe_b); + bind(&probe_a, &probe_b, &probe_c); + return similarity(&probe_c, &probe_a); + } - test hebbian_counts_correct_for_corpus - // Verify: buildHebbianCounts counts pairs correctly - given corpus = "hello" - and counts = buildHebbianCounts(corpus) - then counts['h']['e'] == 1 and counts['l']['l'] == 2 - - test hebbian_lookup_returns_valid_hv - // Verify: hebbianLookup returns valid hypervector - given counts = buildHebbianCounts("ab") - and result = hebbianLookup(&counts, 0, ROLE_DIM) - then result.dim == ROLE_DIM - - test single_head_attention_returns_valid_role - // Verify: singleHeadAttention returns valid role vector - var q_role = RoleVector{.role_id = 1, .name = "Q", .vector = PackedBigInt{}} - var k_role = RoleVector{.role_id = 2, .name = "K", .vector = PackedBigInt{}} - var v_role = RoleVector{.role_id = 3, .name = "V", .vector = PackedBigInt{}} - given result = singleHeadAttention([], &q_role, &k_role, &v_role) - then result.role_id == 3 - - test multi_head_attention_returns_forward_output - // Verify: multiHeadAttention returns valid forward output - var context = [HyperVector{.data = [], .dim = ROLE_DIM}] - var roles = [RoleVector{.role_id = 1, .name = "Q", .vector = PackedBigInt{}}] - given result = multiHeadAttention(&context, &roles, 3) - then result.output == PackedBigInt{} and result.loss == null - - test forward_pass_returns_forward_output - // Verify: forwardPass returns valid forward output - var context = [HyperVector{.data = [], .dim = ROLE_DIM}] - var roles = [RoleVector{.role_id = 1, .name = "Q", .vector = PackedBigInt{}}] - var target = HyperVector{.data = [], .dim = ROLE_DIM} - given result = forwardPass(&context, &roles, &target) - then result.output == PackedBigInt{} and result.loss == null - - test forward_pass_multi_head_returns_forward_output - // Verify: forwardPassMultiHead returns valid forward output - var context = [HyperVector{.data = [], .dim = ROLE_DIM}] - var roles = [RoleVector{.role_id = 1, .name = "Q", .vector = PackedBigInt{}}] - given result = forwardPassMultiHead(&context, &roles) - then result.output == PackedBigInt{} and result.loss == null - - test resonator_train_step_returns_float - // Verify: resonatorTrainStep returns float value - var context = [HyperVector{.data = [], .dim = ROLE_DIM}] - var target = HyperVector{.data = [], .dim = ROLE_DIM} - var roles = [RoleVector{.role_id = 1, .name = "FF1", .vector = PackedBigInt{}}] - given result = resonatorTrainStep(&context, &target, &roles, ROLE_DIM) - then result == 0.0 - - test refine_direct_role_returns_role_vector - // Verify: refineDirectRole returns valid role vector - var initial_role = RoleVector{.role_id = 1, .name = "direct", .vector = PackedBigInt{}} - given result = refineDirectRole("test corpus", ROLE_DIM, &initial_role, 3) - then result.role_id == 1 and result.name == "direct" - - test summarize_context_returns_hyper_vector - // Verify: summarizeContext returns valid hypervector - var context = [HyperVector{.data = [], .dim = ROLE_DIM}] - given result = summarizeContext(&context) - then result.dim == ROLE_DIM - - test compute_direct_role_returns_role_vector - // Verify: computeDirectRole returns valid role vector - given result = computeDirectRole("test corpus", ROLE_DIM, [], 10) - then result.role_id == 0 and result.name == "direct" + // similarity(bundle3(a, b, c), input): 0.5 expected for each input + fn bundle_similarity(which: usize) -> f64 { + char_hv('x', &probe_a); + char_hv('y', &probe_b); + char_hv('z', &probe_c); + bundle3(&probe_a, &probe_b, &probe_c, &probe_d); + if (which == 0) { + return similarity(&probe_d, &probe_a); + } + if (which == 1) { + return similarity(&probe_d, &probe_b); + } + char_hv('w', &probe_a); + return similarity(&probe_d, &probe_a); + } - // ======================================================================== - // TDD - Invariants - // ======================================================================== + // shift = 0: similarity(permute(a, 1), a); shift = 1: permute back by ROLE_DIM - 1 + fn permute_similarity(back: bool) -> f64 { + char_hv('q', &probe_a); + permute(&probe_a, 1, &probe_b); + if (back) { + permute(&probe_b, ROLE_DIM - 1, &probe_c); + return similarity(&probe_c, &probe_a); + } + return similarity(&probe_b, &probe_a); + } + + fn decode_probe(c: u8) -> u8 { + char_hv(c, &probe_a); + return directDecode(&probe_a); + } + + // Context "hello"; the query is position 3 bound with the key role + fn attention_best() -> usize { + char_hv('K', &probe_roles[1].vector); + char_hv('V', &probe_roles[2].vector); + positioned_hv('l', 3, &probe_a); + bind(&probe_a, &probe_roles[1].vector, &probe_b); + return singleHeadAttention("hello", &probe_b, &probe_roles[1], &probe_roles[2], &probe_out); + } + + // The output, unbound from the value role, is the attended positioned vector + fn attention_value_similarity() -> f64 { + const best = attention_best(); + bind(&probe_out, &probe_roles[2].vector, &probe_c); + positioned_hv('l', best, &probe_a); + return similarity(&probe_c, &probe_a); + } + + // Position 2 holds 'l' as well, but at another shift: it must not match + fn attention_other_l_similarity() -> f64 { + if (attention_best() != 3) { + return 1.0; + } + bind(&probe_out, &probe_roles[2].vector, &probe_c); + positioned_hv('l', 2, &probe_a); + return similarity(&probe_c, &probe_a); + } + + fn setup_roles() -> void { + var r : usize = 0; + while (r < 3 * HEAD_COUNT) { + probe_roles[r].role_id = @intCast(r); + char_hv(@intCast(65 + r), &probe_roles[r].vector); + r = r + 1; + } + } + + // Similarity of the multi-head output with head h's output + fn multi_head_similarity(h: usize) -> f64 { + setup_roles(); + multiHeadAttention("the cat", &probe_roles, &probe_out); + return similarity(&probe_out, head_out[h * ROLE_DIM..(h + 1) * ROLE_DIM]); + } + + // Loss against the output itself, then against an unrelated vector + fn forward_loss(self_target: bool) -> f64 { + setup_roles(); + if (self_target) { + multiHeadAttention("the cat", &probe_roles, &probe_d); + return forwardPass("the cat", &probe_roles, &probe_d, &probe_out); + } + char_hv('~', &probe_d); + return forwardPass("the cat", &probe_roles, &probe_d, &probe_out); + } + + fn hebbian_probe(a: u8, b: u8) -> u16 { + buildHebbianCounts("hello"); + return hebbianCount(a, b); + } - invariant role_vector_dimension_matches - // Verify: All role vectors have consistent dimension - // ROLE_DIM must match across all operations - assert true; + // 'a' is followed by 'b' three times and by 'c' once + fn hebbian_next(c: u8) -> u8 { + buildHebbianCounts("abababac"); + hebbianLookup(@as(usize, c) - HEBBIAN_OFFSET, &probe_a); + return directDecode(&probe_a); + } + + const PERIODIC : []const u8 = "abcabcabcabcabcabcabc"; + const SENTENCE : []const u8 = "the cat sat on the mat and the rat sat on the hat by the cat"; + + fn periodic_prediction() -> u8 { + computeDirectRole(PERIODIC, &probe_role); + return directPredict(PERIODIC[0..CONTEXT_WINDOW], &probe_role); + } + + fn periodic_errors() -> usize { + computeDirectRole(PERIODIC, &probe_role); + return directErrors(PERIODIC, &probe_role); + } + + // Generate 5 characters after "abcabcab": expected "cabca" + fn generated(index: usize) -> u8 { + computeDirectRole(PERIODIC, &probe_role); + const n = generateWithDirectRole("abcabcab", &probe_role, &probe_text, 5); + if (n != 5) { + return 0; + } + return probe_text[index]; + } + + fn generated_count(max_tokens: usize) -> usize { + computeDirectRole(PERIODIC, &probe_role); + return generateWithDirectRole("abcabcab", &probe_role, &probe_text, max_tokens); + } - invariant forward_output_dim_consistency - // Verify: Forward output dimension matches ROLE_DIM - // Output HV must match configured dimension - assert true; + fn sentence_errors_before() -> usize { + computeDirectRole(SENTENCE, &probe_role); + return directErrors(SENTENCE, &probe_role); + } - invariant hebbian_matrix_size - // Verify: Hebbian matrix has correct dimensions - // Must be HEBBIAN_CHARS x HEBBIAN_CHARS - assert true; + fn sentence_errors_after(passes: usize) -> usize { + return refineDirectRole(SENTENCE, &probe_role, passes); + } - invariant context_window_fixed - // Verify: CONTEXT_WINDOW is constant - // Autoregressive sliding window must be fixed size - assert true; + // ======================================================================== + // TDD - Tests + // ======================================================================== - invariant forward_version_matches_spec - // Verify: FORWARD_VERSION corresponds to architecture - // Version 2 = multi-head with 3 heads - assert true; + test char_hv_is_quasi_orthogonal + given same = char_similarity('a', 'a') + when other = char_similarity('a', 'b') + then same == 1.0 and @abs(other) < 0.2 and @abs(char_similarity('h', 'e')) < 0.2 + + test bind_is_its_own_inverse + given back = unbind_similarity() + when hidden = bound_similarity() + then back == 1.0 and @abs(hidden) < 0.2 + + test bundle3_stays_similar_to_each_input + given s0 = bundle_similarity(0) + when s1 = bundle_similarity(1) + then s0 > 0.35 and s1 > 0.35 and @abs(bundle_similarity(2)) < 0.2 + + test permute_moves_and_returns + given moved = permute_similarity(false) + when back = permute_similarity(true) + then @abs(moved) < 0.2 and back == 1.0 + + test direct_decode_returns_the_encoded_char + given x = decode_probe('x') + when space = decode_probe(' ') + then x == 'x' and space == ' ' and decode_probe('~') == '~' + + test single_head_attention_finds_the_queried_position + given best = attention_best() + when value = attention_value_similarity() + then best == 3 and value == 1.0 and @abs(attention_other_l_similarity()) < 0.2 + + test multi_head_attention_bundles_three_heads + given s0 = multi_head_similarity(0) + when s2 = multi_head_similarity(2) + then s0 > 0.35 and s2 > 0.35 and multi_head_similarity(1) > 0.35 + + test forward_pass_loss_is_one_minus_similarity + given at_self = forward_loss(true) + when unrelated = forward_loss(false) + then at_self == 0.0 and unrelated > 0.8 and unrelated < 1.2 + + // "hello": h->e, e->l, l->l, l->o once each + test hebbian_counts_correct_for_corpus + given he = hebbian_probe('h', 'e') + when ll = hebbian_probe('l', 'l') + then he == 1 and ll == 1 and hebbian_probe('l', 'o') == 1 and hebbian_probe('o', 'h') == 0 + + test hebbian_lookup_decodes_the_most_frequent_successor + given after_a = hebbian_next('a') + when after_b = hebbian_next('b') + then after_a == 'b' and after_b == 'a' + + test direct_role_predicts_a_periodic_corpus + given next = periodic_prediction() + when errors = periodic_errors() + then next == 'c' and errors == 0 + + test generate_with_direct_role_continues_the_pattern + given c0 = generated(0) + when c4 = generated(4) + then c0 == 'c' and generated(1) == 'a' and generated(2) == 'b' and generated(3) == 'c' and c4 == 'a' + + test generate_with_direct_role_respects_the_buffer + given n = generated_count(40) + when m = generated_count(3) + then n == 16 and m == 3 + + // 51 windows of SENTENCE: the bundled role gets 16 wrong; three perceptron + // passes leave 12 (measured on the lab, 2026-10-18; the vectors are fixed, + // so the counts are exact) + test refine_direct_role_reduces_errors + given before = sentence_errors_before() + when after = sentence_errors_after(MAX_REFINE_PASSES) + then before == 16 and after == 12 // ======================================================================== - // TDD - Benchmarks + // TDD - Invariants // ======================================================================== - bench single_head_attention_latency - // Measure: cycles for single-head attention operation - // Target: < 5000 cycles (similarity scoring) - @setEvalBranchQuota(10000); - var result = singleHeadAttention([], q_role, k_role, v_role); - _ = result; - - bench multi_head_attention_latency - // Measure: cycles for multi-head attention operation - // Target: < 10000 cycles (3 heads + bundle) - @setEvalBranchQuota(10000); - var context = [role1, role2, role3]; - var roles = [q_role, k_role, v_role]; - _ = multiHeadAttention(&context, &roles, HEAD_COUNT); - _ = context; - - bench forward_pass_latency - // Measure: cycles for single-head forward pass - // Target: < 10000 cycles (attention + decode) - @setEvalBranchQuota(10000); - var context = [role1, role2, role3]; - var roles = [q_role, k_role, v_role]; - _ = forwardPass(&context, &roles, target_role); - _ = context; - - bench forward_pass_multi_head_latency - // Measure: cycles for multi-head forward pass - // Target: < 15000 cycles (3 attention heads + bundle) - @setEvalBranchQuota(10000); - var context = [role1, role2, role3]; - var roles = [q_role, k_role, v_role]; - _ = forwardPassMultiHead(&context, &roles, target_role); - _ = context; - - bench summarize_context_latency - // Measure: cycles for context summarization - // Target: < 3000 cycles (permute + 7 bundles) - @setEvalBranchQuota(10000); - var context = [role1, role2, role3, role4, role5, role6, role7, role8]; - _ = summarizeContext(&context); - _ = context; - - bench direct_decode_latency - // Measure: cycles for direct decode (10 tokens) - // Target: < 5000 cycles (10 character predictions) - @setEvalBranchQuota(10000); - var context = [role1, role2, role3, role4, role5, role6, role7, role8]; - _ = directDecode(&context, target_role, ROLE_DIM); - _ = context; - - bench hebbian_counts_build_latency - // Measure: cycles to build Hebbian counts from corpus - // Target: < 100000 cycles (corpus scan + count) - @setEvalBranchQuota(10000); - var corpus = "the quick brown fox jumps over the lazy dog"; - _ = buildHebbianCounts(corpus); - _ = corpus; - - bench hebbian_lookup_latency - // Measure: cycles for Hebbian character lookup - // Target: < 2000 cycles (weighted bundle) - @setEvalBranchQuota(10000); - var corpus = "the quick brown fox"; - var counts = buildHebbianCounts(corpus); - _ = hebbianLookup(&counts, 4, ROLE_DIM); - _ = counts; + invariant multi_head_has_three_heads + assert HEAD_COUNT == 3 + + invariant hebbian_covers_printable_ascii + assert HEBBIAN_OFFSET == 32 and HEBBIAN_OFFSET + HEBBIAN_CHARS == 127 + + invariant context_window_fits_a_role + assert CONTEXT_WINDOW > 0 and CONTEXT_WINDOW < ROLE_DIM + + invariant forward_version_is_the_dense_rewrite + assert FORWARD_VERSION == 3 } diff --git a/specs/igla/own_model.t27 b/specs/igla/own_model.t27 index 727eab2e72..42c1096680 100644 --- a/specs/igla/own_model.t27 +++ b/specs/igla/own_model.t27 @@ -19,22 +19,29 @@ ; nn/hslm (undeclared 'constants'), hslm/forward_pass ('q_role'), nn/attention ; ('PHI_CUBED_INV'), optimizer/adamw ('allocate_array'), and all nine specs/igla/coder/*.t27. ; ternary_matvec.t27 (the FPGA matvec, outside the 17) passed 4/4. +; - re-test on 2026-10-18, same t27c, after the PR that rewrote both HSLM specs (#7778): +; nn/hslm 26/26 and hslm/forward_pass 14/14 pass, 0 vacuous, so 6 of the 17 compile. The +; spec's backward pass is now real, each gradient checked against a finite difference. +; The spec is not the trained model: 6 layers at ctx 81, FFN 243-972-243 with GELU and no +; embedding or vocab head, where trinity-training ran 3 blocks, its best run at ctx 27, and an +; FFN 243-729-243 with ReLU, biases and a TWN scale. ; - live: GET huggingface.co/api/models/playra/tern-tc-9m and .../tern-tc-9m-t27 returned 200; ; .../tern-tc-100m returned 401, the same as a made-up name, so no 100M is public. ; GET board-production-2dec.up.railway.app/state returned 200 and a made-up path 404; its ; snapshot is dated 2026-10-05, so the board is live and 12 days stale. ; - review: no PR behind any cell has a review, so no DELIVER cell is VERIFIED. ; The engine asks an engineering LEARN step for a source link, a PRACTICE step for a passing test -; and a DELIVER step for a review. So: 1 published, 11 verified, 8 drafts, 7 gaps. +; and a DELIVER step for a review. So: 1 published, 12 verified, 7 drafts, 7 gaps. ; PERCENT is a second, separate number per cell: an estimate of how much of that part exists, with ; its basis in SOURCES. It never feeds a status. The lens and overall figures are plain means, every ; cell weighted the same; that weighting is a convention, not a measurement. ; THE ANSWER this map gives: HSLM was trained before, in Zig only (64 files); its best runs -; (PPL 2.96 at ctx 27) were wiped by EXP-002 and survive only in the ledger. In t27 the loss and -; three optimizers pass, the forward pass does not compile, and the backward pass, ternary STE, -; data loader, checkpoint I/O, heap and file I/O and `t27c --backend ml` do not exist yet. +; (PPL 2.96 at ctx 27) were wiped by EXP-002 and survive only in the ledger. In t27 the loss, three +; optimizers and the HSLM spec's forward and backward passes pass, but that spec is not the trained +; model (see MEASURED); ternary STE, data loader, checkpoint I/O, heap and file I/O and +; `t27c --backend ml` do not exist yet. ; Not claimed: no model quality beyond the numbers in SOURCES; no GIF or MP4, stills only. ; phi^2 + 1/phi^2 = 3 | TRINITY @@ -131,14 +138,14 @@ pub const SOURCES : [27]str = [ "t27 specs/ml/loss/cross_entropy_loss.t27", "t27 specs/ml/optimizer/{sgd,adam,lamb}.t27", "t27 specs/nn/hslm.t27 and specs/hslm/forward_pass.t27", - "t27 specs/nn/hslm.t27 backward placeholders", + "t27 specs/nn/hslm.t27 backward, finite-difference checked", "", "", "igla-coder-gpu research/MODEL_ON_FPGA_PLAN.md", "trinity-fpga b2bd5a30, 7682304/7682304 receipts", "board-production-2dec.up.railway.app/state", "t27 specs/port/trinity/fpga/openxc7-synth/ternary_matvec.t27", - "t27c test-report on 17 ML specs: 4 compile", + "t27c test-report on 17 ML specs: 6 compile", "", "", "", @@ -162,20 +169,20 @@ pub const MISSING : [27]str = [ /// The measurements per cell (see the header for how each was taken). pub const GROUNDED : [27]bool = [true, true, true, true, true, true, true, false, false, true, true, true, true, true, true, true, false, false, true, true, true, true, true, false, false, false, true]; -pub const TEST_PASS : [27]bool = [false, false, false, false, false, false, false, false, false, false, false, false, true, true, false, false, false, false, false, false, false, true, false, false, false, false, false]; +pub const TEST_PASS : [27]bool = [false, false, false, false, false, false, false, false, false, false, false, false, true, true, true, false, false, false, false, false, false, true, false, false, false, false, false]; pub const REVIEWED : [27]bool = [false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false]; pub const LIVE : [27]bool = [false, false, false, false, false, false, true, false, false, false, false, false, false, false, false, false, false, false, false, false, true, false, false, false, false, false, false]; /// What a host passes the viewer, per cell: the earned status code and the step number as a label. -pub const STATUS : [27]u8 = [4, 4, 4, 3, 3, 3, 3, 0, 0, 4, 4, 4, 4, 4, 3, 3, 0, 0, 4, 4, 5, 4, 3, 0, 0, 0, 3]; +pub const STATUS : [27]u8 = [4, 4, 4, 3, 3, 3, 3, 0, 0, 4, 4, 4, 4, 4, 4, 3, 0, 0, 4, 4, 5, 4, 3, 0, 0, 0, 3]; pub const LABELS : [27]str = ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27"]; /// Percent of each part that exists, estimated from its source (see the header). Never a status. -pub const PCT : [27]u8 = [60, 70, 80, 80, 60, 10, 50, 0, 0, 100, 100, 20, 80, 65, 35, 5, 0, 0, 70, 15, 60, 40, 23, 0, 0, 0, 50]; +pub const PCT : [27]u8 = [60, 70, 80, 80, 60, 10, 50, 0, 0, 100, 100, 20, 80, 65, 60, 40, 0, 0, 70, 15, 60, 40, 35, 0, 0, 0, 50]; /// The t27c test-report counts the PRACTICE cells quote. pub const ML_SPECS_MEASURED : i32 = 17; -pub const ML_SPECS_COMPILING : i32 = 4; +pub const ML_SPECS_COMPILING : i32 = 6; pub const CODER_SPECS : i32 = 9; pub const CODER_SPECS_COMPILING : i32 = 0; @@ -193,13 +200,13 @@ pub const ROWS : [20]str = [ "Step 11 of 27: the HSLM trainer", "64 Zig files; best PPL 2.96 at ctx 27", "Those checkpoints were wiped", - "In t27: loss and 3 optimizers pass", - "4 of 17 ML specs compile on the lab", + "In t27: loss, optimizers, HSLM pass", + "6 of 17 ML specs compile on the lab", "0 of 9 IGLA CODER specs compile", - "1 published, 11 verified, 8 drafts", - "7 gaps, about 39 percent overall", - "Make nn/hslm.t27 compile", - "Backward pass and ternary STE", + "1 published, 12 verified, 7 drafts", + "7 gaps, about 42 percent overall", + "Match the spec to the trained HSLM", + "Ternary STE, embedding and head", "Loader, checkpoint I/O, --backend ml", "" ]; @@ -267,14 +274,14 @@ test a_gap_is_named_not_filled { test a_status_is_earned_not_chosen { assert statuses_earned(STATUS, DOMAIN, GROUNDED, TEST_PASS, REVIEWED, LIVE); - // the evidence slide's "1 published, 11 verified, 8 drafts" and "7 gaps" + // the evidence slide's "1 published, 12 verified, 7 drafts" and "7 gaps" assert count_status(STATUS, STATUS_PUBLISHED) == 1; - assert count_status(STATUS, STATUS_VERIFIED) == 11; - assert count_status(STATUS, STATUS_DRAFT) == 8; + assert count_status(STATUS, STATUS_VERIFIED) == 12; + assert count_status(STATUS, STATUS_DRAFT) == 7; assert count_status(STATUS, STATUS_GAP) == 7; assert count_true(REVIEWED) == 0; - // every passing test is a PRACTICE step: loss, optimizers, matvec - assert count_true(TEST_PASS) == 3; + // every passing test is a PRACTICE step: loss, optimizers, forward pass, matvec + assert count_true(TEST_PASS) == 4; var c : i32 = 0; while (c < CELL_COUNT) { const i : usize = @intCast(c); @@ -297,7 +304,7 @@ test the_compile_counts_are_the_measured_ones { assert TEST_PASS[5] == false; // the compile cell's percent is the measured share, truncated const share : i32 = share_pct(ML_SPECS_COMPILING, ML_SPECS_MEASURED); - assert share == 23; + assert share == 35; const p22 : i32 = @intCast(PCT[22]); assert p22 == share; assert share_pct(CODER_SPECS_COMPILING, CODER_SPECS) == 0; @@ -306,10 +313,10 @@ test the_compile_counts_are_the_measured_ones { test percent_per_lens_is_a_plain_mean { assert mean_pct(0, 9) == 45; - assert mean_pct(9, 9) == 45; - assert mean_pct(18, 9) == 28; - // the evidence slide's "about 39 percent overall" - assert mean_pct(0, CELL_COUNT) == 39; + assert mean_pct(9, 9) == 51; + assert mean_pct(18, 9) == 30; + // the evidence slide's "about 42 percent overall" + assert mean_pct(0, CELL_COUNT) == 42; } test the_camera_is_the_demo_camera { diff --git a/specs/igla/own_model_board.t27 b/specs/igla/own_model_board.t27 index 144df03edb..abfb6f3639 100644 --- a/specs/igla/own_model_board.t27 +++ b/specs/igla/own_model_board.t27 @@ -48,23 +48,23 @@ pub const LENS_CELLS : i32 = 9; pub const JOB : str = "own_model"; pub const STAGE : str = "own_model"; -pub const MSG : str = "1 published, 11 verified, 8 drafts, 7 gaps; 39%"; +pub const MSG : str = "1 published, 12 verified, 7 drafts, 7 gaps; 42%"; /// What MSG says once its numbers are taken out. pub const MSG_WORDS : str = " published, verified, drafts, gaps; %"; -pub const OVERALL_PCT : u8 = 39; -pub const PROGRESS : f64 = 0.39; +pub const OVERALL_PCT : u8 = 42; +pub const PROGRESS : f64 = 0.42; pub const KEY_LENS : str = "lens_pct"; pub const KEY_CELLS : str = "cells"; pub const KEY_STATUS : str = "status"; pub const KEY_PCT : str = "pct"; -pub const LENS_PCT : [3]u8 = [45, 45, 28]; -pub const CELL_STATUS : [27]u8 = [4, 4, 4, 3, 3, 3, 3, 0, 0, 4, 4, 4, 4, 4, 3, 3, 0, 0, 4, 4, 5, 4, 3, 0, 0, 0, 3]; -pub const CELL_PCT : [27]u8 = [60, 70, 80, 80, 60, 10, 50, 0, 0, 100, 100, 20, 80, 65, 35, 5, 0, 0, 70, 15, 60, 40, 23, 0, 0, 0, 50]; +pub const LENS_PCT : [3]u8 = [45, 51, 30]; +pub const CELL_STATUS : [27]u8 = [4, 4, 4, 3, 3, 3, 3, 0, 0, 4, 4, 4, 4, 4, 4, 3, 0, 0, 4, 4, 5, 4, 3, 0, 0, 0, 3]; +pub const CELL_PCT : [27]u8 = [60, 70, 80, 80, 60, 10, 50, 0, 0, 100, 100, 20, 80, 65, 60, 40, 0, 0, 70, 15, 60, 40, 35, 0, 0, 0, 50]; /// The size of the compact metrics object, in bytes, computed from the keys and values above. -pub const METRICS_BYTES : i32 = 621; +pub const METRICS_BYTES : i32 = 622; pub fn is_digit(ch: u8) -> bool { return ch >= '0' and ch <= '9'; @@ -261,8 +261,8 @@ test the_tally_sums_to_twenty_seven { assert count_status(CELL_STATUS, STATUS_OPEN) == 0; assert published + verified + drafts + gaps == CELL_COUNT; assert published == 1; - assert verified == 11; - assert drafts == 8; + assert verified == 12; + assert drafts == 7; assert gaps == 7; } @@ -273,19 +273,19 @@ test the_message_is_the_tally { const n : i32 = @intCast(MSG.len); assert n <= MSG_MAX; // one number wrong, a word changed, or a number added: each refused - assert msg_matches("1 published, 12 verified, 8 drafts, 7 gaps; 39%", STATUS, overall) == false; - assert msg_matches("1 published, 11 verified, 8 drafts, 7 holes; 39%", STATUS, overall) == false; - assert msg_matches("1 published, 11 verified, 8 drafts, 7 gaps; 39% of 100", STATUS, overall) == false; - assert msg_matches(MSG, STATUS, 40) == false; - assert nth_number(MSG, 1) == 11; - assert nth_number(MSG, 4) == 39; + assert msg_matches("1 published, 11 verified, 7 drafts, 7 gaps; 42%", STATUS, overall) == false; + assert msg_matches("1 published, 12 verified, 7 drafts, 7 holes; 42%", STATUS, overall) == false; + assert msg_matches("1 published, 12 verified, 7 drafts, 7 gaps; 42% of 100", STATUS, overall) == false; + assert msg_matches(MSG, STATUS, 43) == false; + assert nth_number(MSG, 1) == 12; + assert nth_number(MSG, 4) == 42; assert nth_number(MSG, 5) == -1; } test progress_is_the_overall_percent_over_a_hundred { assert progress_matches(PROGRESS, mean_pct(0, CELL_COUNT)); // a bar one point off, or past the end of the board's 0..1, is refused - assert progress_matches(0.40, mean_pct(0, CELL_COUNT)) == false; + assert progress_matches(0.43, mean_pct(0, CELL_COUNT)) == false; assert progress_matches(1.39, 139) == false; assert progress_matches(-0.01, -1) == false; assert JOB.len > 0; diff --git a/specs/nn/hslm.t27 b/specs/nn/hslm.t27 index 5a7afd4a9d..f1795d2d54 100644 --- a/specs/nn/hslm.t27 +++ b/specs/nn/hslm.t27 @@ -1,730 +1,1029 @@ // SPDX-License-Identifier: Apache-2.0 // t27/specs/nn/hslm.t27 // HSLM (Hierarchical Sacred Learning Model) Specification -// Ternary neural network with sacred constants and VSA attention +// Ternary transformer: RMSNorm, causal multi-head attention with a KV cache, +// ternary feed-forward network, and the backward pass for one position. // φ² + 1/φ² = 3 | TRINITY +// +// This spec used to inline nn::attention, which reads constants::PHI_CUBED_INV; +// math::constants does not define it, so the generated Zig never compiled +// ("use of undeclared identifier 'constants'"). The layer is written out here +// instead, with the sacred attention scale 1 / HEAD_DIM^(phi^-3) as a number. +// +// The four backward functions used to fill gradients with zeros. They now +// compute them, and the tests check every gradient against a finite +// difference of the loss: a central difference with a small step for inputs +// and RMSNorm gains, and the -1 / +1 trit flip for ternary weights (exact for +// the last layer's W2, where the loss is linear in the weight). +// +// Measured against the trained model (gHashTag/trinity-training, +// src/hslm/constants.zig and trinity_block.zig), this spec does not describe +// it yet: +// - NUM_LAYERS 6 here; the trained model has DEFAULT_BLOCKS 3. +// - CONTEXT_LEN 81 here; the best run (PPL 2.96) trained at context 27. +// - The FFN here is 243 -> 972 -> 243 with GELU and no bias; the trained +// block is 243 -> 729 -> 243 with ReLU, a TWN alpha scale per matrix, and +// a bias after each projection. +// - No token embedding and no vocab-729 output head here. +// - Attention matches: 3 heads x 81, pre-RMSNorm, sacred scale, residual. module HSLM { - // Import base types, math, numeric, and attention use base::types; - use base::ops; - use math::constants; - use math::sacred_physics; - use numeric::gf16; - use nn::attention; // ═════════════════════════════════════════════════════════════════ // 1. HSLM Configuration - // ═════════════════════════════════════════════════════════════════════════ - - // TRINITY-based architecture - const NUM_LAYERS : usize = 6; // 6 transformer-like layers - const NUM_HEADS : usize = 3; // 3 attention heads per layer - const HEAD_DIM : usize = 81; // 81 dim per head (3⁴) - const EMBED_DIM : usize = 243; // 243 total embedding (3 × 81) - const FF_DIM : usize = 972; // 4 × EMBED_DIM (3⁴ × 4) - const CONTEXT_LEN : usize = 81; // Max sequence length + // ═════════════════════════════════════════════════════════════════ + + const NUM_LAYERS : usize = 6; // transformer layers + const NUM_HEADS : usize = 3; // attention heads per layer + const HEAD_DIM : usize = 81; // 3^4 per head + const EMBED_DIM : usize = 243; // NUM_HEADS * HEAD_DIM + const FF_DIM : usize = 972; // 4 * EMBED_DIM + const CONTEXT_LEN : usize = 81; // KV cache positions const VSA_DIM : usize = 1024; // VSA hypervector dimension + const RMS_EPS : f64 = 0.000001; + // 1 / 81^(phi^-3), phi^-3 = 0.2360679775 + const SACRED_ATTN_SCALE : f64 = 0.3543788557382518; + // sqrt(2 / pi) and the cubic term of the tanh GELU approximation + const SQRT_2_OVER_PI : f64 = 0.7978845608028654; + const GELU_CUBIC : f64 = 0.044715; + // Layer phases - const PHASE_NORM : u8 = 0; // Layer normalization - const PHASE_ATTN : u8 = 1; // Attention block - const PHASE_FFN : u8 = 2; // Feed-forward network - const PHASE_RESIDUAL : u8 = 3; // Residual connection - - // HSLM activation functions - const ACT_RELU : u8 = 0; // ReLU - const ACT_GELU : u8 = 1; // GELU - const ACT_SWISH : u8 = 2; // Swish - const ACT_TERNARY : u8 = 3; // Ternary (sign) - - // HSLM training phases - const PHASE_FORWARD : u8 = 0; // Forward pass - const PHASE_BACKWARD : u8 = 1; // Backward pass - const PHASE_UPDATE : u8 = 2; // Weight update + const PHASE_NORM : u8 = 0; + const PHASE_ATTN : u8 = 1; + const PHASE_FFN : u8 = 2; + const PHASE_RESIDUAL : u8 = 3; + + // Activation functions + const ACT_RELU : u8 = 0; + const ACT_GELU : u8 = 1; + const ACT_SWISH : u8 = 2; + const ACT_TERNARY : u8 = 3; + + // Training phases + const PHASE_FORWARD : u8 = 0; + const PHASE_BACKWARD : u8 = 1; + const PHASE_UPDATE : u8 = 2; + + // Weight matrices, for the test probes + const MAT_Q : u8 = 0; + const MAT_K : u8 = 1; + const MAT_V : u8 = 2; + const MAT_O : u8 = 3; + const MAT_W1 : u8 = 4; + const MAT_W2 : u8 = 5; // ═════════════════════════════════════════════════════════════════ - // 2. HSLM State - // ═════════════════════════════════════════════════════════════════════════ + // 2. Weights and state + // ═════════════════════════════════════════════════════════════════ + + // Ternary weights, row-major in_dim x out_dim: w[j * out_dim + i] + struct LayerWeights { + w_q : [EMBED_DIM * EMBED_DIM]Trit, + w_k : [EMBED_DIM * EMBED_DIM]Trit, + w_v : [EMBED_DIM * EMBED_DIM]Trit, + w_o : [EMBED_DIM * EMBED_DIM]Trit, + w1 : [EMBED_DIM * FF_DIM]Trit, + w2 : [FF_DIM * EMBED_DIM]Trit, + norm1_gamma : [EMBED_DIM]f64, + norm2_gamma : [EMBED_DIM]f64, + } + + struct HSLMWeights { + layers : [NUM_LAYERS]LayerWeights, + } - // Training/inference mode var hslm_mode : u8 = PHASE_FORWARD; - // Layer buffers (per position, per layer) - struct LayerBuffers { - input : [EMBED_DIM]f64, - output : [EMBED_DIM]f64, - temp : [EMBED_DIM]f64, // For residual connections - ffn_intermediate : [FF_DIM]f64, + // The model (about 4.3 MB of trits) lives in module storage, not on a stack + var model : HSLMWeights = undefined; + + // KV cache per layer: [layer][position][EMBED_DIM] + var kv_k : [NUM_LAYERS * CONTEXT_LEN * EMBED_DIM]f64 = undefined; + var kv_v : [NUM_LAYERS * CONTEXT_LEN * EMBED_DIM]f64 = undefined; + + // Activations of the last position each layer processed, kept for backward + var act_pos : [NUM_LAYERS]usize = undefined; + var act_x : [NUM_LAYERS * EMBED_DIM]f64 = undefined; // layer input + var act_n1 : [NUM_LAYERS * EMBED_DIM]f64 = undefined; // RMSNorm 1 output + var act_rms1 : [NUM_LAYERS]f64 = undefined; + var act_q : [NUM_LAYERS * EMBED_DIM]f64 = undefined; + var act_attn : [NUM_LAYERS * NUM_HEADS * CONTEXT_LEN]f64 = undefined; // softmax + var act_ctx : [NUM_LAYERS * EMBED_DIM]f64 = undefined; // attention context + var act_r : [NUM_LAYERS * EMBED_DIM]f64 = undefined; // input + attention + var act_n2 : [NUM_LAYERS * EMBED_DIM]f64 = undefined; // RMSNorm 2 output + var act_rms2 : [NUM_LAYERS]f64 = undefined; + var act_pre : [NUM_LAYERS * FF_DIM]f64 = undefined; // n2 @ W1 + var act_h : [NUM_LAYERS * FF_DIM]f64 = undefined; // gelu(pre) + var act_out : [NUM_LAYERS * EMBED_DIM]f64 = undefined; // layer output + + // Weight gradients (straight-through: d loss / d weight as if it were real), + // accumulated until zero_weight_gradients + var gw_q : [NUM_LAYERS * EMBED_DIM * EMBED_DIM]f64 = undefined; + var gw_k : [NUM_LAYERS * EMBED_DIM * EMBED_DIM]f64 = undefined; + var gw_v : [NUM_LAYERS * EMBED_DIM * EMBED_DIM]f64 = undefined; + var gw_o : [NUM_LAYERS * EMBED_DIM * EMBED_DIM]f64 = undefined; + var gw_1 : [NUM_LAYERS * EMBED_DIM * FF_DIM]f64 = undefined; + var gw_2 : [NUM_LAYERS * FF_DIM * EMBED_DIM]f64 = undefined; + var g_gamma1 : [NUM_LAYERS * EMBED_DIM]f64 = undefined; + var g_gamma2 : [NUM_LAYERS * EMBED_DIM]f64 = undefined; + + // Backward scratch for the layer being processed + var bw_out : [EMBED_DIM]f64 = undefined; // d loss / d layer output + var bw_h : [FF_DIM]f64 = undefined; // d loss / d gelu output + var bw_pre : [FF_DIM]f64 = undefined; // d loss / d W1 output + var bw_n : [EMBED_DIM]f64 = undefined; // d loss / d RMSNorm output + var bw_tmp : [EMBED_DIM]f64 = undefined; + var bw_r : [EMBED_DIM]f64 = undefined; // d loss / d (input + attention) + var bw_ctx : [EMBED_DIM]f64 = undefined; + var bw_q : [EMBED_DIM]f64 = undefined; + var bw_k : [EMBED_DIM]f64 = undefined; // current position's key + var bw_v : [EMBED_DIM]f64 = undefined; // current position's value + var bw_da : [CONTEXT_LEN]f64 = undefined; + var bw_x : [EMBED_DIM]f64 = undefined; // d loss / d layer input + + // ═════════════════════════════════════════════════════════════════ + // 3. Kernels + // ═════════════════════════════════════════════════════════════════ + + fn fill(x: []f64, value: f64) -> void { + var i : usize = 0; + while (i < x.len) { + x[i] = value; + i = i + 1; + } } - // Attention caches (KV cache for causal attention) - struct AttentionCache { - cache_k : [CONTEXT_LEN * EMBED_DIM]f64, - cache_v : [CONTEXT_LEN * EMBED_DIM]f64, + fn rms_of(x: []const f64) -> f64 { + var sum_squares : f64 = 0.0; + var i : usize = 0; + while (i < x.len) { + sum_squares = sum_squares + x[i] * x[i]; + i = i + 1; + } + return sqrt(sum_squares / @as(f64, @floatFromInt(x.len)) + RMS_EPS); } - // Layer weights (ternary) - struct LayerWeights { - w_q : [EMBED_DIM * EMBED_DIM]Trit, // Query projection - w_k : [EMBED_DIM * EMBED_DIM]Trit, // Key projection - w_v : [EMBED_DIM * EMBED_DIM]Trit, // Value projection - w_o : [EMBED_DIM * EMBED_DIM]Trit, // Output projection - w1 : [EMBED_DIM * FF_DIM]Trit, // FFN first projection - w2 : [FF_DIM * EMBED_DIM]Trit, // FFN second projection - norm1_gamma : [EMBED_DIM]f64, // RMSNorm scale - norm2_gamma : [EMBED_DIM]f64, // RMSNorm scale + // RMSNorm in place: x = x / sqrt(mean(x^2) + eps) * gamma + fn rms_norm_forward(x: []f64, gamma: []const f64) -> void { + const rms = rms_of(x); + var i : usize = 0; + while (i < x.len) { + x[i] = (x[i] / rms) * gamma[i]; + i = i + 1; + } } - // Full HSLM weights - struct HSLMWeights { - layers : [NUM_LAYERS]LayerWeights, + // RMSNorm into out; returns the rms, which backward needs + fn rms_norm_into(x: []const f64, gamma: []const f64, out: []f64) -> f64 { + const rms = rms_of(x); + var i : usize = 0; + while (i < x.len) { + out[i] = (x[i] / rms) * gamma[i]; + i = i + 1; + } + return rms; } - // ═════════════════════════════════════════════════════════════════ - // 3. HSLM Forward Pass - // ═════════════════════════════════════════════════════════════════════════ - - // hslm_forward(input, weights, seq_len, output, caches) → void - // Full forward pass through all HSLM layers - fn hslm_forward( - input: [][]f64, // [seq_len][EMBED_DIM] - weights: *HSLMWeights, - seq_len: usize, - output: [][]f64, // [seq_len][EMBED_DIM] - caches: []AttentionCache, // [NUM_LAYERS] - ) -> void { - var position : usize = 0; - - while (position < seq_len) { - // Initialize with input embedding - var layer_input = input[position]; - var layer_output : [EMBED_DIM]f64 = [0.0; EMBED_DIM]; - - // Process through all layers - var layer_idx : usize = 0; - - while (layer_idx < NUM_LAYERS) { - const layer_weights = &weights.layers[layer_idx]; - const layer_cache = &caches[layer_idx]; - - var buffers = LayerBuffers{ - .input = layer_input, - .output = [0.0; EMBED_DIM], - .temp = [0.0; EMBED_DIM], - .ffn_intermediate = [0.0; FF_DIM], - }; - - // Layer forward: norm -> attention -> residual -> norm -> ffn -> residual - hslm_layer_forward(&buffers, layer_weights, position, seq_len, layer_cache); - - layer_input = buffers.output; - layer_output = buffers.output; - - layer_idx = layer_idx + 1; + // y_i = g_i x_i / r: dx_k = g_k dy_k / r - x_k / (n r^3) * sum_i g_i dy_i x_i, + // dgamma_i += dy_i x_i / r + fn rms_norm_backward(x: []const f64, gamma: []const f64, rms: f64, dy: []const f64, dx: []f64, dgamma: []f64) -> void { + const n = @as(f64, @floatFromInt(x.len)); + var s : f64 = 0.0; + var i : usize = 0; + while (i < x.len) { + s = s + gamma[i] * dy[i] * x[i]; + i = i + 1; + } + i = 0; + while (i < x.len) { + dx[i] = gamma[i] * dy[i] / rms - x[i] * s / (n * rms * rms * rms); + dgamma[i] = dgamma[i] + dy[i] * x[i] / rms; + i = i + 1; + } + } + + // output[i] = sum_j input[j] * w[j * out_dim + i], trits as -1 / 0 / +1 + fn ternary_matmul(input: []const f64, weights: []const Trit, output: []f64, in_dim: usize, out_dim: usize) -> void { + var i : usize = 0; + while (i < out_dim) { + var acc : f64 = 0.0; + var j : usize = 0; + while (j < in_dim) { + const w = weights[j * out_dim + i]; + if (w == Trit.pos) { + acc = acc + input[j]; + } else if (w == Trit.neg) { + acc = acc - input[j]; + } + j = j + 1; + } + output[i] = acc; + i = i + 1; + } + } + + // The transpose, accumulated: dx[j] += sum_i w[j * out_dim + i] * dy[i] + fn ternary_matmul_t_add(dy: []const f64, weights: []const Trit, dx: []f64, in_dim: usize, out_dim: usize) -> void { + var j : usize = 0; + while (j < in_dim) { + var acc : f64 = 0.0; + var i : usize = 0; + while (i < out_dim) { + const w = weights[j * out_dim + i]; + if (w == Trit.pos) { + acc = acc + dy[i]; + } else if (w == Trit.neg) { + acc = acc - dy[i]; + } + i = i + 1; } + dx[j] = dx[j] + acc; + j = j + 1; + } + } - output[position] = layer_output; - position = position + 1; + // Weight gradient of a matmul: gw[j * out_dim + i] += x[j] * dy[i] + fn outer_add(x: []const f64, dy: []const f64, gw: []f64, in_dim: usize, out_dim: usize) -> void { + var j : usize = 0; + while (j < in_dim) { + var i : usize = 0; + while (i < out_dim) { + gw[j * out_dim + i] = gw[j * out_dim + i] + x[j] * dy[i]; + i = i + 1; + } + j = j + 1; } } - // hslm_layer_forward(buffers, weights, position, seq_len, cache) → void - // Single transformer layer: Attention + FFN with residual connections - fn hslm_layer_forward( - buffers: *LayerBuffers, - weights: *LayerWeights, - position: usize, - seq_len: usize, - cache: *AttentionCache, - ) -> void { - // Copy input to output for residual + // GELU, tanh approximation: 0.5 x (1 + tanh(sqrt(2/pi) (x + 0.044715 x^3))) + fn gelu(x: f64) -> f64 { + const inner = SQRT_2_OVER_PI * (x + GELU_CUBIC * x * x * x); + return 0.5 * x * (1.0 + tanh(inner)); + } + + // d gelu / dx = 0.5 (1 + t) + 0.5 x (1 - t^2) sqrt(2/pi) (1 + 3 * 0.044715 x^2) + fn gelu_derivative(x: f64) -> f64 { + const inner = SQRT_2_OVER_PI * (x + GELU_CUBIC * x * x * x); + const t = tanh(inner); + return 0.5 * (1.0 + t) + 0.5 * x * (1.0 - t * t) * SQRT_2_OVER_PI * (1.0 + 3.0 * GELU_CUBIC * x * x); + } + + fn gelu_activation(x: []f64) -> void { var i : usize = 0; - while (i < EMBED_DIM) { - buffers.output[i] = buffers.input[i]; + while (i < x.len) { + x[i] = gelu(x[i]); i = i + 1; } + } - // Step 1: RMSNorm before attention - rms_norm_forward(&buffers.output, weights.norm1_gamma); + // ═════════════════════════════════════════════════════════════════ + // 4. Forward pass (one position, causal, KV cache) + // ═════════════════════════════════════════════════════════════════ - // Save attention output for residual - i = 0; - while (i < EMBED_DIM) { - buffers.temp[i] = buffers.output[i]; - i = i + 1; + // q, k, v = n1 @ Wq, Wk, Wv; k and v go into the cache at `position`; + // per head, softmax(q . k_j * scale) over j <= position weights the v_j; + // out = context @ Wo + fn attention_forward(layer_idx: usize, position: usize, out: []f64) -> void { + const w = &model.layers[layer_idx]; + const eo = layer_idx * EMBED_DIM; + const ko = (layer_idx * CONTEXT_LEN + position) * EMBED_DIM; + ternary_matmul(act_n1[eo..eo + EMBED_DIM], &w.w_q, act_q[eo..eo + EMBED_DIM], EMBED_DIM, EMBED_DIM); + ternary_matmul(act_n1[eo..eo + EMBED_DIM], &w.w_k, kv_k[ko..ko + EMBED_DIM], EMBED_DIM, EMBED_DIM); + ternary_matmul(act_n1[eo..eo + EMBED_DIM], &w.w_v, kv_v[ko..ko + EMBED_DIM], EMBED_DIM, EMBED_DIM); + + var head : usize = 0; + while (head < NUM_HEADS) { + const ho = head * HEAD_DIM; + const ao = (layer_idx * NUM_HEADS + head) * CONTEXT_LEN; + var j : usize = 0; + while (j <= position) { + const kj = (layer_idx * CONTEXT_LEN + j) * EMBED_DIM + ho; + var dot : f64 = 0.0; + var d : usize = 0; + while (d < HEAD_DIM) { + dot = dot + act_q[eo + ho + d] * kv_k[kj + d]; + d = d + 1; + } + act_attn[ao + j] = dot * SACRED_ATTN_SCALE; + j = j + 1; + } + var max_score : f64 = act_attn[ao]; + j = 1; + while (j <= position) { + if (act_attn[ao + j] > max_score) { + max_score = act_attn[ao + j]; + } + j = j + 1; + } + var sum : f64 = 0.0; + j = 0; + while (j <= position) { + act_attn[ao + j] = exp(act_attn[ao + j] - max_score); + sum = sum + act_attn[ao + j]; + j = j + 1; + } + j = 0; + while (j <= position) { + act_attn[ao + j] = act_attn[ao + j] / sum; + j = j + 1; + } + var c : usize = 0; + while (c < HEAD_DIM) { + var acc : f64 = 0.0; + var m : usize = 0; + while (m <= position) { + acc = acc + act_attn[ao + m] * kv_v[(layer_idx * CONTEXT_LEN + m) * EMBED_DIM + ho + c]; + m = m + 1; + } + act_ctx[eo + ho + c] = acc; + c = c + 1; + } + head = head + 1; } + ternary_matmul(act_ctx[eo..eo + EMBED_DIM], &w.w_o, out, EMBED_DIM, EMBED_DIM); + } - // Step 2: Multi-head sacred attention - attention::sacred_attention_kernel( - buffers.output, - weights.w_q, weights.w_k, weights.w_v, weights.w_o, - position, seq_len, buffers.output, - cache.cache_k, cache.cache_v, - ); + // f = gelu(n2 @ W1) @ W2 + fn ffn_forward(layer_idx: usize, out: []f64) -> void { + const w = &model.layers[layer_idx]; + const eo = layer_idx * EMBED_DIM; + const fo = layer_idx * FF_DIM; + ternary_matmul(act_n2[eo..eo + EMBED_DIM], &w.w1, act_pre[fo..fo + FF_DIM], EMBED_DIM, FF_DIM); + var i : usize = 0; + while (i < FF_DIM) { + act_h[fo + i] = gelu(act_pre[fo + i]); + i = i + 1; + } + ternary_matmul(act_h[fo..fo + FF_DIM], &w.w2, out, FF_DIM, EMBED_DIM); + } - // Step 3: Residual connection (attention + input) + // r = x + attention(rms_norm(x)); out = r + ffn(rms_norm(r)) + fn hslm_layer_forward(layer_idx: usize, position: usize, input: []const f64, output: []f64) -> void { + const w = &model.layers[layer_idx]; + const eo = layer_idx * EMBED_DIM; + act_pos[layer_idx] = position; + var i : usize = 0; + while (i < EMBED_DIM) { + act_x[eo + i] = input[i]; + i = i + 1; + } + act_rms1[layer_idx] = rms_norm_into(input, &w.norm1_gamma, act_n1[eo..eo + EMBED_DIM]); + attention_forward(layer_idx, position, output); i = 0; while (i < EMBED_DIM) { - buffers.output[i] = buffers.output[i] + buffers.input[i]; + act_r[eo + i] = input[i] + output[i]; i = i + 1; } - - // Save attention output for second residual + act_rms2[layer_idx] = rms_norm_into(act_r[eo..eo + EMBED_DIM], &w.norm2_gamma, act_n2[eo..eo + EMBED_DIM]); + ffn_forward(layer_idx, output); i = 0; while (i < EMBED_DIM) { - buffers.input[i] = buffers.output[i]; + output[i] = act_r[eo + i] + output[i]; i = i + 1; } + } - // Step 4: RMSNorm before FFN - rms_norm_forward(&buffers.output, weights.norm2_gamma); + // All layers at one position; earlier positions must already be in the cache + fn hslm_forward(input: []const f64, position: usize, output: []f64) -> void { + hslm_layer_forward(0, position, input, act_out[0..EMBED_DIM]); + var layer_idx : usize = 1; + while (layer_idx < NUM_LAYERS) { + const prev = (layer_idx - 1) * EMBED_DIM; + const cur = layer_idx * EMBED_DIM; + hslm_layer_forward(layer_idx, position, act_out[prev..prev + EMBED_DIM], act_out[cur..cur + EMBED_DIM]); + layer_idx = layer_idx + 1; + } + const last = (NUM_LAYERS - 1) * EMBED_DIM; + var i : usize = 0; + while (i < EMBED_DIM) { + output[i] = act_out[last + i]; + i = i + 1; + } + } - // Step 5: Feed-forward network - ffn_forward(&buffers, weights); + // ═════════════════════════════════════════════════════════════════ + // 5. Backward pass (the last position each layer processed) + // ═════════════════════════════════════════════════════════════════ + // + // The gradient is that of the current position's output. Keys and values + // of earlier positions are cached constants: no gradient flows into them. + // The `grads` argument of the backward functions carries the ternary weights + // the gradient flows back through; the f64 gradients land in gw_*, g_gamma*. + + fn zero_weight_gradients() -> void { + fill(&gw_q, 0.0); + fill(&gw_k, 0.0); + fill(&gw_v, 0.0); + fill(&gw_o, 0.0); + fill(&gw_1, 0.0); + fill(&gw_2, 0.0); + fill(&g_gamma1, 0.0); + fill(&g_gamma2, 0.0); + } - // Step 6: Residual connection (ffn + attention_output) + // grad_input = d loss / d input, given grad_output = d loss / d output + fn hslm_backward(grad_output: []const f64, grad_input: []f64) -> void { + var i : usize = 0; + while (i < EMBED_DIM) { + bw_out[i] = grad_output[i]; + i = i + 1; + } + var layer_idx : usize = NUM_LAYERS; + while (layer_idx > 0) { + layer_idx = layer_idx - 1; + hslm_layer_backward(&model, layer_idx); + i = 0; + while (i < EMBED_DIM) { + bw_out[i] = bw_x[i]; + i = i + 1; + } + } i = 0; while (i < EMBED_DIM) { - buffers.output[i] = buffers.output[i] + buffers.input[i]; + grad_input[i] = bw_x[i]; i = i + 1; } } - // ═════════════════════════════════════════════════════════════════ - // 4. RMS Normalization - // ═════════════════════════════════════════════════════════════════════════ - - // rms_norm_forward(x, gamma) → void (in-place) - // RMSNorm: output = (input / sqrt(mean(input²) + eps)) * gamma - // Uses sacred gamma from math/sacred_physics - fn rms_norm_forward(x: []f64, gamma: []f64) -> void { - // Compute mean of squares - var sum_squares : f64 = 0.0; - var i : usize = 0; + // bw_out -> bw_x through FFN, RMSNorm 2, attention, RMSNorm 1 and both residuals + fn hslm_layer_backward(grads: *HSLMWeights, layer_idx: usize) -> void { + ffn_backward(grads, layer_idx); + attention_backward(grads, layer_idx); + } - while (i < x.len()) { - sum_squares = sum_squares + x[i] * x[i]; + // bw_out -> bw_r = bw_out + d loss / d r through the FFN and RMSNorm 2 + fn ffn_backward(grads: *HSLMWeights, layer_idx: usize) -> void { + const w = &grads.layers[layer_idx]; + const eo = layer_idx * EMBED_DIM; + ffn_backward_w2(grads, layer_idx); + gelu_backward(layer_idx); + ffn_backward_w1(grads, layer_idx); + rms_norm_backward(act_r[eo..eo + EMBED_DIM], &w.norm2_gamma, act_rms2[layer_idx], &bw_n, &bw_tmp, g_gamma2[eo..eo + EMBED_DIM]); + var i : usize = 0; + while (i < EMBED_DIM) { + bw_r[i] = bw_out[i] + bw_tmp[i]; i = i + 1; } + } - // rms = sqrt(mean + eps) - const mean = sum_squares / (x.len() as f64); - const rms = sqrt(mean + 1e-6); + // f = h @ W2: dW2 += h (x) bw_out, bw_h = W2^T bw_out + fn ffn_backward_w2(grads: *HSLMWeights, layer_idx: usize) -> void { + const w = &grads.layers[layer_idx]; + const fo = layer_idx * FF_DIM; + const go = layer_idx * FF_DIM * EMBED_DIM; + outer_add(act_h[fo..fo + FF_DIM], &bw_out, gw_2[go..go + FF_DIM * EMBED_DIM], FF_DIM, EMBED_DIM); + fill(&bw_h, 0.0); + ternary_matmul_t_add(&bw_out, &w.w2, &bw_h, FF_DIM, EMBED_DIM); + } - // Normalize and scale by gamma - i = 0; - while (i < x.len()) { - x[i] = (x[i] / rms) * gamma[i]; + // h = gelu(pre): bw_pre = gelu'(pre) * bw_h + fn gelu_backward(layer_idx: usize) -> void { + const fo = layer_idx * FF_DIM; + var i : usize = 0; + while (i < FF_DIM) { + bw_pre[i] = gelu_derivative(act_pre[fo + i]) * bw_h[i]; i = i + 1; } } - // ═════════════════════════════════════════════════════════════════ - // 5. Feed-Forward Network - // ═════════════════════════════════════════════════════════════════════════ - - // ffn_forward(buffers, weights) → void - // Feed-forward network: Gelu(x @ W1) @ W2 - // Using ternary weights, sacred expansion - fn ffn_forward(buffers: *LayerBuffers, weights: *LayerWeights) -> void { - // Step 1: Project to FF_DIM: intermediate = input @ W1 - ternary_matmul(buffers.output, weights.w1, buffers.ffn_intermediate, EMBED_DIM, FF_DIM); - - // Step 2: Apply GELU activation - gelu_activation(buffers.ffn_intermediate); - - // Step 3: Project back to EMBED_DIM: output = intermediate @ W2 - ternary_matmul(buffers.ffn_intermediate, weights.w2, buffers.output, FF_DIM, EMBED_DIM); - } - - // ternary_matmul(input, weights, output, in_dim, out_dim) → void - // Matrix multiplication with ternary weights - fn ternary_matmul( - input: []f64, - weights: []Trit, - output: []f64, - in_dim: usize, - out_dim: usize, - ) -> void { - var i : usize = 0; + // pre = n2 @ W1: dW1 += n2 (x) bw_pre, bw_n = W1^T bw_pre + fn ffn_backward_w1(grads: *HSLMWeights, layer_idx: usize) -> void { + const w = &grads.layers[layer_idx]; + const eo = layer_idx * EMBED_DIM; + const go = layer_idx * EMBED_DIM * FF_DIM; + outer_add(act_n2[eo..eo + EMBED_DIM], &bw_pre, gw_1[go..go + EMBED_DIM * FF_DIM], EMBED_DIM, FF_DIM); + fill(&bw_n, 0.0); + ternary_matmul_t_add(&bw_pre, &w.w1, &bw_n, EMBED_DIM, FF_DIM); + } - while (i < out_dim) { - var acc : f64 = 0.0; + // bw_r -> bw_x: Wo, softmax, the current position's q, k, v, Wq/Wk/Wv, + // RMSNorm 1, and the residual around the block + fn attention_backward(grads: *HSLMWeights, layer_idx: usize) -> void { + const w = &grads.layers[layer_idx]; + const eo = layer_idx * EMBED_DIM; + const go = layer_idx * EMBED_DIM * EMBED_DIM; + const position = act_pos[layer_idx]; + + outer_add(act_ctx[eo..eo + EMBED_DIM], &bw_r, gw_o[go..go + EMBED_DIM * EMBED_DIM], EMBED_DIM, EMBED_DIM); + fill(&bw_ctx, 0.0); + ternary_matmul_t_add(&bw_r, &w.w_o, &bw_ctx, EMBED_DIM, EMBED_DIM); + + var head : usize = 0; + while (head < NUM_HEADS) { + const ho = head * HEAD_DIM; + const ao = (layer_idx * NUM_HEADS + head) * CONTEXT_LEN; + // da_j = dctx . v_j, then the softmax: ds_j = a_j (da_j - sum_m a_m da_m) + var mean_da : f64 = 0.0; var j : usize = 0; - - while (j < in_dim) { - const weight_val = weights[j * out_dim + i] as i8; - - if (weight_val == 1) { - acc = acc + input[j]; - } else if (weight_val == -1) { - acc = acc - input[j]; + while (j <= position) { + const vj = (layer_idx * CONTEXT_LEN + j) * EMBED_DIM + ho; + var dot : f64 = 0.0; + var d : usize = 0; + while (d < HEAD_DIM) { + dot = dot + bw_ctx[ho + d] * kv_v[vj + d]; + d = d + 1; } - + bw_da[j] = dot; + mean_da = mean_da + act_attn[ao + j] * dot; + j = j + 1; + } + j = 0; + while (j <= position) { + bw_da[j] = act_attn[ao + j] * (bw_da[j] - mean_da); j = j + 1; } + // dq = scale * sum_j ds_j k_j; dk_p = scale * ds_p q; dv_p = a_p dctx + var c : usize = 0; + while (c < HEAD_DIM) { + var acc : f64 = 0.0; + var m : usize = 0; + while (m <= position) { + acc = acc + bw_da[m] * kv_k[(layer_idx * CONTEXT_LEN + m) * EMBED_DIM + ho + c]; + m = m + 1; + } + bw_q[ho + c] = SACRED_ATTN_SCALE * acc; + bw_k[ho + c] = SACRED_ATTN_SCALE * bw_da[position] * act_q[eo + ho + c]; + bw_v[ho + c] = act_attn[ao + position] * bw_ctx[ho + c]; + c = c + 1; + } + head = head + 1; + } - output[i] = acc; + outer_add(act_n1[eo..eo + EMBED_DIM], &bw_q, gw_q[go..go + EMBED_DIM * EMBED_DIM], EMBED_DIM, EMBED_DIM); + outer_add(act_n1[eo..eo + EMBED_DIM], &bw_k, gw_k[go..go + EMBED_DIM * EMBED_DIM], EMBED_DIM, EMBED_DIM); + outer_add(act_n1[eo..eo + EMBED_DIM], &bw_v, gw_v[go..go + EMBED_DIM * EMBED_DIM], EMBED_DIM, EMBED_DIM); + fill(&bw_n, 0.0); + ternary_matmul_t_add(&bw_q, &w.w_q, &bw_n, EMBED_DIM, EMBED_DIM); + ternary_matmul_t_add(&bw_k, &w.w_k, &bw_n, EMBED_DIM, EMBED_DIM); + ternary_matmul_t_add(&bw_v, &w.w_v, &bw_n, EMBED_DIM, EMBED_DIM); + + rms_norm_backward(act_x[eo..eo + EMBED_DIM], &w.norm1_gamma, act_rms1[layer_idx], &bw_n, &bw_tmp, g_gamma1[eo..eo + EMBED_DIM]); + var i : usize = 0; + while (i < EMBED_DIM) { + bw_x[i] = bw_r[i] + bw_tmp[i]; i = i + 1; } } - // gelu_activation(x) → void (in-place) - // GELU: x * Φ(x) where Φ is standard normal CDF - // Approximation: x * 0.5 * (1 + tanh(√(2/π) * (x + 0.044715x³))) - fn gelu_activation(x: []f64) -> void { - const sqrt_2_over_pi = sqrt(2.0 / 3.141592653589793); - var i : usize = 0; + // ═════════════════════════════════════════════════════════════════ + // 6. Phase management + // ═════════════════════════════════════════════════════════════════ - while (i < x.len()) { - const val = x[i]; - const cube = val * val * val; - const inner = sqrt_2_over_pi * (val + 0.044715 * cube); - const tanh_val = tanh(inner); - x[i] = 0.5 * val * (1.0 + tanh_val); - i = i + 1; + // An unknown phase leaves the mode as it was + fn hslm_phase(phase: u8) -> void { + if (phase == PHASE_FORWARD or phase == PHASE_BACKWARD or phase == PHASE_UPDATE) { + hslm_mode = phase; } } + fn get_hslm_mode() -> u8 { + return hslm_mode; + } + + // ═════════════════════════════════════════════════════════════════ + // 7. Test probes // ═════════════════════════════════════════════════════════════════ - // 6. HSLM Backward Pass - // ═════════════════════════════════════════════════════════════════════════ - - // hslm_backward(grad_output, weights, seq_len, grad_input) → void - // Full backward pass with gradient computation - fn hslm_backward( - grad_output: [][]f64, // [seq_len][EMBED_DIM] - weights: *HSLMWeights, - seq_len: usize, - grad_input: [][]f64, // [seq_len][EMBED_DIM] - ) -> void { - // Initialize weight gradients to zero - var weight_grads = zero_weight_gradients(); - - // Backward through layers (reverse order) - var layer_idx : usize = NUM_LAYERS; - while (layer_idx > 0) { - layer_idx = layer_idx - 1; - // Layer backward: FFN + Attention gradients - hslm_layer_backward(&weight_grads, layer_idx); + var probe_x : [EMBED_DIM]f64 = undefined; + var probe_out : [EMBED_DIM]f64 = undefined; + var probe_up : [EMBED_DIM]f64 = undefined; + var probe_gin : [EMBED_DIM]f64 = undefined; + + // About 2 in 9 trits nonzero, a different pattern per matrix and layer. + // A multiplicative hash mod a prime: 243 and 972 are multiples of 9, so a + // linear pattern mod 9 repeats down every column (measured: whole columns + // of one trit, and weight gradients that were exactly zero). + fn trit_pattern(seed: usize) -> Trit { + const m = ((seed * 2654435761) % 1000003) % 9; + if (m == 0) { + return Trit.pos; + } + if (m == 1) { + return Trit.neg; } + return Trit.zero; } - // zero_weight_gradients() → HSLMWeights - // Initialize all weight gradients to zero - fn zero_weight_gradients() -> HSLMWeights { - var weights : HSLMWeights = undefined; - var layer : usize = 0; + fn init_matrix(m: []Trit, salt: usize) -> void { + var i : usize = 0; + while (i < m.len) { + m[i] = trit_pattern(i + salt); + i = i + 1; + } + } - while (layer < NUM_LAYERS) { - // All ternary weight gradients stored as f64 for accumulation + // Small gains keep the attention scores and GELU inputs near 1, where the + // softmax and GELU are far from saturated + fn init_model(gain: f64) -> void { + var l : usize = 0; + while (l < NUM_LAYERS) { + const w = &model.layers[l]; + const salt = l * 1000003; + init_matrix(&w.w_q, salt + 11); + init_matrix(&w.w_k, salt + 23); + init_matrix(&w.w_v, salt + 37); + init_matrix(&w.w_o, salt + 41); + init_matrix(&w.w1, salt + 53); + init_matrix(&w.w2, salt + 67); var i : usize = 0; - while (i < EMBED_DIM * EMBED_DIM) { - // In real implementation, would use proper gradient storage + while (i < EMBED_DIM) { + w.norm1_gamma[i] = gain * (1.0 + 0.01 * @as(f64, @floatFromInt(i % 7))); + w.norm2_gamma[i] = gain * (1.0 + 0.01 * @as(f64, @floatFromInt(i % 5))); i = i + 1; } - layer = layer + 1; + l = l + 1; } - - return weights; } - // hslm_layer_backward(grads, layer_idx) → void - // Single layer backward: FFN + Attention gradients - fn hslm_layer_backward(grads: *HSLMWeights, layer_idx: usize) -> void { - // Gradient flows backward through: residual -> ffn -> norm -> residual -> attn -> norm - // FFN backward (with residual) - ffn_backward(grads, layer_idx); - - // Attention backward (with residual) - attention_backward(grads, layer_idx); + fn set_trit(which: u8, layer_idx: usize, index: usize, t: Trit) -> void { + const w = &model.layers[layer_idx]; + if (which == MAT_Q) { + w.w_q[index] = t; + } else if (which == MAT_K) { + w.w_k[index] = t; + } else if (which == MAT_V) { + w.w_v[index] = t; + } else if (which == MAT_O) { + w.w_o[index] = t; + } else if (which == MAT_W1) { + w.w1[index] = t; + } else { + w.w2[index] = t; + } } - // ffn_backward(grads, layer_idx) → void - // Backward through FFN: compute gradients for W2, W1, and input - fn ffn_backward(grads: *HSLMWeights, layer_idx: usize) -> void { - // Step 1: Gradient through second projection (W2) - ffn_backward_w2(grads, layer_idx); - - // Step 2: Gradient through GELU activation - gelu_backward(layer_idx); + fn weight_gradient(which: u8, layer_idx: usize, index: usize) -> f64 { + if (which == MAT_Q) { + return gw_q[layer_idx * EMBED_DIM * EMBED_DIM + index]; + } + if (which == MAT_K) { + return gw_k[layer_idx * EMBED_DIM * EMBED_DIM + index]; + } + if (which == MAT_V) { + return gw_v[layer_idx * EMBED_DIM * EMBED_DIM + index]; + } + if (which == MAT_O) { + return gw_o[layer_idx * EMBED_DIM * EMBED_DIM + index]; + } + if (which == MAT_W1) { + return gw_1[layer_idx * EMBED_DIM * FF_DIM + index]; + } + return gw_2[layer_idx * FF_DIM * EMBED_DIM + index]; + } - // Step 3: Gradient through first projection (W1) - ffn_backward_w1(grads, layer_idx); + // The upstream gradient: loss = sum_i out_i * upstream(i) + fn upstream(i: usize) -> f64 { + return 0.01 * @as(f64, @floatFromInt((i * 5) % 11)) - 0.05; } - // ffn_backward_w2(grads, layer_idx) → void - // Gradient through second FFN projection - fn ffn_backward_w2(grads: *HSLMWeights, layer_idx: usize) -> void { - // Gradient: dL/dW2 = dL/dout * x^T - // dL/dx = W2^T * dL/dout - // Implementation: accumulate gradients in weight structure - var layer_grads = &grads.layers[layer_idx]; - - // For now, initialize gradients to zero (placeholder for actual gradient computation) - // In a full implementation, this would compute dL/dW2 from upstream gradients - // and stored intermediate activations from the forward pass + // Run one position with input element k moved by delta + fn probe_run(position: usize, k: usize, delta: f64) -> void { var i : usize = 0; - while (i < FF_DIM * EMBED_DIM) { - // Initialize gradient to zero - actual computation would use stored activations - layer_grads.w2[i] = 0.0; // Placeholder for gradient computation + while (i < EMBED_DIM) { + probe_x[i] = 0.1 * @as(f64, @floatFromInt((i * 3 + position * 7) % 13)) - 0.6; i = i + 1; } + probe_x[k] = probe_x[k] + delta; + hslm_forward(&probe_x, position, &probe_out); } - // gelu_backward(layer_idx) → void - // Gradient through GELU activation - fn gelu_backward(layer_idx: usize) -> void { - // GELU derivative: dGELU/dx = Φ(x) + x * φ(x) - // where φ is standard normal PDF - // Implementation: compute derivative using the same approximation as forward pass - const sqrt_2_over_pi = sqrt(2.0 / 3.141592653589793); - const sqrt_2_pi = sqrt(2.0 * 3.141592653589793); - - // For now, this is a placeholder implementation - // In a full implementation, this would: - // 1. Access stored intermediate activations from forward pass - // 2. Compute dGELU/dx for each activation - // 3. Multiply by upstream gradients - // 4. Pass gradients backward to previous layer - - // Placeholder: compute derivative values (would use stored activations in real implementation) + // The same, returning the loss + fn probe_loss(position: usize, k: usize, delta: f64) -> f64 { + probe_run(position, k, delta); + var loss : f64 = 0.0; var i : usize = 0; - while (i < FF_DIM) { - // This would access stored intermediate values and compute gradients - // For now, just a placeholder computation - const x = 0.0; // Would be stored intermediate activation - const cube = x * x * x; - const inner = sqrt_2_over_pi * (x + 0.044715 * cube); - const tanh_val = tanh(inner); - const gelu_derivative = 0.5 * (1.0 + tanh_val) + 0.5 * x * (1.0 - tanh_val * tanh_val) * sqrt_2_over_pi * (1.0 + 0.134145 * cube); - - // Store or apply gradient (placeholder) + while (i < EMBED_DIM) { + loss = loss + probe_out[i] * upstream(i); i = i + 1; } + return loss; } - // ffn_backward_w1(grads, layer_idx) → void - // Gradient through first FFN projection - fn ffn_backward_w1(grads: *HSLMWeights, layer_idx: usize) -> void { - // Gradient: dL/dW1 = dL/dmid * x^T - // dL/dx = W1^T * dL/dmid - // Implementation: accumulate gradients in weight structure - var layer_grads = &grads.layers[layer_idx]; - - // For now, initialize gradients to zero (placeholder for actual gradient computation) - // In a full implementation, this would: - // 1. Access downstream gradients (dL/dmid from gelu_backward) - // 2. Access original input activations from forward pass - // 3. Compute dL/dW1 = dL/dmid * x^T using ternary matrix multiplication pattern + // Model at gain 0.05 with position 0 in the cache; position 1 is probed + fn probe_setup() -> void { + init_model(0.05); + probe_run(0, 0, 0.0); + } + + // Forward position 1 at the unperturbed point, then backward from upstream + fn probe_backward() -> void { + probe_run(1, 0, 0.0); + zero_weight_gradients(); var i : usize = 0; - while (i < EMBED_DIM * FF_DIM) { - // Initialize gradient to zero - actual computation would use downstream gradients - // and stored input activations from forward pass - layer_grads.w1[i] = 0.0; // Placeholder for gradient computation + while (i < EMBED_DIM) { + probe_up[i] = upstream(i); i = i + 1; } + hslm_backward(&probe_up, &probe_gin); } - // attention_backward(grads, layer_idx) → void - // Gradient through attention block - fn attention_backward(grads: *HSLMWeights, layer_idx: usize) -> void { - // Gradient flows through: output projection -> attention weights -> Q/K/V projections - // Implementation: compute gradients for attention weights and projections - var layer_grads = &grads.layers[layer_idx]; - - // For now, initialize gradients to zero (placeholder for actual gradient computation) - // In a full implementation, this would: - // 1. Access downstream gradients from output projection - // 2. Compute gradients for attention weights (softmax derivatives) - // 3. Compute gradients for Q, K, V projections - // 4. Pass gradients backward to input embeddings - - // Initialize output projection gradients - var i : usize = 0; - while (i < EMBED_DIM * EMBED_DIM) { - layer_grads.w_o[i] = 0.0; // Placeholder for output projection gradients - i = i + 1; + fn input_slope(k: usize) -> f64 { + probe_setup(); + return (probe_loss(1, k, 0.0001) - probe_loss(1, k, -0.0001)) / 0.0002; + } + + fn input_gradient(k: usize) -> f64 { + probe_setup(); + probe_backward(); + return probe_gin[k]; + } + + // Flip one trit to +1 and -1 (position 0 cached with it at 0): + // (loss(+1) - loss(-1)) / 2 + fn weight_slope(which: u8, layer_idx: usize, index: usize) -> f64 { + init_model(0.05); + set_trit(which, layer_idx, index, Trit.zero); + probe_run(0, 0, 0.0); + set_trit(which, layer_idx, index, Trit.pos); + const lp = probe_loss(1, 0, 0.0); + set_trit(which, layer_idx, index, Trit.neg); + const lm = probe_loss(1, 0, 0.0); + return (lp - lm) / 2.0; + } + + fn weight_grad_at(which: u8, layer_idx: usize, index: usize) -> f64 { + init_model(0.05); + set_trit(which, layer_idx, index, Trit.zero); + probe_run(0, 0, 0.0); + probe_backward(); + return weight_gradient(which, layer_idx, index); + } + + fn gamma_slope(second: bool, layer_idx: usize, index: usize) -> f64 { + probe_setup(); + const w = &model.layers[layer_idx]; + var g0 : f64 = w.norm1_gamma[index]; + if (second) { + g0 = w.norm2_gamma[index]; } - - // Initialize attention projection gradients - i = 0; - while (i < EMBED_DIM * EMBED_DIM) { - layer_grads.w_q[i] = 0.0; // Placeholder for query projection gradients - layer_grads.w_k[i] = 0.0; // Placeholder for key projection gradients - layer_grads.w_v[i] = 0.0; // Placeholder for value projection gradients - i = i + 1; + if (second) { + w.norm2_gamma[index] = g0 + 0.00001; + } else { + w.norm1_gamma[index] = g0 + 0.00001; } + const lp = probe_loss(1, 0, 0.0); + if (second) { + w.norm2_gamma[index] = g0 - 0.00001; + } else { + w.norm1_gamma[index] = g0 - 0.00001; + } + const lm = probe_loss(1, 0, 0.0); + return (lp - lm) / 0.00002; } - // ═════════════════════════════════════════════════════════════════ - // 7. Phase Management - // ═════════════════════════════════════════════════════════════════════════ + fn gamma_gradient(second: bool, layer_idx: usize, index: usize) -> f64 { + probe_setup(); + probe_backward(); + if (second) { + return g_gamma2[layer_idx * EMBED_DIM + index]; + } + return g_gamma1[layer_idx * EMBED_DIM + index]; + } - // hslm_phase(phase: u8) → void - // Switch HSLM phase (forward, backward, update) - // Used for streaming inference and training - fn hslm_phase(phase: u8) -> void { - if (phase == PHASE_FORWARD) { - hslm_mode = 1; - } else if (phase == PHASE_BACKWARD) { - hslm_mode = 2; - } else if (phase == PHASE_UPDATE) { - hslm_mode = 3; + // gelu_backward on layer 2 with pre = x and an upstream gradient of 3 + fn gelu_backward_at(x: f64) -> f64 { + act_pre[2 * FF_DIM + 7] = x; + bw_h[7] = 3.0; + gelu_backward(2); + return bw_pre[7]; + } + + fn rms_norm_probe(index: usize) -> f64 { + var x : [4]f64 = [1.0, 2.0, 3.0, 4.0]; + const gamma : [4]f64 = [1.0, 1.0, 1.0, 2.0]; + rms_norm_forward(&x, &gamma); + return x[index]; + } + + fn rms_norm_zero_probe() -> f64 { + var x : [3]f64 = [0.0, 0.0, 0.0]; + const gamma : [3]f64 = [1.0, 1.0, 1.0]; + rms_norm_forward(&x, &gamma); + return @abs(x[0]) + @abs(x[1]) + @abs(x[2]); + } + + fn matmul_probe(negate: bool, index: usize) -> f64 { + const input : [3]f64 = [1.0, 2.0, 3.0]; + var weights : [9]Trit = [Trit.pos, Trit.zero, Trit.zero, Trit.zero, Trit.pos, Trit.zero, Trit.zero, Trit.zero, Trit.pos]; + if (negate) { + var i : usize = 0; + while (i < 9) { + weights[i] = Trit.neg; + i = i + 1; + } } + var output : [3]f64 = [9.0, 9.0, 9.0]; + ternary_matmul(&input, &weights, &output, 3, 3); + return output[index]; } - // get_hslm_mode() → u8 - // Get current HSLM mode - fn get_hslm_mode() -> u8 { - return hslm_mode; + // A 3 x 2 matrix: output[i] = sum_j input[j] w[j * 2 + i] + fn matmul_rect_probe(index: usize) -> f64 { + const input : [3]f64 = [1.0, 2.0, 3.0]; + const weights : [6]Trit = [Trit.pos, Trit.neg, Trit.zero, Trit.pos, Trit.neg, Trit.neg]; + var output : [2]f64 = [0.0, 0.0]; + ternary_matmul(&input, &weights, &output, 3, 2); + return output[index]; + } + + // Sum of the softmax weights of layer 0, head 1, after position 1 + fn attention_weight_sum() -> f64 { + probe_setup(); + probe_run(1, 0, 0.0); + const ao = (0 * NUM_HEADS + 1) * CONTEXT_LEN; + return act_attn[ao] + act_attn[ao + 1]; + } + + fn attention_weight(j: usize) -> f64 { + probe_setup(); + probe_run(1, 0, 0.0); + return act_attn[(0 * NUM_HEADS + 1) * CONTEXT_LEN + j]; + } + + // At position 0 a head can only attend to itself: its context is v_0 + fn self_attention_gap() -> f64 { + init_model(0.05); + probe_run(0, 0, 0.0); + var gap : f64 = 0.0; + var i : usize = 0; + while (i < EMBED_DIM) { + gap = gap + @abs(act_ctx[3 * EMBED_DIM + i] - kv_v[(3 * CONTEXT_LEN + 0) * EMBED_DIM + i]); + i = i + 1; + } + return gap; + } + + // The output of position 1 depends on what position 0 left in the cache + fn context_effect() -> f64 { + init_model(0.05); + probe_run(0, 0, 0.0); + const with_first = probe_loss(1, 0, 0.0); + probe_run(0, 0, 0.5); + const with_other_first = probe_loss(1, 0, 0.0); + return with_first - with_other_first; + } + + fn phase_probe(phase: u8) -> u8 { + hslm_phase(PHASE_FORWARD); + hslm_phase(phase); + return get_hslm_mode(); } - // ═══════════════════════════════════════════════════════════════════════════════════════════════════════ + // ═════════════════════════════════════════════════════════════════ // TDD-Inside-Spec: Tests and Invariants for HSLM - // ═══════════════════════════════════════════════════════════════════════════════════════════════════════ - - test hslm_num_layers_is_six - given layers = NUM_LAYERS - then layers == 6 - - test hslm_num_heads_is_trinity - given heads = NUM_HEADS - then heads == 3 - - test hslm_head_dim_is_three_pow_four - given dim = HEAD_DIM - then dim == 81 // 3^4 - - test hslm_embed_dim_is_heads_times_head_dim - given embed = EMBED_DIM - and computed = NUM_HEADS * HEAD_DIM - then embed == computed - - test hslm_ff_dim_is_four_times_embed_dim - given ff_dim = FF_DIM - and computed = 4 * EMBED_DIM - then ff_dim == computed - - test hslm_context_len_is_eighty_one - given ctx = CONTEXT_LEN - then ctx == 81 - - test hslm_vsa_dim_is_1024 - given vsa = VSA_DIM - then vsa == 1024 - - test hslm_phase_constants_are_unique - given a = PHASE_NORM and b = PHASE_ATTN and c = PHASE_FFN and d = PHASE_RESIDUAL - then a != b and b != c and c != d - - test hslm_activation_constants_are_unique - given a = ACT_RELU and b = ACT_GELU and c = ACT_SWISH and d = ACT_TERNARY - then a != b and b != c and c != d - - test hslm_training_phase_constants_are_unique - given a = PHASE_FORWARD and b = PHASE_BACKWARD and c = PHASE_UPDATE - then a != b and b != c - - test hslm_phase_forward_sets_mode_one - given hslm_phase(PHASE_FORWARD) - and mode = get_hslm_mode() - then mode == 1 - - test hslm_phase_backward_sets_mode_two - given hslm_phase(PHASE_BACKWARD) - and mode = get_hslm_mode() - then mode == 2 - - test hslm_phase_update_sets_mode_three - given hslm_phase(PHASE_UPDATE) - and mode = get_hslm_mode() - then mode == 3 - - test hslm_rms_norm_preserves_shape - given x = [1.0, 2.0, 3.0, 4.0, 5.0] - and gamma = [1.0, 1.0, 1.0, 1.0, 1.0] - and len_before = x.len() - when rms_norm_forward(x, gamma) - then x.len() == len_before + // ═════════════════════════════════════════════════════════════════ + + test hslm_phase_switches_mode + given backward = phase_probe(PHASE_BACKWARD) + when update = phase_probe(PHASE_UPDATE) + then backward == PHASE_BACKWARD and update == PHASE_UPDATE and phase_probe(PHASE_FORWARD) == PHASE_FORWARD + + test hslm_phase_ignores_unknown + given mode = phase_probe(9) + then mode == PHASE_FORWARD + + // mean(1, 4, 9, 16) = 7.5, rms = sqrt(7.5 + 1e-6) = 2.7386130 + test hslm_rms_norm_scales_by_rms_and_gamma + given x0 = rms_norm_probe(0) + when x3 = rms_norm_probe(3) + then @abs(x0 - 0.3651483473) < 0.0000001 and @abs(x3 - 2.9211867786) < 0.0000001 test hslm_rms_norm_zero_input_returns_zero - given x = [0.0, 0.0, 0.0] - and gamma = [1.0, 1.0, 1.0] - when rms_norm_forward(x, gamma) - then x[0] == 0.0 and x[1] == 0.0 and x[2] == 0.0 + given total = rms_norm_zero_probe() + then total == 0.0 test hslm_ternary_matmul_identity - given input = [1.0, 2.0, 3.0] - and weights = [Trit.pos, Trit.zero, Trit.zero, - Trit.zero, Trit.pos, Trit.zero, - Trit.zero, Trit.zero, Trit.pos] - and output = [0.0; 3] - when ternary_matmul(input, weights, output, 3, 3) - then output[0] == 1.0 and output[1] == 2.0 and output[2] == 3.0 + given a = matmul_probe(false, 0) + when c = matmul_probe(false, 2) + then a == 1.0 and matmul_probe(false, 1) == 2.0 and c == 3.0 test hslm_ternary_matmul_negation - given input = [1.0, 2.0, 3.0] - and weights = [Trit.neg, Trit.neg, Trit.neg, - Trit.neg, Trit.neg, Trit.neg, - Trit.neg, Trit.neg, Trit.neg] - and output = [0.0; 3] - when ternary_matmul(input, weights, output, 3, 3) - then output[0] == -6.0 and output[1] == -6.0 and output[2] == -6.0 - - test hslm_gelu_activation_preserves_shape - given x = [1.0, 2.0, -1.0, 0.0] - and len_before = x.len() - when gelu_activation(x) - then x.len() == len_before - - test hslm_gelu_activation_positive_is_positive - given x = [1.0; 10] - when gelu_activation(x) - then x[0] > 0.0 - - test hslm_gelu_activation_zero_is_zero - given x = [0.0] - when gelu_activation(x) - then abs(x[0]) < 0.0001 - - test hslm_gelu_activation_negative_is_negative - given x = [-1.0] - when gelu_activation(x) - then x[0] < 0.0 - - test hslm_ffn_forward_intermediate_dimension - given buffers = LayerBuffers{...} - and weights = LayerWeights{...} - when buffers.output = [1.0; EMBED_DIM] - and ffn_forward(&buffers, &weights) - then buffers.ffn_intermediate.len() == FF_DIM - - test hslm_zero_weight_gradients_initializes_all - given grads = zero_weight_gradients() - then grads.layers.len() == NUM_LAYERS - - invariant hslm_num_layers_constant - assert NUM_LAYERS == 6 - - invariant hslm_num_heads_constant - assert NUM_HEADS == 3 - - invariant hslm_head_dim_constant - assert HEAD_DIM == 81 - - invariant hslm_embed_dim_constant - assert EMBED_DIM == 243 - - invariant hslm_ff_dim_constant - assert FF_DIM == 972 - - invariant hslm_context_len_constant - assert CONTEXT_LEN == 81 - - invariant hslm_vsa_dim_constant - assert VSA_DIM == 1024 + given a = matmul_probe(true, 0) + then a == -6.0 and matmul_probe(true, 2) == -6.0 + + // column 0: 1 - 3 = -2 (w = +1, 0, -1); column 1: -1 + 2 - 3 = -2 + test hslm_ternary_matmul_row_major_layout + given c0 = matmul_rect_probe(0) + when c1 = matmul_rect_probe(1) + then c0 == -2.0 and c1 == -2.0 + + test hslm_gelu_values + given at_one = gelu(1.0) + when at_minus_one = gelu(-1.0) + then @abs(at_one - 0.8411919906) < 0.0000001 and @abs(at_minus_one + 0.1588080094) < 0.0000001 and gelu(0.0) == 0.0 + + // #7778: the derivative against a central difference at -2.0, 0.0 and 1.5 + test hslm_gelu_derivative_matches_finite_difference + given at_minus_two = (gelu(-2.0 + 0.00001) - gelu(-2.0 - 0.00001)) / 0.00002 + when at_zero = (gelu(0.00001) - gelu(-0.00001)) / 0.00002 + then @abs(gelu_derivative(-2.0) - at_minus_two) < 0.000001 and @abs(gelu_derivative(0.0) - at_zero) < 0.000001 and @abs(gelu_derivative(1.5) - (gelu(1.5 + 0.00001) - gelu(1.5 - 0.00001)) / 0.00002) < 0.000001 + + test hslm_gelu_derivative_at_zero_is_one_half + given d = gelu_derivative(0.0) + then d == 0.5 + + test hslm_gelu_backward_scales_upstream_by_slope + given y = gelu_backward_at(1.5) + when slope = 3.0 * (gelu(1.5 + 0.00001) - gelu(1.5 - 0.00001)) / 0.00002 + then @abs(y - slope) < 0.000001 + + // gelu_backward multiplies the upstream gradient by gelu'(pre) + test hslm_gelu_backward_uses_gelu_derivative + given at_minus_two = gelu_backward_at(-2.0) + when at_zero = gelu_backward_at(0.0) + then @abs(at_minus_two - 3.0 * gelu_derivative(-2.0)) < 0.000000001 and at_zero == 1.5 + + test hslm_attention_softmax_sums_to_one + given total = attention_weight_sum() + when w0 = attention_weight(0) + then @abs(total - 1.0) < 0.000000000001 and w0 > 0.01 and w0 < 0.99 + + test hslm_attention_position_zero_reads_its_own_value + given gap = self_attention_gap() + then gap < 0.000000000001 + + test hslm_attention_reads_the_cache + given effect = context_effect() + then @abs(effect) > 0.000001 + + // Through all six layers: d loss / d input against a central difference + test hslm_backward_input_gradient_matches_finite_difference + given slope = input_slope(5) + when grad = input_gradient(5) + then @abs(grad - slope) < 0.000000001 * (1.0 + @abs(slope)) and @abs(slope) > 0.000001 + + test hslm_backward_input_gradient_second_element + given slope = input_slope(200) + when grad = input_gradient(200) + then @abs(grad - slope) < 0.000000001 * (1.0 + @abs(slope)) and @abs(slope) > 0.000001 + + // The loss is linear in the last layer's W2, so the trit flip is exact. + // Measured on the lab (2026-10-18): input and gain gradients agree with the + // central difference to 2e-11, W2 to 1e-16, the other trit flips to 7e-4 + // relative (the flip is a step of 1, so curvature shows). + test hslm_ffn_backward_w2_matches_trit_flip + given slope = weight_slope(MAT_W2, NUM_LAYERS - 1, 100 * EMBED_DIM + 7) + when grad = weight_grad_at(MAT_W2, NUM_LAYERS - 1, 100 * EMBED_DIM + 7) + then @abs(grad - slope) < 0.000000000001 and @abs(slope) > 0.000001 + + test hslm_ffn_backward_w1_matches_trit_flip + given slope = weight_slope(MAT_W1, NUM_LAYERS - 1, 9 * FF_DIM + 40) + when grad = weight_grad_at(MAT_W1, NUM_LAYERS - 1, 9 * FF_DIM + 40) + then @abs(grad - slope) < 0.002 * @abs(slope) and @abs(slope) > 0.0000005 + + test hslm_attention_backward_wo_matches_trit_flip + given slope = weight_slope(MAT_O, NUM_LAYERS - 1, 12 * EMBED_DIM + 30) + when grad = weight_grad_at(MAT_O, NUM_LAYERS - 1, 12 * EMBED_DIM + 30) + then @abs(grad - slope) < 0.002 * @abs(slope) and @abs(slope) > 0.0000005 + + test hslm_attention_backward_wq_matches_trit_flip + given slope = weight_slope(MAT_Q, NUM_LAYERS - 1, 20 * EMBED_DIM + 90) + when grad = weight_grad_at(MAT_Q, NUM_LAYERS - 1, 20 * EMBED_DIM + 90) + then @abs(grad - slope) < 0.002 * @abs(slope) and @abs(slope) > 0.0000005 + + test hslm_attention_backward_wk_matches_trit_flip + given slope = weight_slope(MAT_K, NUM_LAYERS - 1, 20 * EMBED_DIM + 90) + when grad = weight_grad_at(MAT_K, NUM_LAYERS - 1, 20 * EMBED_DIM + 90) + then @abs(grad - slope) < 0.002 * @abs(slope) and @abs(slope) > 0.0000005 + + test hslm_attention_backward_wv_matches_trit_flip + given slope = weight_slope(MAT_V, NUM_LAYERS - 1, 20 * EMBED_DIM + 170) + when grad = weight_grad_at(MAT_V, NUM_LAYERS - 1, 20 * EMBED_DIM + 170) + then @abs(grad - slope) < 0.002 * @abs(slope) and @abs(slope) > 0.0000005 + + // A first-layer weight: the gradient crosses five more layers + test hslm_backward_first_layer_w1_matches_trit_flip + given slope = weight_slope(MAT_W1, 0, 3 * FF_DIM + 500) + when grad = weight_grad_at(MAT_W1, 0, 3 * FF_DIM + 500) + then @abs(grad - slope) < 0.002 * @abs(slope) and @abs(slope) > 0.0000005 + + test hslm_backward_norm1_gamma_matches_finite_difference + given slope = gamma_slope(false, 2, 17) + when grad = gamma_gradient(false, 2, 17) + then @abs(grad - slope) < 0.000000001 * (1.0 + @abs(slope)) and @abs(slope) > 0.000001 + + test hslm_backward_norm2_gamma_matches_finite_difference + given slope = gamma_slope(true, 4, 101) + when grad = gamma_gradient(true, 4, 101) + then @abs(grad - slope) < 0.000000001 * (1.0 + @abs(slope)) and @abs(slope) > 0.000001 invariant hslm_embedding_dimensional_consistency assert EMBED_DIM == NUM_HEADS * HEAD_DIM + invariant hslm_head_dim_is_three_pow_four + assert HEAD_DIM == 3 * 3 * 3 * 3 + invariant hslm_ffn_expansion_ratio assert FF_DIM == 4 * EMBED_DIM - invariant hslm_layer_weights_size - given weights = LayerWeights{...} - and wq_size = weights.w_q.len() - and wk_size = weights.w_k.len() - and wv_size = weights.w_v.len() - and wo_size = weights.w_o.len() - assert wq_size == EMBED_DIM * EMBED_DIM - assert wk_size == EMBED_DIM * EMBED_DIM - assert wv_size == EMBED_DIM * EMBED_DIM - assert wo_size == EMBED_DIM * EMBED_DIM - - invariant hslm_ffn_weights_size - given weights = LayerWeights{...} - and w1_size = weights.w1.len() - and w2_size = weights.w2.len() - assert w1_size == EMBED_DIM * FF_DIM - assert w2_size == FF_DIM * EMBED_DIM - - invariant hslm_rms_norm_gamma_size - given weights = LayerWeights{...} - assert weights.norm1_gamma.len() == EMBED_DIM - assert weights.norm2_gamma.len() == EMBED_DIM - - invariant hslm_ternary_matmul_output_dim - given input = [0.0; 10] - and weights = [Trit.zero; 100] - and output = [0.0; 5] - and ternary_matmul(input, weights, output, 10, 5) - assert output.len() == 5 - - invariant hslm_gelu_monotonic_positive - given x = [0.0, 1.0, 2.0, 3.0] - and gelu_activation(x) - assert x[0] <= x[1] and x[1] <= x[2] and x[2] <= x[3] - - invariant hslm_gelu_smooth - // GELU is C1 continuous (smooth) - verified by lack of discontinuities - given x = [0.0, 0.001, 0.002] - and gelu_activation(x) - and delta1 = x[1] - x[0] - and delta2 = x[2] - x[1] - // Derivative should not have large jumps - assert abs(delta2 - delta1) < 0.01 - - invariant hslm_mode_in_valid_range - given mode = get_hslm_mode() - assert mode >= 1 and mode <= 3 - - bench hslm_rms_norm_latency - measure: nanoseconds to rms_norm_forward([1.0; EMBED_DIM], [1.0; EMBED_DIM]) - target: < 1000ns - - bench hslm_ternary_matmul_embed_to_ff - measure: nanoseconds to ternary_matmul([1.0; EMBED_DIM], [Trit.pos; EMBED_DIM*FF_DIM], [0.0; FF_DIM], EMBED_DIM, FF_DIM) - target: < 5000ns - - bench hslm_gelu_activation_latency - measure: nanoseconds to gelu_activation([1.0; FF_DIM]) - target: < 2000ns - - bench hslm_ffn_forward_latency - measure: nanoseconds for ffn_forward on single layer - target: < 10000ns - - bench hslm_layer_forward_latency - measure: nanoseconds for hslm_layer_forward on single position - target: < 30000ns - - bench hslm_forward_throughput - measure: microseconds for full forward pass on [81, 243] input - target: < 50000us - - test hslm_ffn_backward_w2_initializes_grads - given grads = zero_weight_gradients() - and layer_idx = 0 - when ffn_backward_w2(&grads, layer_idx) - then grads.layers[layer_idx].w2[0] == 0.0 // Should initialize gradients - - test hslm_gelu_backward_no_crash - given layer_idx = 0 - when gelu_backward(layer_idx) - then true // Should not crash - - test hslm_ffn_backward_w1_initializes_grads - given grads = zero_weight_gradients() - and layer_idx = 0 - when ffn_backward_w1(&grads, layer_idx) - then grads.layers[layer_idx].w1[0] == 0.0 // Should initialize gradients - - test hslm_attention_backward_initializes_grads - given grads = zero_weight_gradients() - and layer_idx = 0 - when attention_backward(&grads, layer_idx) - then grads.layers[layer_idx].w_q[0] == 0.0 // Should initialize gradients + invariant hslm_context_fits_cache + assert CONTEXT_LEN == HEAD_DIM + + invariant hslm_phase_constants_are_unique + assert PHASE_FORWARD != PHASE_BACKWARD and PHASE_BACKWARD != PHASE_UPDATE and PHASE_FORWARD != PHASE_UPDATE + + invariant hslm_attention_scale_is_sacred + assert SACRED_ATTN_SCALE > 0.3543788 and SACRED_ATTN_SCALE < 0.3543789 }