From cc56fcf80b9b519e49c9b78926043a4f19bede98 Mon Sep 17 00:00:00 2001 From: Dmitrii Vasilev Date: Sun, 18 Oct 2026 01:50:51 +0700 Subject: [PATCH] spec(hslm): nn/hslm runs the per-block VSA stream (Closes #10099) Adds the per-block VSA stream of trinity-training's HSLM (@3e9648c1, trinity_block.zig, model.zig forwardTrain) to nn/hslm: bind and rotate, the max_sim >= PHI_INV gate, +0.1 * project(context) when it fires, and the block output handed to the next block. The offset never reads a float parameter, so the existing finite-difference gradient checks still hold with the stream on. The ternary STE update is left to its own lane (#10076, PR #10086). nn/hslm on the Railway lab: 61/61 tests pass, 12 invariants proved, 0 vacuous (master: 46 tests). Mutation control: gate > for >= fails 2 tests, VSA gain 0 fails 1. HSLM and nn_HSLM are re-sealed on the lab. own_model: the forward-pass cell goes 85 to 95 percent (lens 2 now 58, overall still 44); own_model_board mirrors it. Cells 15-17 (STE) are untouched. Both pass 8/8 and 6/6, 0 vacuous. Deletes conformance/nn_hslm_vectors.json: it described the 6-layer, FF 972, ctx 81 model and nothing read it. Co-Authored-By: Claude Opus 5.5 --- .trinity/seals/HSLM.json | 16 +- .trinity/seals/nn_HSLM.json | 18 +- conformance/nn_hslm_vectors.json | 581 ------------------------- specs/igla/own_model.t27 | 16 +- specs/igla/own_model_board.t27 | 4 +- specs/nn/hslm.t27 | 716 ++++++++++++++++++++++++++++++- 6 files changed, 733 insertions(+), 618 deletions(-) delete mode 100644 conformance/nn_hslm_vectors.json diff --git a/.trinity/seals/HSLM.json b/.trinity/seals/HSLM.json index 45b7345a3b..b8ea0142c4 100644 --- a/.trinity/seals/HSLM.json +++ b/.trinity/seals/HSLM.json @@ -1,19 +1,19 @@ { - "gen_hash_c": "sha256:5e6c04320ab12b7f1451bdfd4162ca52a3eb54c1badfb47d84d6fa8411724d19", - "gen_hash_rust": "sha256:435b3e5b895f99ea146078d091715ee5a71c82aa13a204090bd4b53939983b26", - "gen_hash_verilog": "sha256:cafcce94d7360dcf804d9ef3dc6fe5de0b4de7335c7fcc05a747ee4db0be9800", - "gen_hash_zig": "sha256:0b168aaa160d077bfbcb66a19487dce330289422c64da51cec4154fd5ff7799a", + "gen_hash_c": "sha256:8fba76e317e8efda01839809c0f64ecd10289d564676e38e0d489c61d9ca8969", + "gen_hash_rust": "sha256:5c9fdd9e8a59b3982c0c1c248425d5a77d6911c9358a1e99ecec3f9e949a3f77", + "gen_hash_verilog": "sha256:8ac9dd632d4465248c1a6db8e3e5764e033524dbb42c8188d487bce7480ee8e3", + "gen_hash_zig": "sha256:2b7bab65264b9fadeca255e9397f0066f6d297736f3521983a4a6db06913dc49", "module": "HSLM", "ring": 12, - "sealed_at": "2026-10-17T18:02:24Z", - "spec_hash": "sha256:df24bc0cb83391c7131fc940940bae4804f95fde9e854ae03065ec8caff85a08", + "sealed_at": "2026-10-17T18:57:41Z", + "spec_hash": "sha256:568052c9f37f23e243158d6ec41b3b1530d4fc57b12420cc6aaf22401a1a2009", "spec_path": "specs/nn/hslm.t27", "tests": { "failed": 0, "failing": [], "forced": false, - "passed": 46, - "total": 46, + "passed": 61, + "total": 61, "vacuous": 0 } } \ No newline at end of file diff --git a/.trinity/seals/nn_HSLM.json b/.trinity/seals/nn_HSLM.json index a0e9e7c4ef..cb3be034a9 100644 --- a/.trinity/seals/nn_HSLM.json +++ b/.trinity/seals/nn_HSLM.json @@ -7,27 +7,27 @@ } ], "config": "gen=c,rust,verilog,zig;tests=zig", - "gen_hash_c": "sha256:5e6c04320ab12b7f1451bdfd4162ca52a3eb54c1badfb47d84d6fa8411724d19", - "gen_hash_rust": "sha256:435b3e5b895f99ea146078d091715ee5a71c82aa13a204090bd4b53939983b26", - "gen_hash_verilog": "sha256:cafcce94d7360dcf804d9ef3dc6fe5de0b4de7335c7fcc05a747ee4db0be9800", - "gen_hash_zig": "sha256:0b168aaa160d077bfbcb66a19487dce330289422c64da51cec4154fd5ff7799a", + "gen_hash_c": "sha256:8fba76e317e8efda01839809c0f64ecd10289d564676e38e0d489c61d9ca8969", + "gen_hash_rust": "sha256:5c9fdd9e8a59b3982c0c1c248425d5a77d6911c9358a1e99ecec3f9e949a3f77", + "gen_hash_verilog": "sha256:8ac9dd632d4465248c1a6db8e3e5764e033524dbb42c8188d487bce7480ee8e3", + "gen_hash_zig": "sha256:2b7bab65264b9fadeca255e9397f0066f6d297736f3521983a4a6db06913dc49", "module": "HSLM", "ring": 12, "seal_schema": 2, - "sealed_at": "2026-10-17T18:02:24Z", + "sealed_at": "2026-10-17T18:57:41Z", "sealed_by": "t27c-bootstrap@0.5.2", - "spec_hash": "sha256:df24bc0cb83391c7131fc940940bae4804f95fde9e854ae03065ec8caff85a08", + "spec_hash": "sha256:568052c9f37f23e243158d6ec41b3b1530d4fc57b12420cc6aaf22401a1a2009", "spec_path": "specs/nn/hslm.t27", "tests": { "failed": 0, "failing": [], "forced": false, - "passed": 46, - "total": 46, + "passed": 61, + "total": 61, "vacuous": 0 }, "toolchain": { - "test_runner": "sha256:1982ae948e24d00c5d221ccec12dfce1eb01204b53e37b0f857f93b16d64629d", + "test_runner": "missing", "zig": "0.16.0" } } \ No newline at end of file diff --git a/conformance/nn_hslm_vectors.json b/conformance/nn_hslm_vectors.json deleted file mode 100644 index 20b9772481..0000000000 --- a/conformance/nn_hslm_vectors.json +++ /dev/null @@ -1,581 +0,0 @@ -{ - "created_at": "2026-04-06T15:54:08Z", - "format_family": "Conformance", - "generated_at": "2026-04-04T19:00:00Z", - "invariants": [ - { - "condition": "NUM_LAYERS == 6", - "name": "hslm_num_layers_constant" - }, - { - "condition": "NUM_HEADS == 3", - "name": "hslm_num_heads_constant" - }, - { - "condition": "HEAD_DIM == 81", - "name": "hslm_head_dim_constant" - }, - { - "condition": "EMBED_DIM == 243", - "name": "hslm_embed_dim_constant" - }, - { - "condition": "FF_DIM == 972", - "name": "hslm_ff_dim_constant" - }, - { - "condition": "CONTEXT_LEN == 81", - "name": "hslm_context_len_constant" - }, - { - "condition": "VSA_DIM == 1024", - "name": "hslm_vsa_dim_constant" - }, - { - "condition": "EMBED_DIM == NUM_HEADS * HEAD_DIM", - "name": "hslm_embedding_dimensional_consistency" - }, - { - "condition": "FF_DIM == 4 * EMBED_DIM", - "name": "hslm_ffn_expansion_ratio" - }, - { - "condition": "w_q.len == EMBED_DIM * EMBED_DIM && w_k.len == EMBED_DIM * EMBED_DIM && w_v.len == EMBED_DIM * EMBED_DIM && w_o.len == EMBED_DIM * EMBED_DIM", - "name": "hslm_layer_weights_size" - }, - { - "condition": "w1.len == EMBED_DIM * FF_DIM && w2.len == FF_DIM * EMBED_DIM", - "name": "hslm_ffn_weights_size" - }, - { - "condition": "norm1_gamma.len == EMBED_DIM && norm2_gamma.len == EMBED_DIM", - "name": "hslm_rms_norm_gamma_size" - }, - { - "condition": "ternary_matmul output length == out_dim", - "name": "hslm_ternary_matmul_output_dim" - }, - { - "condition": "gelu(0) <= gelu(1) <= gelu(2) <= gelu(3)", - "name": "hslm_gelu_monotonic_positive" - }, - { - "condition": "abs(gelu'(x+eps) - gelu'(x)) < threshold for small eps", - "name": "hslm_gelu_smooth" - }, - { - "condition": "get_hslm_mode() >= 1 && get_hslm_mode() <= 3", - "name": "hslm_mode_in_valid_range" - } - ], - "module": "HSLM", - "ring": 29, - "schema_version": 2, - "seal": "sha256:9059f997327a6bd0c833501de9b4dee0e3a88fa0333040363b1755001ed12f9a", - "spec_path": "specs/nn/hslm.t27", - "updated_at": "2026-04-06T15:54:08Z", - "validated_at": "2026-04-06T15:54:08Z", - "vector_name": "nn_hslm Vectors", - "vectors": [ - { - "description": "NUM_LAYERS must equal 6", - "expected": { - "NUM_LAYERS": 6 - }, - "input": {}, - "name": "hslm_num_layers_is_six", - "verdict": "CLEAN" - }, - { - "description": "NUM_HEADS must equal 3 (TRINITY)", - "expected": { - "NUM_HEADS": 3 - }, - "input": {}, - "name": "hslm_num_heads_is_trinity", - "verdict": "CLEAN" - }, - { - "description": "HEAD_DIM must equal 81 = 3^4", - "expected": { - "HEAD_DIM": 81 - }, - "input": {}, - "name": "hslm_head_dim_is_three_pow_four", - "verdict": "CLEAN" - }, - { - "description": "EMBED_DIM = NUM_HEADS * HEAD_DIM = 243", - "expected": { - "EMBED_DIM": 243, - "computed": 243 - }, - "input": {}, - "name": "hslm_embed_dim_is_heads_times_head_dim", - "verdict": "CLEAN" - }, - { - "description": "FF_DIM = 4 * EMBED_DIM = 972", - "expected": { - "FF_DIM": 972, - "computed": 972 - }, - "input": {}, - "name": "hslm_ff_dim_is_four_times_embed_dim", - "verdict": "CLEAN" - }, - { - "description": "CONTEXT_LEN must equal 81", - "expected": { - "CONTEXT_LEN": 81 - }, - "input": {}, - "name": "hslm_context_len_is_eighty_one", - "verdict": "CLEAN" - }, - { - "description": "VSA_DIM must equal 1024", - "expected": { - "VSA_DIM": 1024 - }, - "input": {}, - "name": "hslm_vsa_dim_is_1024", - "verdict": "CLEAN" - }, - { - "description": "PHASE_NORM, PHASE_ATTN, PHASE_FFN, PHASE_RESIDUAL are distinct", - "expected": { - "PHASE_ATTN": 1, - "PHASE_FFN": 2, - "PHASE_NORM": 0, - "PHASE_RESIDUAL": 3, - "all_unique": true - }, - "input": {}, - "name": "hslm_phase_constants_are_unique", - "verdict": "CLEAN" - }, - { - "description": "ACT_RELU, ACT_GELU, ACT_SWISH, ACT_TERNARY are distinct", - "expected": { - "ACT_GELU": 1, - "ACT_RELU": 0, - "ACT_SWISH": 2, - "ACT_TERNARY": 3, - "all_unique": true - }, - "input": {}, - "name": "hslm_activation_constants_are_unique", - "verdict": "CLEAN" - }, - { - "description": "PHASE_FORWARD, PHASE_BACKWARD, PHASE_UPDATE are distinct", - "expected": { - "PHASE_BACKWARD": 1, - "PHASE_FORWARD": 0, - "PHASE_UPDATE": 2, - "all_unique": true - }, - "input": {}, - "name": "hslm_training_phase_constants_are_unique", - "verdict": "CLEAN" - }, - { - "description": "hslm_phase(PHASE_FORWARD) sets mode to 1", - "expected": { - "mode": 1 - }, - "input": { - "phase": 0 - }, - "name": "hslm_phase_forward_sets_mode_one", - "verdict": "CLEAN" - }, - { - "description": "hslm_phase(PHASE_BACKWARD) sets mode to 2", - "expected": { - "mode": 2 - }, - "input": { - "phase": 1 - }, - "name": "hslm_phase_backward_sets_mode_two", - "verdict": "CLEAN" - }, - { - "description": "hslm_phase(PHASE_UPDATE) sets mode to 3", - "expected": { - "mode": 3 - }, - "input": { - "phase": 2 - }, - "name": "hslm_phase_update_sets_mode_three", - "verdict": "CLEAN" - }, - { - "description": "RMSNorm preserves input dimension", - "expected": { - "output_length": 5 - }, - "input": { - "gamma": [ - 1.0, - 1.0, - 1.0, - 1.0, - 1.0 - ], - "x": [ - 1.0, - 2.0, - 3.0, - 4.0, - 5.0 - ] - }, - "name": "hslm_rms_norm_preserves_shape", - "verdict": "CLEAN" - }, - { - "description": "RMSNorm of zero input returns zero (with eps)", - "expected": { - "output": [ - 0.0, - 0.0, - 0.0 - ] - }, - "input": { - "gamma": [ - 1.0, - 1.0, - 1.0 - ], - "x": [ - 0.0, - 0.0, - 0.0 - ] - }, - "name": "hslm_rms_norm_zero_input_returns_zero", - "verdict": "CLEAN" - }, - { - "description": "Identity ternary matrix preserves input", - "expected": { - "output": [ - 1.0, - 2.0, - 3.0 - ] - }, - "input": { - "in_dim": 3, - "input": [ - 1.0, - 2.0, - 3.0 - ], - "out_dim": 3, - "weights_trit": [ - 1, - 0, - 0, - 0, - 1, - 0, - 0, - 0, - 1 - ] - }, - "name": "hslm_ternary_matmul_identity", - "verdict": "CLEAN" - }, - { - "description": "All-negative ternary matrix negates and sums", - "expected": { - "output": [ - -6.0, - -6.0, - -6.0 - ] - }, - "input": { - "in_dim": 3, - "input": [ - 1.0, - 2.0, - 3.0 - ], - "out_dim": 3, - "weights_trit": [ - -1, - -1, - -1, - -1, - -1, - -1, - -1, - -1, - -1 - ] - }, - "name": "hslm_ternary_matmul_negation", - "verdict": "CLEAN" - }, - { - "description": "GELU preserves input dimension", - "expected": { - "output_length": 4 - }, - "input": { - "x": [ - 1.0, - 2.0, - -1.0, - 0.0 - ] - }, - "name": "hslm_gelu_activation_preserves_shape", - "verdict": "CLEAN" - }, - { - "description": "GELU of positive input is positive", - "expected": { - "sign": "positive" - }, - "input": { - "x": [ - 1.0 - ] - }, - "name": "hslm_gelu_activation_positive_is_positive", - "verdict": "CLEAN" - }, - { - "description": "GELU(0) ~ 0", - "expected": { - "tolerance_abs": 0.0001, - "value": 0.0 - }, - "input": { - "x": [ - 0.0 - ] - }, - "name": "hslm_gelu_activation_zero_is_zero", - "verdict": "CLEAN" - }, - { - "description": "GELU(-1) is negative", - "expected": { - "sign": "negative" - }, - "input": { - "x": [ - -1.0 - ] - }, - "name": "hslm_gelu_activation_negative_is_negative", - "verdict": "CLEAN" - }, - { - "description": "FFN intermediate buffer has FF_DIM elements", - "expected": { - "ffn_intermediate_length": 972 - }, - "input": {}, - "name": "hslm_ffn_forward_intermediate_dimension", - "verdict": "CLEAN" - }, - { - "description": "zero_weight_gradients creates structure with NUM_LAYERS layers", - "expected": { - "num_layers": 6 - }, - "input": {}, - "name": "hslm_zero_weight_gradients_initializes_all", - "verdict": "CLEAN" - }, - { - "description": "Invariant: ", - "expected": { - "value": true - }, - "input": { - "assertion": "" - }, - "name": "inv_hslm_num_layers_constant", - "verdict": "CLEAN" - }, - { - "description": "Invariant: ", - "expected": { - "value": true - }, - "input": { - "assertion": "" - }, - "name": "inv_hslm_num_heads_constant", - "verdict": "CLEAN" - }, - { - "description": "Invariant: ", - "expected": { - "value": true - }, - "input": { - "assertion": "" - }, - "name": "inv_hslm_head_dim_constant", - "verdict": "CLEAN" - }, - { - "description": "Invariant: ", - "expected": { - "value": true - }, - "input": { - "assertion": "" - }, - "name": "inv_hslm_embed_dim_constant", - "verdict": "CLEAN" - }, - { - "description": "Invariant: ", - "expected": { - "value": true - }, - "input": { - "assertion": "" - }, - "name": "inv_hslm_ff_dim_constant", - "verdict": "CLEAN" - }, - { - "description": "Invariant: ", - "expected": { - "value": true - }, - "input": { - "assertion": "" - }, - "name": "inv_hslm_context_len_constant", - "verdict": "CLEAN" - }, - { - "description": "Invariant: ", - "expected": { - "value": true - }, - "input": { - "assertion": "" - }, - "name": "inv_hslm_vsa_dim_constant", - "verdict": "CLEAN" - }, - { - "description": "Invariant: ", - "expected": { - "value": true - }, - "input": { - "assertion": "" - }, - "name": "inv_hslm_embedding_dimensional_consistency", - "verdict": "CLEAN" - }, - { - "description": "Invariant: ", - "expected": { - "value": true - }, - "input": { - "assertion": "" - }, - "name": "inv_hslm_ffn_expansion_ratio", - "verdict": "CLEAN" - }, - { - "description": "Invariant: ", - "expected": { - "value": true - }, - "input": { - "assertion": "" - }, - "name": "inv_hslm_layer_weights_size", - "verdict": "CLEAN" - }, - { - "description": "Invariant: ", - "expected": { - "value": true - }, - "input": { - "assertion": "" - }, - "name": "inv_hslm_ffn_weights_size", - "verdict": "CLEAN" - }, - { - "description": "Invariant: ", - "expected": { - "value": true - }, - "input": { - "assertion": "" - }, - "name": "inv_hslm_rms_norm_gamma_size", - "verdict": "CLEAN" - }, - { - "description": "Invariant: ", - "expected": { - "value": true - }, - "input": { - "assertion": "" - }, - "name": "inv_hslm_ternary_matmul_output_dim", - "verdict": "CLEAN" - }, - { - "description": "Invariant: ", - "expected": { - "value": true - }, - "input": { - "assertion": "" - }, - "name": "inv_hslm_gelu_monotonic_positive", - "verdict": "CLEAN" - }, - { - "description": "Invariant: ", - "expected": { - "value": true - }, - "input": { - "assertion": "" - }, - "name": "inv_hslm_gelu_smooth", - "verdict": "CLEAN" - }, - { - "description": "Invariant: ", - "expected": { - "value": true - }, - "input": { - "assertion": "" - }, - "name": "inv_hslm_mode_in_valid_range", - "verdict": "CLEAN" - } - ], - "verdict": "CLEAN", - "version": "2.0" -} \ No newline at end of file diff --git a/specs/igla/own_model.t27 b/specs/igla/own_model.t27 index 5a4fb11fe1..33fa9138a4 100644 --- a/specs/igla/own_model.t27 +++ b/specs/igla/own_model.t27 @@ -30,7 +30,13 @@ ; phi position encoding and phi-RoPE, a ReLU FFN 243-729-243 with biases and TWN alphas, the ; vocab-729 head and cross-entropy), its backward reaches the embedding, and every gradient is ; checked against a central difference; a mutant that drops one alpha fails 10 of the 46. -; Not in it: the per-block VSA path, f32, and the trainer's STE update (shadow weights). +; Not in it then: the per-block VSA path, f32, and the trainer's STE update (shadow weights). +; - re-test on 2026-10-18, same t27c, after the PR that added the VSA path: nn/hslm 61/61 tests +; pass, 12 invariants proved, 0 vacuous. Each block now runs the VSA stream of model.zig forwardTrain (bind, rotate, the +; PHI_INV gate, +0.1 * project(context) when it fires, the output into the next block); the +; stream never reads a float parameter, so the gradient checks hold with it on. A gate +; mutant (> for >=) fails 2 tests and a gain-0 mutant fails 1. Still not in it: f32 and the +; trainer's STE update. ; - live: GET huggingface.co/api/models/playra/tern-tc-9m and .../tern-tc-9m-t27 returned 200; ; .../tern-tc-100m returned 401, the same as a made-up name, so no 100M is public. ; GET board-production-2dec.up.railway.app/state returned 200 and a made-up path 404; its @@ -46,7 +52,7 @@ ; THE ANSWER this map gives: HSLM was trained before, in Zig only (64 files); its best runs ; (PPL 2.96 at ctx 27) were wiped by EXP-002 and survive only in the ledger. In t27 the loss, three ; optimizers and the HSLM spec's forward and backward passes pass, and that spec is now the trained -; model but for its VSA path (see MEASURED); ternary STE with shadow weights, data loader, +; model, its VSA path included (see MEASURED); ternary STE with shadow weights, data loader, ; checkpoint I/O, heap and file I/O and `t27c --backend ml` do not exist yet. ; Not claimed: no model quality beyond the numbers in SOURCES; no GIF or MP4, stills only. @@ -184,7 +190,7 @@ pub const STATUS : [27]u8 = [4, 4, 4, 3, 3, 3, 3, 0, 0, 4, 4, 4, 4, 4, 4, 3, 0, pub const LABELS : [27]str = ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27"]; /// Percent of each part that exists, estimated from its source (see the header). Never a status. -pub const PCT : [27]u8 = [60, 70, 80, 80, 60, 10, 50, 0, 0, 100, 100, 20, 80, 65, 85, 70, 0, 0, 70, 15, 60, 40, 35, 0, 0, 0, 50]; +pub const PCT : [27]u8 = [60, 70, 80, 80, 60, 10, 50, 0, 0, 100, 100, 20, 80, 65, 95, 70, 0, 0, 70, 15, 60, 40, 35, 0, 0, 0, 50]; /// The t27c test-report counts the PRACTICE cells quote. pub const ML_SPECS_MEASURED : i32 = 17; @@ -211,7 +217,7 @@ pub const ROWS : [20]str = [ "0 of 9 IGLA CODER specs compile", "1 published, 12 verified, 7 drafts", "7 gaps, about 44 percent overall", - "HSLM spec: add the VSA path", + "Batched f32 HSLM trainer", "Ternary STE with shadow weights", "Loader, checkpoint I/O, --backend ml", "" @@ -319,7 +325,7 @@ test the_compile_counts_are_the_measured_ones { test percent_per_lens_is_a_plain_mean { assert mean_pct(0, 9) == 45; - assert mean_pct(9, 9) == 57; + assert mean_pct(9, 9) == 58; assert mean_pct(18, 9) == 30; // the evidence slide's "about 44 percent overall" assert mean_pct(0, CELL_COUNT) == 44; diff --git a/specs/igla/own_model_board.t27 b/specs/igla/own_model_board.t27 index 6a5d0cf671..f78f50aee9 100644 --- a/specs/igla/own_model_board.t27 +++ b/specs/igla/own_model_board.t27 @@ -59,9 +59,9 @@ pub const KEY_CELLS : str = "cells"; pub const KEY_STATUS : str = "status"; pub const KEY_PCT : str = "pct"; -pub const LENS_PCT : [3]u8 = [45, 57, 30]; +pub const LENS_PCT : [3]u8 = [45, 58, 30]; pub const CELL_STATUS : [27]u8 = [4, 4, 4, 3, 3, 3, 3, 0, 0, 4, 4, 4, 4, 4, 4, 3, 0, 0, 4, 4, 5, 4, 3, 0, 0, 0, 3]; -pub const CELL_PCT : [27]u8 = [60, 70, 80, 80, 60, 10, 50, 0, 0, 100, 100, 20, 80, 65, 85, 70, 0, 0, 70, 15, 60, 40, 35, 0, 0, 0, 50]; +pub const CELL_PCT : [27]u8 = [60, 70, 80, 80, 60, 10, 50, 0, 0, 100, 100, 20, 80, 65, 95, 70, 0, 0, 70, 15, 60, 40, 35, 0, 0, 0, 50]; /// The size of the compact metrics object, in bytes, computed from the keys and values above. pub const METRICS_BYTES : i32 = 622; diff --git a/specs/nn/hslm.t27 b/specs/nn/hslm.t27 index 35e5039825..4b1c77b11e 100644 --- a/specs/nn/hslm.t27 +++ b/specs/nn/hslm.t27 @@ -22,22 +22,40 @@ // logits = alpha_out (n @ W_out) / 243^(phi^-3) + b_out // src/hslm/ste.zig applyAlpha: the TWN scale, one per matrix // src/hslm/trainer.zig cross-entropy on the last position's logits +// src/hslm/model.zig forwardTrain: per block and position, the VSA +// attention, the consciousness gate, and the +// 0.1 * projectVsaToEmbed(reasoned) offset +// src/hslm/attention.zig VSAAttention.forward, cosineSimilarityTrit +// src/hslm/consciousness.zig ConsciousnessGate: max_sim >= phi^-1, EMA 0.1 +// src/hslm/reasoning.zig forward = blend([context, analogy], [0.618, 0.382]) // Context 27: gHashTag/trinity @ ee512359254bcd7438bb6a79c8a4a7f8b554b620, // docs/experiments/FOUND_EXPERIMENTS_SUMMARY.md: ctx 18 -> PPL 5.58, ctx 27 -> // PPL 2.96 (run R5, the best), ctx 54 -> PPL 6.05; ctx must divide HEAD_DIM // (EXP-012). constants.zig still says CONTEXT_LEN 81; the run used 27. // +// The VSA path (section 4b) is a second stream next to the float one: each +// token also has a 1024-trit hypervector, rotated by its position; per block +// the VSA attention bundles the earlier hypervectors by cosine similarity, +// and when the gate fires the block's float output gets 0.1 * the projected +// hypervector. That stream depends on tokens and positions only, never on a +// float parameter, so it adds a constant per (block, position) and leaves +// every gradient as it was; the gradient checks below run with it on. Two +// facts of the trained code, checked here: the reasoning step returns the +// context unchanged (blend 6 * c + 3 * c * current^2 has the sign of c), and +// at position 0 the gate always fires (max_sim is the self-similarity, 1). +// // What still differs from the trained model, named so nobody reads this spec // as more than it is: -// - The VSA path is not here: per block the trained model also runs VSA -// attention over 1024-trit hypervectors, and when its consciousness gate -// fires (similarity > phi^-1) adds 0.1 * projectVsaToEmbed(reasoned) to -// the block output. // - Floats: the trained model computes in f32, this spec in f64. +// - The trit table of the VSA stream is Zig's PRNG there and a hash here. +// - The trainer's attention backward sums W_K / W_V gradients over every +// position of the sequence; this spec treats earlier positions' keys and +// values as cached constants (the gradient of the last position only). // - Training: the trainer's STE backward leaves the alphas out of the input // gradients and stops at the embedding (the table is never trained); this // spec computes the exact gradient, embedding included. Alphas and the -// ternary requantize step are inputs here, not learned. +// ternary requantize step are inputs here, not learned; the STE update is +// its own spec (issue #10076). // // The backward pass is checked against central differences of the loss: on // inputs, embedding rows, gains, biases and, for ternary weights, a real-valued @@ -60,7 +78,8 @@ module HSLM { const VOCAB_SIZE : usize = 729; // 3^6 tokens const CONTEXT_LEN : usize = 27; // 3^3, the PPL 2.96 run const ROPE_PAIRS : usize = 40; // HEAD_DIM / 2; the last dim is unrotated - const VSA_DIM : usize = 1024; // VSA hypervector dimension (path not specified here) + const VSA_DIM : usize = 1024; // VSA hypervector dimension, section 4b + const VSA_RATIO : usize = 4; // VSA_DIM / EMBED_DIM, integer division const RMS_EPS : f64 = 0.000001; // 1 / 81^(phi^-3), phi^-3 = 0.2360679775 @@ -71,6 +90,11 @@ module HSLM { const LN_PHI : f64 = 0.48121182505960347; // 3 / pi, the position encoding's frequency scale const TRINITY_SCALE : f64 = 0.954929658551372; + // phi^-1, the consciousness gate's threshold + const PHI_INV : f64 = 0.6180339887498949; + // The VSA offset's gain: next_float += projectVsaToEmbed(reasoned) * 0.1 + const VSA_GAIN : f64 = 0.1; + const GATE_EMA : f64 = 0.1; // Layer phases const PHASE_NORM : u8 = 0; @@ -193,6 +217,25 @@ module HSLM { var nudge_index : usize = 0; var nudge_delta : f64 = 0.0; + // The VSA stream (section 4b). vsa_in holds each block's input hypervector + // per position: block 0 gets the token's rotated hypervector, block b + 1 + // the one block b put out. On by default: the trained model has it. + var vsa_on : bool = true; + var vsa_in : [NUM_BLOCKS * CONTEXT_LEN * VSA_DIM]Trit = undefined; + var vsa_out : [NUM_BLOCKS * VSA_DIM]Trit = undefined; // next_trit, last position + var vsa_ctx : [VSA_DIM]Trit = undefined; // VSA attention context + var vsa_reasoned : [VSA_DIM]Trit = undefined; + var vsa_temp : [VSA_DIM]Trit = undefined; + var vsa_temp2 : [VSA_DIM]Trit = undefined; + var vsa_sim : [CONTEXT_LEN]f64 = undefined; + var vsa_accum : [VSA_DIM]i64 = undefined; + var vsa_max_sim : [NUM_BLOCKS]f64 = undefined; + var vsa_fired : [NUM_BLOCKS]bool = undefined; + // ConsciousnessGate state, one gate per block; it does not change the output + var gate_total : [NUM_BLOCKS]usize = undefined; + var gate_conscious : [NUM_BLOCKS]usize = undefined; + var gate_ema : [NUM_BLOCKS]f64 = undefined; + // ═════════════════════════════════════════════════════════════════ // 3. Kernels // ═════════════════════════════════════════════════════════════════ @@ -581,6 +624,9 @@ module HSLM { output[i] = act_r[eo + i] + output[i]; i = i + 1; } + if (vsa_on) { + vsa_block_forward(block_idx, position, output); + } } // n = x / rms(x); logits = alpha_out (n @ W_out) * SACRED_LOGIT_SCALE + b_out @@ -605,6 +651,9 @@ module HSLM { fn hslm_forward(token: usize, position: usize, logits: []f64) -> void { act_token = token; embed_into(token, position, &bw_tmp); + if (vsa_on) { + vsa_embed(token, position, vsa_in[position * VSA_DIM..(position + 1) * VSA_DIM]); + } hslm_block_forward(0, position, &bw_tmp, act_out[0..EMBED_DIM]); var block_idx : usize = 1; while (block_idx < NUM_BLOCKS) { @@ -617,6 +666,240 @@ module HSLM { head_forward(act_out[last..last + EMBED_DIM], logits); } + // ═════════════════════════════════════════════════════════════════ + // 4b. The VSA stream and the consciousness gate (model.zig forwardTrain) + // ═════════════════════════════════════════════════════════════════ + + // The token's base hypervector (embedding.zig trit_table, a PRNG there): + // a two-step multiplicative hash mod primes, about 2 in 3 trits nonzero + fn vsa_table(token: usize, d: usize) -> Trit { + const h1 = ((token * VSA_DIM + d + 1) * 2654435761) % 1000003; + const h2 = ((h1 + 7) * 48271) % 2147483647; + const m = h2 % 3; + if (m == 0) { + return Trit.pos; + } + if (m == 1) { + return Trit.neg; + } + return Trit.zero; + } + + // embedTrit: the table row, cyclically permuted by the position. + // cyclicPermute (reverse all, reverse [0, shift), reverse [shift, n)) is a + // right rotation: out[(i + shift) % n] = in[i] + fn vsa_embed(token: usize, position: usize, out: []Trit) -> void { + const shift = position % VSA_DIM; + var i : usize = 0; + while (i < VSA_DIM) { + out[(i + shift) % VSA_DIM] = vsa_table(token, i); + i = i + 1; + } + } + + fn trit_int(t: Trit) -> i64 { + if (t == Trit.pos) { + return 1; + } + if (t == Trit.neg) { + return 0 - 1; + } + return 0; + } + + fn copy_trits(src: []const Trit, dst: []Trit) -> void { + var i : usize = 0; + while (i < src.len) { + dst[i] = src[i]; + i = i + 1; + } + } + + // cosineSimilarityTrit: integer dot and norms; 0 if either vector is zero + fn cosine_trit(a: []const Trit, b: []const Trit) -> f64 { + var dot : i64 = 0; + var norm_a : i64 = 0; + var norm_b : i64 = 0; + var i : usize = 0; + while (i < a.len) { + const x = trit_int(a[i]); + const y = trit_int(b[i]); + dot = dot + x * y; + norm_a = norm_a + x * x; + norm_b = norm_b + y * y; + i = i + 1; + } + if (norm_a == 0 or norm_b == 0) { + return 0.0; + } + return @as(f64, @floatFromInt(dot)) / (sqrt(@as(f64, @floatFromInt(norm_a))) * sqrt(@as(f64, @floatFromInt(norm_b)))); + } + + // @intFromFloat(@max(1.0, x)) for x >= 0: the largest integer w >= 1 with w <= x + fn vsa_weight(x: f64) -> i64 { + var w : i64 = 1; + while (@as(f64, @floatFromInt(w + 1)) <= x) { + w = w + 1; + } + return w; + } + + // Majority vote: the sign of each accumulator + fn majority_vote(accum: []const i64, out: []Trit) -> void { + var d : usize = 0; + while (d < VSA_DIM) { + if (accum[d] > 0) { + out[d] = Trit.pos; + } else if (accum[d] < 0) { + out[d] = Trit.neg; + } else { + out[d] = Trit.zero; + } + d = d + 1; + } + } + + // VSAAttention.forwardCausal over the block's input hypervectors 0..position: + // query, keys and values are all that sequence. max_sim leaves out the + // query's own position (it falls back to the self-similarity at position 0); + // the context bundles every value with a positive score, weight + // int(max(1, 10 score)), then takes the majority vote + fn vsa_attention(block_idx: usize, position: usize) -> f64 { + const base = block_idx * CONTEXT_LEN * VSA_DIM; + const qo = base + position * VSA_DIM; + var i : usize = 0; + while (i <= position) { + const ko = base + i * VSA_DIM; + vsa_sim[i] = cosine_trit(vsa_in[qo..qo + VSA_DIM], vsa_in[ko..ko + VSA_DIM]); + i = i + 1; + } + var max_sim : f64 = 0.0 - 1.0; + i = 0; + while (i < position) { + if (vsa_sim[i] > max_sim) { + max_sim = vsa_sim[i]; + } + i = i + 1; + } + if (position == 0) { + max_sim = vsa_sim[0]; + } + var d : usize = 0; + while (d < VSA_DIM) { + vsa_accum[d] = 0; + d = d + 1; + } + i = 0; + while (i <= position) { + const score = vsa_sim[i]; + if (score > 0.0) { + const w = vsa_weight(score * 10.0); + const vo = base + i * VSA_DIM; + d = 0; + while (d < VSA_DIM) { + vsa_accum[d] = vsa_accum[d] + trit_int(vsa_in[vo + d]) * w; + d = d + 1; + } + } + i = i + 1; + } + majority_vote(&vsa_accum, &vsa_ctx); + return max_sim; + } + + fn gate_reset() -> void { + var b : usize = 0; + while (b < NUM_BLOCKS) { + gate_total[b] = 0; + gate_conscious[b] = 0; + gate_ema[b] = 0.0; + b = b + 1; + } + } + + // ConsciousnessGate.isConscious: count, EMA 0.1, fire when max_sim >= phi^-1 + fn gate_is_conscious(block_idx: usize, max_sim: f64) -> bool { + gate_total[block_idx] = gate_total[block_idx] + 1; + gate_ema[block_idx] = GATE_EMA * max_sim + (1.0 - GATE_EMA) * gate_ema[block_idx]; + const activated = max_sim >= PHI_INV; + if (activated) { + gate_conscious[block_idx] = gate_conscious[block_idx] + 1; + } + return activated; + } + + // bindVec: element-wise trit product + fn vsa_bind(a: []const Trit, b: []const Trit, out: []Trit) -> void { + var d : usize = 0; + while (d < VSA_DIM) { + const p = trit_int(a[d]) * trit_int(b[d]); + if (p > 0) { + out[d] = Trit.pos; + } else if (p < 0) { + out[d] = Trit.neg; + } else { + out[d] = Trit.zero; + } + d = d + 1; + } + } + + // Reasoning.forward: temp2 = analogy(current, context, current) + // = bind(bind(context, current), current); out = blend([context, temp2], + // [0.618, 0.382]) with integer weights int(max(1, 10 |w|)) = 6 and 3 + fn vsa_reasoning(current: []const Trit, context: []const Trit, out: []Trit) -> void { + vsa_bind(context, current, &vsa_temp); + vsa_bind(&vsa_temp, current, &vsa_temp2); + const w0 = vsa_weight(0.618 * 10.0); + const w1 = vsa_weight(0.382 * 10.0); + var d : usize = 0; + while (d < VSA_DIM) { + vsa_accum[d] = trit_int(context[d]) * w0 + trit_int(vsa_temp2[d]) * w1; + d = d + 1; + } + majority_vote(&vsa_accum, out); + } + + // projectVsaToEmbed: dim i is the mean of trits 4i .. 4i + 3 (1024 / 243 = 4, + // so trits 972 .. 1023 are never read) + fn project_vsa(v: []const Trit, i: usize) -> f64 { + const start = i * VSA_RATIO; + var sum : f64 = 0.0; + var j : usize = start; + while (j < start + VSA_RATIO) { + sum = sum + @as(f64, @floatFromInt(trit_int(v[j]))); + j = j + 1; + } + return sum / @as(f64, @floatFromInt(VSA_RATIO)); + } + + // After the block's float output: VSA attention, the gate, and when it fires + // the reasoning step and output += 0.1 * project(reasoned). The block's + // output hypervector (reasoned or context) is the next block's input. + fn vsa_block_forward(block_idx: usize, position: usize, output: []f64) -> void { + const max_sim = vsa_attention(block_idx, position); + vsa_max_sim[block_idx] = max_sim; + const qo = (block_idx * CONTEXT_LEN + position) * VSA_DIM; + const vo = block_idx * VSA_DIM; + const fired = gate_is_conscious(block_idx, max_sim); + vsa_fired[block_idx] = fired; + if (fired) { + vsa_reasoning(vsa_in[qo..qo + VSA_DIM], &vsa_ctx, &vsa_reasoned); + var i : usize = 0; + while (i < EMBED_DIM) { + output[i] = output[i] + project_vsa(&vsa_reasoned, i) * VSA_GAIN; + i = i + 1; + } + copy_trits(&vsa_reasoned, vsa_out[vo..vo + VSA_DIM]); + } else { + copy_trits(&vsa_ctx, vsa_out[vo..vo + VSA_DIM]); + } + if (block_idx + 1 < NUM_BLOCKS) { + const no = ((block_idx + 1) * CONTEXT_LEN + position) * VSA_DIM; + copy_trits(vsa_out[vo..vo + VSA_DIM], vsa_in[no..no + VSA_DIM]); + } + } + // ═════════════════════════════════════════════════════════════════ // 5. Backward pass (the last position each block processed) // ═════════════════════════════════════════════════════════════════ @@ -837,6 +1120,10 @@ module HSLM { var probe_logits : [VOCAB_SIZE]f64 = undefined; var probe_dl : [VOCAB_SIZE]f64 = undefined; var probe_gin : [EMBED_DIM]f64 = undefined; + // The position and token probe_loss and probe_backward run; a check that + // moves them puts them back + var probe_position : usize = 1; + var probe_token : usize = TOKEN_PROBE; // About 2 in 9 trits nonzero, a different pattern per matrix and block. // A multiplicative hash mod a prime: 243 and 729 are multiples of 9, so a @@ -925,7 +1212,7 @@ module HSLM { } fn probe_loss(target: usize) -> f64 { - probe_run(1, TOKEN_PROBE); + probe_run(probe_position, probe_token); if (target < VOCAB_SIZE) { return cross_entropy(&probe_logits, target); } @@ -946,7 +1233,7 @@ module HSLM { // Forward position 1 at the unperturbed point, then backward from the loss fn probe_backward(target: usize) -> void { - probe_run(1, TOKEN_PROBE); + probe_run(probe_position, probe_token); zero_weight_gradients(); if (target < VOCAB_SIZE) { cross_entropy_backward(&probe_logits, target, &probe_dl); @@ -963,19 +1250,19 @@ module HSLM { // Embedding entry k of the probed token, moved by +-1e-5 fn embed_slope(k: usize, target: usize) -> f64 { probe_setup(); - const e0 = model.embed[TOKEN_PROBE * EMBED_DIM + k]; - model.embed[TOKEN_PROBE * EMBED_DIM + k] = e0 + 0.00001; + const e0 = model.embed[probe_token * EMBED_DIM + k]; + model.embed[probe_token * EMBED_DIM + k] = e0 + 0.00001; const lp = probe_loss(target); - model.embed[TOKEN_PROBE * EMBED_DIM + k] = e0 - 0.00001; + model.embed[probe_token * EMBED_DIM + k] = e0 - 0.00001; const lm = probe_loss(target); - model.embed[TOKEN_PROBE * EMBED_DIM + k] = e0; + model.embed[probe_token * EMBED_DIM + k] = e0; return (lp - lm) / 0.00002; } fn embed_gradient(k: usize, target: usize) -> f64 { probe_setup(); probe_backward(target); - return g_embed[TOKEN_PROBE * EMBED_DIM + k]; + return g_embed[probe_token * EMBED_DIM + k]; } // The input gradient hslm_backward returns is the embedding row's @@ -1393,6 +1680,308 @@ module HSLM { return get_hslm_mode(); } + // ── Probes for the VSA stream (section 4b) ── + + var vsa_a : [VSA_DIM]Trit = undefined; + var vsa_b : [VSA_DIM]Trit = undefined; + var block_out_scratch : [EMBED_DIM]f64 = undefined; + + fn trit_mismatches(a: []const Trit, b: []const Trit) -> usize { + var bad : usize = 0; + var i : usize = 0; + while (i < a.len) { + if (a[i] != b[i]) { + bad = bad + 1; + } + i = i + 1; + } + return bad; + } + + fn zero_trits(t: []Trit) -> void { + var i : usize = 0; + while (i < t.len) { + t[i] = Trit.zero; + i = i + 1; + } + } + + // A token's hypervector against itself (kind 0), its negation (1), zero (2) + fn cosine_probe(kind: u8) -> f64 { + vsa_embed(TOKEN_PROBE, 0, &vsa_a); + if (kind == 0) { + copy_trits(&vsa_a, &vsa_b); + } else if (kind == 1) { + var d : usize = 0; + while (d < VSA_DIM) { + if (vsa_a[d] == Trit.pos) { + vsa_b[d] = Trit.neg; + } else if (vsa_a[d] == Trit.neg) { + vsa_b[d] = Trit.pos; + } else { + vsa_b[d] = Trit.zero; + } + d = d + 1; + } + } else { + zero_trits(&vsa_b); + } + return cosine_trit(&vsa_a, &vsa_b); + } + + // The embedded hypervector against the table row rotated right by position + fn rotation_mismatches(position: usize) -> usize { + vsa_embed(TOKEN_PROBE, position, &vsa_a); + var bad : usize = 0; + var i : usize = 0; + while (i < VSA_DIM) { + if (vsa_a[(i + position) % VSA_DIM] != vsa_table(TOKEN_PROBE, i)) { + bad = bad + 1; + } + i = i + 1; + } + return bad; + } + + fn rotation_moves(position: usize) -> usize { + vsa_embed(TOKEN_PROBE, 0, &vsa_a); + vsa_embed(TOKEN_PROBE, position, &vsa_b); + return trit_mismatches(&vsa_a, &vsa_b); + } + + fn table_nonzero(token: usize) -> usize { + var n : usize = 0; + var i : usize = 0; + while (i < VSA_DIM) { + if (vsa_table(token, i) != Trit.zero) { + n = n + 1; + } + i = i + 1; + } + return n; + } + + fn gate_probe(max_sim: f64) -> bool { + gate_reset(); + return gate_is_conscious(0, max_sim); + } + + // Three gate calls, 0.617, phi^-1, 0.9: total * 100 + conscious * 10 + fired + fn gate_counts() -> usize { + gate_reset(); + var fired : usize = 0; + if (gate_is_conscious(1, 0.617)) { + fired = fired + 1; + } + if (gate_is_conscious(1, PHI_INV)) { + fired = fired + 1; + } + if (gate_is_conscious(1, 0.9)) { + fired = fired + 1; + } + return gate_total[1] * 100 + gate_conscious[1] * 10 + fired; + } + + // The gate's EMA after max_sim 1.0, then 0.5: 0.1 * 0.5 + 0.9 * 0.1 + fn gate_ema_probe() -> f64 { + gate_reset(); + var fired : usize = 0; + if (gate_is_conscious(2, 1.0)) { + fired = fired + 1; + } + if (gate_is_conscious(2, 0.5)) { + fired = fired + 1; + } + if (fired != 1) { + return 0.0 - 1.0; + } + return gate_ema[2]; + } + + fn reasoning_inputs() -> void { + vsa_embed(TOKEN_PROBE, 3, &vsa_a); + vsa_embed(TOKEN_FIRST, 0, &vsa_b); + } + + // Reasoning.forward against its context input + fn reasoning_mismatches() -> usize { + reasoning_inputs(); + vsa_reasoning(&vsa_a, &vsa_b, &vsa_reasoned); + return trit_mismatches(&vsa_reasoned, &vsa_b); + } + + fn reasoning_input_gap() -> usize { + reasoning_inputs(); + return trit_mismatches(&vsa_a, &vsa_b); + } + + // Trits 968 .. 971 at +1, trit 0 and the unread 972 and 1023 at -1 + fn projection_probe(i: usize) -> f64 { + zero_trits(&vsa_a); + vsa_a[0] = Trit.neg; + vsa_a[968] = Trit.pos; + vsa_a[969] = Trit.pos; + vsa_a[970] = Trit.pos; + vsa_a[971] = Trit.pos; + vsa_a[972] = Trit.neg; + vsa_a[1023] = Trit.neg; + return project_vsa(&vsa_a, i); + } + + // Blocks whose gate fired at position 0 + fn fired_at_zero() -> usize { + init_model(); + probe_run(0, TOKEN_FIRST); + var n : usize = 0; + var b : usize = 0; + while (b < NUM_BLOCKS) { + if (vsa_fired[b]) { + n = n + 1; + } + b = b + 1; + } + return n; + } + + // At position 0 every block's output hypervector is its input + fn stream_identity_at_zero() -> usize { + init_model(); + probe_run(0, TOKEN_FIRST); + var bad : usize = 0; + var b : usize = 0; + while (b < NUM_BLOCKS) { + const io = b * CONTEXT_LEN * VSA_DIM; + bad = bad + trit_mismatches(vsa_out[b * VSA_DIM..(b + 1) * VSA_DIM], vsa_in[io..io + VSA_DIM]); + b = b + 1; + } + return bad; + } + + // The next block's input at position 1 is the block's output hypervector + fn stream_hand_off_gap() -> usize { + probe_setup(); + probe_run(1, TOKEN_PROBE); + var bad : usize = 0; + var b : usize = 0; + while (b + 1 < NUM_BLOCKS) { + const io = ((b + 1) * CONTEXT_LEN + 1) * VSA_DIM; + bad = bad + trit_mismatches(vsa_out[b * VSA_DIM..(b + 1) * VSA_DIM], vsa_in[io..io + VSA_DIM]); + b = b + 1; + } + return bad; + } + + // Block 0's output at a position, VSA stream on minus off + fn vsa_on_minus_off(position: usize, token: usize, i: usize) -> f64 { + probe_setup(); + probe_run(position, token); + copy_floats(act_out[0..EMBED_DIM], &block_out_scratch); + vsa_on = false; + probe_run(position, token); + vsa_on = true; + return block_out_scratch[i] - act_out[i]; + } + + fn copy_floats(src: []const f64, dst: []f64) -> void { + var i : usize = 0; + while (i < src.len) { + dst[i] = src[i]; + i = i + 1; + } + } + + // Position 0, block 0: on minus off against 0.1 * project(the token's + // hypervector), summed over the dims + fn vsa_offset_gap() -> f64 { + init_model(); + probe_run(0, TOKEN_FIRST); + copy_floats(act_out[0..EMBED_DIM], &block_out_scratch); + vsa_on = false; + probe_run(0, TOKEN_FIRST); + vsa_on = true; + vsa_embed(TOKEN_FIRST, 0, &vsa_a); + var gap : f64 = 0.0; + var i : usize = 0; + while (i < EMBED_DIM) { + gap = gap + @abs((block_out_scratch[i] - act_out[i]) - project_vsa(&vsa_a, i) * VSA_GAIN); + i = i + 1; + } + return gap; + } + + fn vsa_offset_size() -> f64 { + vsa_embed(TOKEN_FIRST, 0, &vsa_a); + var size : f64 = 0.0; + var i : usize = 0; + while (i < EMBED_DIM) { + size = size + @abs(project_vsa(&vsa_a, i) * VSA_GAIN); + i = i + 1; + } + return size; + } + + // Position 1: block 0's gate and its similarity + fn block0_fired_at_one() -> bool { + probe_setup(); + probe_run(1, TOKEN_PROBE); + return vsa_fired[0]; + } + + fn block0_sim_at_one() -> f64 { + probe_setup(); + probe_run(1, TOKEN_PROBE); + return vsa_max_sim[0]; + } + + fn block0_gap_at_one() -> f64 { + var gap : f64 = 0.0; + var i : usize = 0; + while (i < EMBED_DIM) { + gap = gap + @abs(vsa_on_minus_off(1, TOKEN_PROBE, i)); + i = i + 27; + } + return gap; + } + + // The gradient checks at position 0, where every gate fires + fn at_position_zero(on: bool) -> void { + if (on) { + probe_position = 0; + probe_token = TOKEN_FIRST; + } else { + probe_position = 1; + probe_token = TOKEN_PROBE; + } + } + + fn embed_slope0(k: usize, target: usize) -> f64 { + at_position_zero(true); + const s = embed_slope(k, target); + at_position_zero(false); + return s; + } + + fn embed_gradient0(k: usize, target: usize) -> f64 { + at_position_zero(true); + const g = embed_gradient(k, target); + at_position_zero(false); + return g; + } + + fn gamma_slope0(block_idx: usize, index: usize) -> f64 { + at_position_zero(true); + const s = gamma_slope(block_idx, index); + at_position_zero(false); + return s; + } + + fn gamma_gradient0(block_idx: usize, index: usize) -> f64 { + at_position_zero(true); + const g = gamma_gradient(block_idx, index); + at_position_zero(false); + return g; + } + // ═════════════════════════════════════════════════════════════════ // TDD-Inside-Spec: Tests and Invariants for HSLM // ═════════════════════════════════════════════════════════════════ @@ -1679,4 +2268,105 @@ module HSLM { invariant hslm_linear_loss_marker_is_vocab assert LINEAR_LOSS == VOCAB_SIZE and PROBE_TARGET < VOCAB_SIZE and TOKEN_PROBE < VOCAB_SIZE + + // ── VSA stream: hypervectors, attention, the consciousness gate ── + + test hslm_vsa_cosine_values + given same = cosine_probe(0) + when flipped = cosine_probe(1) + then @abs(same - 1.0) < 0.000000000000001 and @abs(flipped + 1.0) < 0.000000000000001 and cosine_probe(2) == 0.0 + + // The token hypervector is about 2 in 3 nonzero, and position p rotates + // it right by p + test hslm_vsa_embed_rotates_by_position + given at0 = rotation_mismatches(0) + when at5 = rotation_mismatches(5) + then at0 == 0 and at5 == 0 and rotation_moves(5) > 100 and table_nonzero(TOKEN_PROBE) > 600 and table_nonzero(TOKEN_PROBE) < 770 + + // The gate fires at max_sim >= phi^-1 and not below + test hslm_vsa_gate_threshold_is_phi_inverse + given below = gate_probe(0.617) + when at = gate_probe(PHI_INV) + then below == false and at and gate_probe(0.62) + + // Three calls, two of them at or above phi^-1: total 3, conscious 2, fired 2 + test hslm_vsa_gate_counts + given n = gate_counts() + when again = gate_counts() + then n == 322 and again == 322 + + // EMA 0.1: 0.1 * 1.0 = 0.1, then 0.1 * 0.5 + 0.9 * 0.1 = 0.14 + test hslm_vsa_gate_ema + given ema = gate_ema_probe() + when same = gate_ema_probe() + then @abs(ema - 0.14) < 0.000000000000001 and ema == same + + // int(max(1, 10 score)): the floor, at least 1 + test hslm_vsa_attention_weight_is_floor_at_least_one + given small = vsa_weight(0.5) + when mid = vsa_weight(3.5) + then small == 1 and mid == 3 and vsa_weight(10.0) == 10 and vsa_weight(9.9) == 9 and vsa_weight(6.18) == 6 and vsa_weight(3.82) == 3 + + // Reasoning.forward as trained returns its context: bind(bind(c, x), x) + // is c where x is nonzero and 0 elsewhere, and 6 c + 3 that has c's sign + test hslm_vsa_reasoning_returns_context + given bad = reasoning_mismatches() + when inputs = reasoning_input_gap() + then bad == 0 and inputs > 100 + + // Dim i is the mean of trits 4i .. 4i + 3; trits 972 .. 1023 are not read + test hslm_vsa_projection_values + given top = projection_probe(242) + when below = projection_probe(241) + then top == 1.0 and below == 0.0 and projection_probe(0) == -0.25 + + // Position 0: every block's similarity is its self-similarity 1, every + // gate fires, and the stream passes the token hypervector through + test hslm_vsa_position_zero_fires_every_block + given fired = fired_at_zero() + when bad = stream_identity_at_zero() + then fired == NUM_BLOCKS and bad == 0 + + test hslm_vsa_stream_hands_off_between_blocks + given bad = stream_hand_off_gap() + when again = stream_hand_off_gap() + then bad == 0 and again == 0 + + // Position 0, block 0: the stream adds exactly 0.1 * project(token + // hypervector) to the block's output + test hslm_vsa_offset_is_gain_times_projection + given gap = vsa_offset_gap() + when size = vsa_offset_size() + then gap < 0.000000000001 and size > 1.0 + + // Position 1, block 0: the token's hypervector against TOKEN_FIRST's sits + // well under PHI_INV (measured between 0.2 and 0.5), the gate stays shut, + // and the output is the stream-off output + test hslm_vsa_block0_shut_at_position_one + given fired = block0_fired_at_one() + when sim = block0_sim_at_one() + then fired == false and @abs(sim) < 0.5 and block0_gap_at_one() == 0.0 + + // The offset does not depend on the float parameters, so the gradient + // matches the loss at position 0, where every gate fires + test hslm_vsa_embed_gradient_at_position_zero + given slope = embed_slope0(5, LINEAR_LOSS) + when grad = embed_gradient0(5, LINEAR_LOSS) + then @abs(grad - slope) < 0.000000001 * (1.0 + @abs(slope)) and @abs(slope) > 0.000001 + + test hslm_vsa_embed_gradient_at_position_zero_cross_entropy + given slope = embed_slope0(77, PROBE_TARGET) + when grad = embed_gradient0(77, PROBE_TARGET) + then @abs(grad - slope) < 0.000000001 * (1.0 + @abs(slope)) and @abs(slope) > 0.000001 + + test hslm_vsa_gamma_gradient_at_position_zero + given slope = gamma_slope0(1, 40) + when grad = gamma_gradient0(1, 40) + then @abs(grad - slope) < 0.000000001 * (1.0 + @abs(slope)) and @abs(slope) > 0.000001 + + invariant hslm_vsa_ratio + assert VSA_RATIO == VSA_DIM / EMBED_DIM and VSA_RATIO * EMBED_DIM <= VSA_DIM + + invariant hslm_phi_inverse_identity + assert @abs(PHI_INV * (1.0 + PHI_INV) - 1.0) < 0.000000000000001 }