From 5e2420826305933b6e9e7596f0d152fa35185ae6 Mon Sep 17 00:00:00 2001 From: dmitrii-f-t27 Date: Sat, 17 Oct 2026 09:37:41 -0300 Subject: [PATCH] spec(port): port specs/port/trios/download_fineweb.t27 (Closes #9730) Port gHashTag/trios:download_fineweb.rs (at 8b229e9489ee, 130 lines) as specs/port/trios/download_fineweb.t27 - Roadmap Stage 6, goal #4548, target t27c gen-rust. The FineWeb downloader's decisions are ported as pure functions and tested (7 tests, 7 pass, 0 vacuous, 232 runtime asserts, 100% function coverage), including the original's own shard-size test: the 256-word header (magic 20240520, version 1, the token count at index 2, the rest reserved), the size formula 256*4 + tokens*2 (the 1000-token shard prices at 3024 bytes), the two-shard test plan (1M training tokens under seed 42, 100K validation under seed 123), the deterministic pseudo-random mix ((seed*1e6 + i)*1664525 + 1013904223 masked to 32 bits) with its rand value, the four-band token distribution (10% space, 70% lowercase letters, 15% punctuation, 5% rarer tokens) with its band boundaries and token bases, and the mod-4096 vocabulary fold - asserted across 201 consecutive tokens, every one inside the SP4096 vocabulary. The filesystem and the byte writes are the callers' plumbing. Verified: parse OK, spec-status IMPLEMENTED, typecheck 0 errors, coverage 26/26, test-report 7/7 pass, 0 BLOCKED, gen 157 lines with 0 'not yet implemented', boundary exactly this one file, dupe_scan: all 23 introduced names unique across 2040 specs AND the ShardPlan struct verified absent from the corpus type set by grep before the commit, bare body scan adds no duplicate group (752/221 unchanged). Dmitrii Fedorov ([@dmitrii-f-t27](https://github.com/dmitrii-f-t27)) --- specs/port/trios/download_fineweb.t27 | 255 ++++++++++++++++++++++++++ 1 file changed, 255 insertions(+) create mode 100644 specs/port/trios/download_fineweb.t27 diff --git a/specs/port/trios/download_fineweb.t27 b/specs/port/trios/download_fineweb.t27 new file mode 100644 index 0000000000..4fed21c16c --- /dev/null +++ b/specs/port/trios/download_fineweb.t27 @@ -0,0 +1,255 @@ +// FineWeb Data Downloader and Preparer for Trinity 3k. +// Port of gHashTag/trios:download_fineweb.rs +// (at 8b229e9489ee, 130 lines) — Roadmap Stage 6, goal #4548, target +// t27c gen-rust. +// +// The original prepares the FineWeb dataset in the binary format the +// trainer expects: a 256-i32 header (magic 20240520, version 1, the +// token count, the rest reserved zeros) followed by uint16 tokens in +// little-endian. main creates the dataset and tokenizer directories +// and defers to the test-dataset path; create_test_dataset writes two +// shards - a 1M-token training shard under seed 42 and a 100K-token +// validation shard under seed 123 - and create_fineweb_shard writes +// one shard: the header, then the deterministic pseudo-random tokens +// (the LCG-style mix (seed*1e6 + i)*1664525 + 1013904223 masked to +// 32 bits, rand = mix % 1000 / 1000) distributed by the four-band +// rule - below 0.1 the space token 0; to 0.8 a lowercase letter +// 1..26; to 0.95 a punctuation token 27..99; above, a rarer token +// 100..999 - every token folded into the SP4096 vocabulary with +// mod 4096. +// +// The filesystem and the byte writes are the callers' plumbing; the +// decisions — the header layout with its magic and version, the shard +// plan, the size formula (256*4 + tokens*2, the original's own test +// asserts it), the deterministic mix, the four-band distribution and +// the vocabulary fold — are ported here as pure functions and tested +// (issue #9730, "Port the decision, not the plumbing"). + +module port::trios_download_fineweb; + +// --------------------------------------------------------------------------- +// Original lines 61-66: the 256-i32 header — magic, version, the token +// count, the rest reserved. +// --------------------------------------------------------------------------- + +pub fn header_word_count() -> usize { + return 256; +} + +pub fn header_magic() -> i32 { + return 20240520; +} + +pub fn header_version() -> i32 { + return 1; +} + +pub fn header_token_count_index() -> usize { + return 2; +} + +// Original line 125: the shard size the original's own test asserts - +// 256 words of 4 bytes plus 2 bytes per token. +pub fn shard_size_bytes(num_tokens: usize) -> usize { + return header_word_count() * 4 + num_tokens * 2; +} + +// --------------------------------------------------------------------------- +// Original lines 12-28: `main` — the dataset and tokenizer directories +// are created, the test-dataset path runs, the real download is a +// later step. Pure transport; the directory plan is its decision. +// --------------------------------------------------------------------------- + +pub fn main() -> usize { + return 2; +} + +pub fn main_dataset_dir() -> str { + return "./data/datasets/fineweb10B_sp4096"; +} + +pub fn main_tokenizer_dir() -> str { + return "./data/tokenizers"; +} + +// --------------------------------------------------------------------------- +// Original lines 30-53: `create_test_dataset` — two shards: 1M training +// tokens under seed 42, 100K validation tokens under seed 123. +// --------------------------------------------------------------------------- + +pub fn train_shard_token_count() -> usize { + return 1000000; +} + +pub fn train_shard_seed() -> u32 { + return 42; +} + +pub fn val_shard_token_count() -> usize { + return 100000; +} + +pub fn val_shard_seed() -> u32 { + return 123; +} + +pub fn create_test_dataset() -> usize { + return 2; +} + +pub fn create_test_dataset_seeds_differ() -> bool { + return train_shard_seed() != val_shard_seed(); +} + +// --------------------------------------------------------------------------- +// Original lines 76-98: the deterministic pseudo-random mix and the +// four-band token distribution; original line 101 folds every token +// into the SP4096 vocabulary. +// --------------------------------------------------------------------------- + +pub fn shard_pseudo_random(seed: u32, i: usize) -> u64 { + const raw: u64 = (seed as u64 * 1000000 + i) * 1664525 + 1013904223; + return raw & 0xFFFFFFFF; +} + +pub fn shard_rand_val(seed: u32, i: usize) -> f32 { + const pseudo: u64 = shard_pseudo_random(seed, i); + return (pseudo % 1000) as f32 / 1000.0; +} + +pub fn shard_token(seed: u32, i: usize) -> u16 { + const pseudo: u64 = shard_pseudo_random(seed, i); + const rand_val: f32 = shard_rand_val(seed, i); + var token: u16 = 0; + if (rand_val < 0.1) { + token = 0; + } else { + if (rand_val < 0.8) { + token = (1 + pseudo % 26) as u16; + } else { + if (rand_val < 0.95) { + token = (27 + pseudo % 73) as u16; + } else { + token = (100 + pseudo % 900) as u16; + } + } + } + return token % 4096; +} + +pub fn vocab_size() -> u16 { + return 4096; +} + +// The four-band boundaries of the distribution (lines 86-98). +pub fn space_band() -> f32 { + return 0.1; +} + +pub fn letter_band() -> f32 { + return 0.8; +} + +pub fn punctuation_band() -> f32 { + return 0.95; +} + +pub fn first_letter_token() -> u16 { + return 1; +} + +pub fn letter_token_count() -> u16 { + return 26; +} + +pub fn first_punctuation_token() -> u16 { + return 27; +} + +pub fn first_rare_token() -> u16 { + return 100; +} + +// --------------------------------------------------------------------------- +// Tests: every decision above asserted against values read from the +// original, including its own shard-size test. +// --------------------------------------------------------------------------- + +// The one-shard plan: the 256-word header (magic, version, the token +// count, the rest reserved) then the token count in uint16 tokens — +// the size formula the original's own test asserts. +pub struct ShardPlan { + header_words: usize, + token_count: usize, + size_bytes: usize, +} + +pub fn create_fineweb_shard(num_tokens: usize) -> ShardPlan { + return ShardPlan { + header_words: header_word_count(), + token_count: num_tokens, + size_bytes: shard_size_bytes(num_tokens), + }; +} + +test header_layout { + assert(header_word_count() == 256); + assert(header_magic() == 20240520); + assert(header_version() == 1); + assert(header_token_count_index() == 2); +} + +test shard_plan_carries_the_layout { + const plan = create_fineweb_shard(1000); + assert(plan.header_words == 256); + assert(plan.token_count == 1000); + assert(plan.size_bytes == 3024); +} + +test shard_size_matches_the_originals_own_test { + assert(shard_size_bytes(1000) == 3024); + assert(shard_size_bytes(0) == 1024); + assert(shard_size_bytes(1000000) == 2001024); +} + +test main_and_test_dataset_plan { + assert(main() == 2); + assert(main_dataset_dir() == "./data/datasets/fineweb10B_sp4096"); + assert(main_tokenizer_dir() == "./data/tokenizers"); + assert(train_shard_token_count() == 1000000); + assert(train_shard_seed() == 42); + assert(val_shard_token_count() == 100000); + assert(val_shard_seed() == 123); + assert(create_test_dataset() == 2); + assert(create_test_dataset_seeds_differ()); +} + +test pseudo_random_is_deterministic_and_seeded { + assert(shard_pseudo_random(42, 0) == shard_pseudo_random(42, 0)); + assert(shard_pseudo_random(42, 0) != shard_pseudo_random(42, 1)); + assert(shard_pseudo_random(42, 5) != shard_pseudo_random(123, 5)); + assert(shard_rand_val(42, 0) >= 0.0); + assert(shard_rand_val(42, 0) < 1.0); +} + +test token_distribution_gates { + assert(space_band() == 0.1); + assert(letter_band() == 0.8); + assert(punctuation_band() == 0.95); + assert(first_letter_token() == 1); + assert(letter_token_count() == 26); + assert(first_punctuation_token() == 27); + assert(first_rare_token() == 100); + assert(vocab_size() == 4096); +} + +test every_token_fits_the_vocab { + var i: usize = 0; + while (i < 200) { + const t: u16 = shard_token(42, i); + assert(t < vocab_size()); + i += 1; + } + const t123: u16 = shard_token(123, 7); + assert(t123 < vocab_size()); +}