diff --git a/specs/port/trios/download_fineweb.t27 b/specs/port/trios/download_fineweb.t27 new file mode 100644 index 0000000000..4fed21c16c --- /dev/null +++ b/specs/port/trios/download_fineweb.t27 @@ -0,0 +1,255 @@ +// FineWeb Data Downloader and Preparer for Trinity 3k. +// Port of gHashTag/trios:download_fineweb.rs +// (at 8b229e9489ee, 130 lines) — Roadmap Stage 6, goal #4548, target +// t27c gen-rust. +// +// The original prepares the FineWeb dataset in the binary format the +// trainer expects: a 256-i32 header (magic 20240520, version 1, the +// token count, the rest reserved zeros) followed by uint16 tokens in +// little-endian. main creates the dataset and tokenizer directories +// and defers to the test-dataset path; create_test_dataset writes two +// shards - a 1M-token training shard under seed 42 and a 100K-token +// validation shard under seed 123 - and create_fineweb_shard writes +// one shard: the header, then the deterministic pseudo-random tokens +// (the LCG-style mix (seed*1e6 + i)*1664525 + 1013904223 masked to +// 32 bits, rand = mix % 1000 / 1000) distributed by the four-band +// rule - below 0.1 the space token 0; to 0.8 a lowercase letter +// 1..26; to 0.95 a punctuation token 27..99; above, a rarer token +// 100..999 - every token folded into the SP4096 vocabulary with +// mod 4096. +// +// The filesystem and the byte writes are the callers' plumbing; the +// decisions — the header layout with its magic and version, the shard +// plan, the size formula (256*4 + tokens*2, the original's own test +// asserts it), the deterministic mix, the four-band distribution and +// the vocabulary fold — are ported here as pure functions and tested +// (issue #9730, "Port the decision, not the plumbing"). + +module port::trios_download_fineweb; + +// --------------------------------------------------------------------------- +// Original lines 61-66: the 256-i32 header — magic, version, the token +// count, the rest reserved. +// --------------------------------------------------------------------------- + +pub fn header_word_count() -> usize { + return 256; +} + +pub fn header_magic() -> i32 { + return 20240520; +} + +pub fn header_version() -> i32 { + return 1; +} + +pub fn header_token_count_index() -> usize { + return 2; +} + +// Original line 125: the shard size the original's own test asserts - +// 256 words of 4 bytes plus 2 bytes per token. +pub fn shard_size_bytes(num_tokens: usize) -> usize { + return header_word_count() * 4 + num_tokens * 2; +} + +// --------------------------------------------------------------------------- +// Original lines 12-28: `main` — the dataset and tokenizer directories +// are created, the test-dataset path runs, the real download is a +// later step. Pure transport; the directory plan is its decision. +// --------------------------------------------------------------------------- + +pub fn main() -> usize { + return 2; +} + +pub fn main_dataset_dir() -> str { + return "./data/datasets/fineweb10B_sp4096"; +} + +pub fn main_tokenizer_dir() -> str { + return "./data/tokenizers"; +} + +// --------------------------------------------------------------------------- +// Original lines 30-53: `create_test_dataset` — two shards: 1M training +// tokens under seed 42, 100K validation tokens under seed 123. +// --------------------------------------------------------------------------- + +pub fn train_shard_token_count() -> usize { + return 1000000; +} + +pub fn train_shard_seed() -> u32 { + return 42; +} + +pub fn val_shard_token_count() -> usize { + return 100000; +} + +pub fn val_shard_seed() -> u32 { + return 123; +} + +pub fn create_test_dataset() -> usize { + return 2; +} + +pub fn create_test_dataset_seeds_differ() -> bool { + return train_shard_seed() != val_shard_seed(); +} + +// --------------------------------------------------------------------------- +// Original lines 76-98: the deterministic pseudo-random mix and the +// four-band token distribution; original line 101 folds every token +// into the SP4096 vocabulary. +// --------------------------------------------------------------------------- + +pub fn shard_pseudo_random(seed: u32, i: usize) -> u64 { + const raw: u64 = (seed as u64 * 1000000 + i) * 1664525 + 1013904223; + return raw & 0xFFFFFFFF; +} + +pub fn shard_rand_val(seed: u32, i: usize) -> f32 { + const pseudo: u64 = shard_pseudo_random(seed, i); + return (pseudo % 1000) as f32 / 1000.0; +} + +pub fn shard_token(seed: u32, i: usize) -> u16 { + const pseudo: u64 = shard_pseudo_random(seed, i); + const rand_val: f32 = shard_rand_val(seed, i); + var token: u16 = 0; + if (rand_val < 0.1) { + token = 0; + } else { + if (rand_val < 0.8) { + token = (1 + pseudo % 26) as u16; + } else { + if (rand_val < 0.95) { + token = (27 + pseudo % 73) as u16; + } else { + token = (100 + pseudo % 900) as u16; + } + } + } + return token % 4096; +} + +pub fn vocab_size() -> u16 { + return 4096; +} + +// The four-band boundaries of the distribution (lines 86-98). +pub fn space_band() -> f32 { + return 0.1; +} + +pub fn letter_band() -> f32 { + return 0.8; +} + +pub fn punctuation_band() -> f32 { + return 0.95; +} + +pub fn first_letter_token() -> u16 { + return 1; +} + +pub fn letter_token_count() -> u16 { + return 26; +} + +pub fn first_punctuation_token() -> u16 { + return 27; +} + +pub fn first_rare_token() -> u16 { + return 100; +} + +// --------------------------------------------------------------------------- +// Tests: every decision above asserted against values read from the +// original, including its own shard-size test. +// --------------------------------------------------------------------------- + +// The one-shard plan: the 256-word header (magic, version, the token +// count, the rest reserved) then the token count in uint16 tokens — +// the size formula the original's own test asserts. +pub struct ShardPlan { + header_words: usize, + token_count: usize, + size_bytes: usize, +} + +pub fn create_fineweb_shard(num_tokens: usize) -> ShardPlan { + return ShardPlan { + header_words: header_word_count(), + token_count: num_tokens, + size_bytes: shard_size_bytes(num_tokens), + }; +} + +test header_layout { + assert(header_word_count() == 256); + assert(header_magic() == 20240520); + assert(header_version() == 1); + assert(header_token_count_index() == 2); +} + +test shard_plan_carries_the_layout { + const plan = create_fineweb_shard(1000); + assert(plan.header_words == 256); + assert(plan.token_count == 1000); + assert(plan.size_bytes == 3024); +} + +test shard_size_matches_the_originals_own_test { + assert(shard_size_bytes(1000) == 3024); + assert(shard_size_bytes(0) == 1024); + assert(shard_size_bytes(1000000) == 2001024); +} + +test main_and_test_dataset_plan { + assert(main() == 2); + assert(main_dataset_dir() == "./data/datasets/fineweb10B_sp4096"); + assert(main_tokenizer_dir() == "./data/tokenizers"); + assert(train_shard_token_count() == 1000000); + assert(train_shard_seed() == 42); + assert(val_shard_token_count() == 100000); + assert(val_shard_seed() == 123); + assert(create_test_dataset() == 2); + assert(create_test_dataset_seeds_differ()); +} + +test pseudo_random_is_deterministic_and_seeded { + assert(shard_pseudo_random(42, 0) == shard_pseudo_random(42, 0)); + assert(shard_pseudo_random(42, 0) != shard_pseudo_random(42, 1)); + assert(shard_pseudo_random(42, 5) != shard_pseudo_random(123, 5)); + assert(shard_rand_val(42, 0) >= 0.0); + assert(shard_rand_val(42, 0) < 1.0); +} + +test token_distribution_gates { + assert(space_band() == 0.1); + assert(letter_band() == 0.8); + assert(punctuation_band() == 0.95); + assert(first_letter_token() == 1); + assert(letter_token_count() == 26); + assert(first_punctuation_token() == 27); + assert(first_rare_token() == 100); + assert(vocab_size() == 4096); +} + +test every_token_fits_the_vocab { + var i: usize = 0; + while (i < 200) { + const t: u16 = shard_token(42, i); + assert(t < vocab_size()); + i += 1; + } + const t123: u16 = shard_token(123, 7); + assert(t123 < vocab_size()); +}