spec(port): port specs/port/trios/download_fineweb.t27 (Closes #9730)

specs/port/trios/download_fineweb.t27

@@ -0,0 +1,255 @@
+// FineWeb Data Downloader and Preparer for Trinity 3k.
+// Port of gHashTag/trios:download_fineweb.rs
+// (at 8b229e9489ee, 130 lines) — Roadmap Stage 6, goal #4548, target
+// t27c gen-rust.
+//
+// The original prepares the FineWeb dataset in the binary format the
+// trainer expects: a 256-i32 header (magic 20240520, version 1, the
+// token count, the rest reserved zeros) followed by uint16 tokens in
+// little-endian. main creates the dataset and tokenizer directories
+// and defers to the test-dataset path; create_test_dataset writes two
+// shards - a 1M-token training shard under seed 42 and a 100K-token
+// validation shard under seed 123 - and create_fineweb_shard writes
+// one shard: the header, then the deterministic pseudo-random tokens
+// (the LCG-style mix (seed*1e6 + i)*1664525 + 1013904223 masked to
+// 32 bits, rand = mix % 1000 / 1000) distributed by the four-band
+// rule - below 0.1 the space token 0; to 0.8 a lowercase letter
+// 1..26; to 0.95 a punctuation token 27..99; above, a rarer token
+// 100..999 - every token folded into the SP4096 vocabulary with
+// mod 4096.
+//
+// The filesystem and the byte writes are the callers' plumbing; the
+// decisions — the header layout with its magic and version, the shard
+// plan, the size formula (256*4 + tokens*2, the original's own test
+// asserts it), the deterministic mix, the four-band distribution and
+// the vocabulary fold — are ported here as pure functions and tested
+// (issue #9730, "Port the decision, not the plumbing").
+
+module port::trios_download_fineweb;
+
+// ---------------------------------------------------------------------------
+// Original lines 61-66: the 256-i32 header — magic, version, the token
+// count, the rest reserved.
+// ---------------------------------------------------------------------------
+
+pub fn header_word_count() -> usize {
+    return 256;
+}
+
+pub fn header_magic() -> i32 {
+    return 20240520;
+}
+
+pub fn header_version() -> i32 {
+    return 1;
+}
+
+pub fn header_token_count_index() -> usize {
+    return 2;
+}
+
+// Original line 125: the shard size the original's own test asserts -
+// 256 words of 4 bytes plus 2 bytes per token.
+pub fn shard_size_bytes(num_tokens: usize) -> usize {
+    return header_word_count() * 4 + num_tokens * 2;
+}
+
+// ---------------------------------------------------------------------------
+// Original lines 12-28: `main` — the dataset and tokenizer directories
+// are created, the test-dataset path runs, the real download is a
+// later step. Pure transport; the directory plan is its decision.
+// ---------------------------------------------------------------------------
+
+pub fn main() -> usize {
+    return 2;
+}
+
+pub fn main_dataset_dir() -> str {
+    return "./data/datasets/fineweb10B_sp4096";
+}
+
+pub fn main_tokenizer_dir() -> str {
+    return "./data/tokenizers";
+}
+
+// ---------------------------------------------------------------------------
+// Original lines 30-53: `create_test_dataset` — two shards: 1M training
+// tokens under seed 42, 100K validation tokens under seed 123.
+// ---------------------------------------------------------------------------
+
+pub fn train_shard_token_count() -> usize {
+    return 1000000;
+}
+
+pub fn train_shard_seed() -> u32 {
+    return 42;
+}
+
+pub fn val_shard_token_count() -> usize {
+    return 100000;
+}
+
+pub fn val_shard_seed() -> u32 {
+    return 123;
+}
+
+pub fn create_test_dataset() -> usize {
+    return 2;
+}
+
+pub fn create_test_dataset_seeds_differ() -> bool {
+    return train_shard_seed() != val_shard_seed();
+}
+
+// ---------------------------------------------------------------------------
+// Original lines 76-98: the deterministic pseudo-random mix and the
+// four-band token distribution; original line 101 folds every token
+// into the SP4096 vocabulary.
+// ---------------------------------------------------------------------------
+
+pub fn shard_pseudo_random(seed: u32, i: usize) -> u64 {
+    const raw: u64 = (seed as u64 * 1000000 + i) * 1664525 + 1013904223;
+    return raw & 0xFFFFFFFF;
+}
+
+pub fn shard_rand_val(seed: u32, i: usize) -> f32 {
+    const pseudo: u64 = shard_pseudo_random(seed, i);
+    return (pseudo % 1000) as f32 / 1000.0;
+}
+
+pub fn shard_token(seed: u32, i: usize) -> u16 {
+    const pseudo: u64 = shard_pseudo_random(seed, i);
+    const rand_val: f32 = shard_rand_val(seed, i);
+    var token: u16 = 0;
+    if (rand_val < 0.1) {
+        token = 0;
+    } else {
+        if (rand_val < 0.8) {
+            token = (1 + pseudo % 26) as u16;
+        } else {
+            if (rand_val < 0.95) {
+                token = (27 + pseudo % 73) as u16;
+            } else {
+                token = (100 + pseudo % 900) as u16;
+            }
+        }
+    }
+    return token % 4096;
+}
+
+pub fn vocab_size() -> u16 {
+    return 4096;
+}
+
+// The four-band boundaries of the distribution (lines 86-98).
+pub fn space_band() -> f32 {
+    return 0.1;
+}
+
+pub fn letter_band() -> f32 {
+    return 0.8;
+}
+
+pub fn punctuation_band() -> f32 {
+    return 0.95;
+}
+
+pub fn first_letter_token() -> u16 {
+    return 1;
+}
+
+pub fn letter_token_count() -> u16 {
+    return 26;
+}
+
+pub fn first_punctuation_token() -> u16 {
+    return 27;
+}
+
+pub fn first_rare_token() -> u16 {
+    return 100;
+}
+
+// ---------------------------------------------------------------------------
+// Tests: every decision above asserted against values read from the
+// original, including its own shard-size test.
+// ---------------------------------------------------------------------------
+
+// The one-shard plan: the 256-word header (magic, version, the token
+// count, the rest reserved) then the token count in uint16 tokens —
+// the size formula the original's own test asserts.
+pub struct ShardPlan {
+    header_words: usize,
+    token_count: usize,
+    size_bytes: usize,
+}
+
+pub fn create_fineweb_shard(num_tokens: usize) -> ShardPlan {
+    return ShardPlan {
+        header_words: header_word_count(),
+        token_count: num_tokens,
+        size_bytes: shard_size_bytes(num_tokens),
+    };
+}
+
+test header_layout {
+    assert(header_word_count() == 256);
+    assert(header_magic() == 20240520);
+    assert(header_version() == 1);
+    assert(header_token_count_index() == 2);
+}
+
+test shard_plan_carries_the_layout {
+    const plan = create_fineweb_shard(1000);
+    assert(plan.header_words == 256);
+    assert(plan.token_count == 1000);
+    assert(plan.size_bytes == 3024);
+}
+
+test shard_size_matches_the_originals_own_test {
+    assert(shard_size_bytes(1000) == 3024);
+    assert(shard_size_bytes(0) == 1024);
+    assert(shard_size_bytes(1000000) == 2001024);
+}
+
+test main_and_test_dataset_plan {
+    assert(main() == 2);
+    assert(main_dataset_dir() == "./data/datasets/fineweb10B_sp4096");
+    assert(main_tokenizer_dir() == "./data/tokenizers");
+    assert(train_shard_token_count() == 1000000);
+    assert(train_shard_seed() == 42);
+    assert(val_shard_token_count() == 100000);
+    assert(val_shard_seed() == 123);
+    assert(create_test_dataset() == 2);
+    assert(create_test_dataset_seeds_differ());
+}
+
+test pseudo_random_is_deterministic_and_seeded {
+    assert(shard_pseudo_random(42, 0) == shard_pseudo_random(42, 0));
+    assert(shard_pseudo_random(42, 0) != shard_pseudo_random(42, 1));
+    assert(shard_pseudo_random(42, 5) != shard_pseudo_random(123, 5));
+    assert(shard_rand_val(42, 0) >= 0.0);
+    assert(shard_rand_val(42, 0) < 1.0);
+}
+
+test token_distribution_gates {
+    assert(space_band() == 0.1);
+    assert(letter_band() == 0.8);
+    assert(punctuation_band() == 0.95);
+    assert(first_letter_token() == 1);
+    assert(letter_token_count() == 26);
+    assert(first_punctuation_token() == 27);
+    assert(first_rare_token() == 100);
+    assert(vocab_size() == 4096);
+}
+
+test every_token_fits_the_vocab {
+    var i: usize = 0;
+    while (i < 200) {
+        const t: u16 = shard_token(42, i);
+        assert(t < vocab_size());
+        i += 1;
+    }
+    const t123: u16 = shard_token(123, 7);
+    assert(t123 < vocab_size());
+}