diff --git a/specs/port/trios/crates/trios-data/src/lib.t27 b/specs/port/trios/crates/trios-data/src/lib.t27 new file mode 100644 index 0000000000..8179d0b86a --- /dev/null +++ b/specs/port/trios/crates/trios-data/src/lib.t27 @@ -0,0 +1,203 @@ +// trios-data — the batched FineWeb data loader for Trinity training (Φ5.5). +// Port of gHashTag/trios:crates/trios-data/src/lib.rs +// (at 8b229e9489ee, 123 lines) — Roadmap Stage 6, goal #4548, target t27c +// gen-rust. +// +// The original is a stub loader: `new` builds a FineWebBatch over a dataset +// path, `next_batch` answers batches of dummy token sequences until the +// stub epoch limit, `offset` / `batch_size` read the state back, and +// `reset` rewinds to the start. The dataset path is plumbing the stub +// ignores (the original's own comment says so); the decisions the file +// makes — the loader's initial state (offset 0 under the 1000-sample stub +// epoch limit), the exhaustion gate at `offset >= max_samples`, the batch +// shape (batch_size sequences of 128 tokens, every token the dummy ID 42), +// the offset advancing by batch_size, and the reset to offset 0 — are +// ported here as pure functions over the state and tested, including the +// original's own three tests (lines 76-113). No function here moves data +// in or out, so none carries an `undefined;` transport body: the Vec of +// Vecs the original allocates is the growable plumbing, and the outcome +// struct below carries its decisions instead (issue #9499, "Port the +// decision, not the plumbing"). +// +// Rust's `Option>>` becomes the availability verdict on the +// outcome; the `&mut self` advance becomes the `next_offset` the caller +// feeds back through the state. + +module port::trios_data_lib; + +// --------------------------------------------------------------------------- +// Original lines 10-15: `pub struct FineWebBatch { batch_size, offset, +// max_samples }`. The state the decisions read; the data path is not part +// of it (the stub never reads it). +// --------------------------------------------------------------------------- + +pub struct FineWebBatchState { + batch_size: usize, + offset: usize, + max_samples: usize, +} + +// The answer `next_batch` decides: whether a batch is available, how many +// sequences it carries, how many tokens each sequence holds, and where the +// offset stands after taking it. +pub struct FineWebBatchOutcome { + available: bool, + sequence_count: usize, + tokens_per_sequence: usize, + next_offset: usize, +} + +// --------------------------------------------------------------------------- +// Original line 24: `pub fn new(_data_path: &str, batch_size: usize) -> +// Self`. The loader starts at offset 0 under the stub epoch limit of 1000 +// dummy samples; the data path is the unused stub parameter, kept in the +// port's signature exactly as the original keeps it. +// --------------------------------------------------------------------------- + +pub fn FineWebBatch_new(data_path: str, batch_size: usize) -> FineWebBatchState { + return FineWebBatchState { + batch_size: batch_size, + offset: 0, + max_samples: 1000, + }; +} + +// --------------------------------------------------------------------------- +// Original lines 40-53: `pub fn next_batch(&mut self) -> +// Option>>`. The core decision: the dataset is exhausted once +// the offset reaches the stub limit (None); otherwise the batch carries +// batch_size sequences of 128 tokens — every token the dummy ID 42 — and +// the offset advances by batch_size. +// --------------------------------------------------------------------------- + +pub fn FineWebBatch_next_batch(state: FineWebBatchState) -> FineWebBatchOutcome { + if (state.offset >= state.max_samples) { + return FineWebBatchOutcome { + available: false, + sequence_count: 0, + tokens_per_sequence: 0, + next_offset: state.offset, + }; + } + return FineWebBatchOutcome { + available: true, + sequence_count: state.batch_size, + tokens_per_sequence: sequence_length(), + next_offset: state.offset + state.batch_size, + }; +} + +// --------------------------------------------------------------------------- +// Original line 56: `pub fn offset(&self) -> usize`. Reads the current +// position in the dataset. +// --------------------------------------------------------------------------- + +pub fn FineWebBatch_offset(state: FineWebBatchState) -> usize { + return state.offset; +} + +// --------------------------------------------------------------------------- +// Original line 61: `pub fn batch_size(&self) -> usize`. Reads the number +// of sequences per batch. +// --------------------------------------------------------------------------- + +pub fn FineWebBatch_batch_size(state: FineWebBatchState) -> usize { + return state.batch_size; +} + +// --------------------------------------------------------------------------- +// Original line 66: `pub fn reset(&mut self)`. Rewinds the loader to the +// beginning of the dataset: offset back to 0, everything else kept. +// --------------------------------------------------------------------------- + +pub fn FineWebBatch_reset(state: FineWebBatchState) -> FineWebBatchState { + return FineWebBatchState { + batch_size: state.batch_size, + offset: 0, + max_samples: state.max_samples, + }; +} + +// --------------------------------------------------------------------------- +// Original lines 45-48: the stub's dummy content — every sequence holds 128 +// tokens, every token the ID 42 — and line 28: the stub epoch limit of +// 1000 samples. +// --------------------------------------------------------------------------- + +pub fn sequence_length() -> usize { + return 128; +} + +pub fn dummy_token_id() -> u32 { + return 42; +} + +pub fn stub_epoch_limit() -> usize { + return 1000; +} + +// --------------------------------------------------------------------------- +// Tests: every decision above asserted against values read from the +// original, including the original's own three tests (lines 76-113). +// --------------------------------------------------------------------------- + +test new_batch_loader_state { + const state = FineWebBatch_new("/dummy/path", 32); + assert(FineWebBatch_batch_size(state) == 32); + assert(FineWebBatch_offset(state) == 0); + assert(state.max_samples == 1000); +} + +test next_batch_returns_correct_shape { + const outcome = FineWebBatch_next_batch(FineWebBatch_new("/dummy/path", 4)); + assert(outcome.available); + assert(outcome.sequence_count == 4); + assert(outcome.tokens_per_sequence == 128); +} + +test multiple_batches_advance_by_batch_size { + const first = FineWebBatch_next_batch(FineWebBatch_new("/dummy/path", 100)); + assert(first.available); + assert(first.next_offset == 100); + const second = FineWebBatch_next_batch(FineWebBatchState { + batch_size: 100, + offset: 100, + max_samples: 1000, + }); + assert(second.available); + assert(second.next_offset == 200); +} + +test exhaustion_at_epoch_limit { + const at_limit = FineWebBatch_next_batch(FineWebBatchState { + batch_size: 4, + offset: 1000, + max_samples: 1000, + }); + assert(!at_limit.available); + assert(at_limit.next_offset == 1000); + const before_limit = FineWebBatch_next_batch(FineWebBatchState { + batch_size: 4, + offset: 996, + max_samples: 1000, + }); + assert(before_limit.available); + assert(before_limit.next_offset == 1000); +} + +test reset_returns_to_start { + const reset = FineWebBatch_reset(FineWebBatchState { + batch_size: 32, + offset: 200, + max_samples: 1000, + }); + assert(FineWebBatch_offset(reset) == 0); + assert(FineWebBatch_batch_size(reset) == 32); + assert(reset.max_samples == 1000); +} + +test stub_content_constants { + assert(sequence_length() == 128); + assert(dummy_token_id() == 42); + assert(stub_epoch_limit() == 1000); +}