spec(port): port specs/port/trios/crates/trios-data/src/lib.t27 (Closes #9499)

specs/port/trios/crates/trios-data/src/lib.t27

@@ -0,0 +1,203 @@
+// trios-data — the batched FineWeb data loader for Trinity training (Φ5.5).
+// Port of gHashTag/trios:crates/trios-data/src/lib.rs
+// (at 8b229e9489ee, 123 lines) — Roadmap Stage 6, goal #4548, target t27c
+// gen-rust.
+//
+// The original is a stub loader: `new` builds a FineWebBatch over a dataset
+// path, `next_batch` answers batches of dummy token sequences until the
+// stub epoch limit, `offset` / `batch_size` read the state back, and
+// `reset` rewinds to the start. The dataset path is plumbing the stub
+// ignores (the original's own comment says so); the decisions the file
+// makes — the loader's initial state (offset 0 under the 1000-sample stub
+// epoch limit), the exhaustion gate at `offset >= max_samples`, the batch
+// shape (batch_size sequences of 128 tokens, every token the dummy ID 42),
+// the offset advancing by batch_size, and the reset to offset 0 — are
+// ported here as pure functions over the state and tested, including the
+// original's own three tests (lines 76-113). No function here moves data
+// in or out, so none carries an `undefined;` transport body: the Vec of
+// Vecs the original allocates is the growable plumbing, and the outcome
+// struct below carries its decisions instead (issue #9499, "Port the
+// decision, not the plumbing").
+//
+// Rust's `Option<Vec<Vec<u32>>>` becomes the availability verdict on the
+// outcome; the `&mut self` advance becomes the `next_offset` the caller
+// feeds back through the state.
+
+module port::trios_data_lib;
+
+// ---------------------------------------------------------------------------
+// Original lines 10-15: `pub struct FineWebBatch { batch_size, offset,
+// max_samples }`. The state the decisions read; the data path is not part
+// of it (the stub never reads it).
+// ---------------------------------------------------------------------------
+
+pub struct FineWebBatchState {
+    batch_size: usize,
+    offset: usize,
+    max_samples: usize,
+}
+
+// The answer `next_batch` decides: whether a batch is available, how many
+// sequences it carries, how many tokens each sequence holds, and where the
+// offset stands after taking it.
+pub struct FineWebBatchOutcome {
+    available: bool,
+    sequence_count: usize,
+    tokens_per_sequence: usize,
+    next_offset: usize,
+}
+
+// ---------------------------------------------------------------------------
+// Original line 24: `pub fn new(_data_path: &str, batch_size: usize) ->
+// Self`. The loader starts at offset 0 under the stub epoch limit of 1000
+// dummy samples; the data path is the unused stub parameter, kept in the
+// port's signature exactly as the original keeps it.
+// ---------------------------------------------------------------------------
+
+pub fn FineWebBatch_new(data_path: str, batch_size: usize) -> FineWebBatchState {
+    return FineWebBatchState {
+        batch_size: batch_size,
+        offset: 0,
+        max_samples: 1000,
+    };
+}
+
+// ---------------------------------------------------------------------------
+// Original lines 40-53: `pub fn next_batch(&mut self) ->
+// Option<Vec<Vec<u32>>>`. The core decision: the dataset is exhausted once
+// the offset reaches the stub limit (None); otherwise the batch carries
+// batch_size sequences of 128 tokens — every token the dummy ID 42 — and
+// the offset advances by batch_size.
+// ---------------------------------------------------------------------------
+
+pub fn FineWebBatch_next_batch(state: FineWebBatchState) -> FineWebBatchOutcome {
+    if (state.offset >= state.max_samples) {
+        return FineWebBatchOutcome {
+            available: false,
+            sequence_count: 0,
+            tokens_per_sequence: 0,
+            next_offset: state.offset,
+        };
+    }
+    return FineWebBatchOutcome {
+        available: true,
+        sequence_count: state.batch_size,
+        tokens_per_sequence: sequence_length(),
+        next_offset: state.offset + state.batch_size,
+    };
+}
+
+// ---------------------------------------------------------------------------
+// Original line 56: `pub fn offset(&self) -> usize`. Reads the current
+// position in the dataset.
+// ---------------------------------------------------------------------------
+
+pub fn FineWebBatch_offset(state: FineWebBatchState) -> usize {
+    return state.offset;
+}
+
+// ---------------------------------------------------------------------------
+// Original line 61: `pub fn batch_size(&self) -> usize`. Reads the number
+// of sequences per batch.
+// ---------------------------------------------------------------------------
+
+pub fn FineWebBatch_batch_size(state: FineWebBatchState) -> usize {
+    return state.batch_size;
+}
+
+// ---------------------------------------------------------------------------
+// Original line 66: `pub fn reset(&mut self)`. Rewinds the loader to the
+// beginning of the dataset: offset back to 0, everything else kept.
+// ---------------------------------------------------------------------------
+
+pub fn FineWebBatch_reset(state: FineWebBatchState) -> FineWebBatchState {
+    return FineWebBatchState {
+        batch_size: state.batch_size,
+        offset: 0,
+        max_samples: state.max_samples,
+    };
+}
+
+// ---------------------------------------------------------------------------
+// Original lines 45-48: the stub's dummy content — every sequence holds 128
+// tokens, every token the ID 42 — and line 28: the stub epoch limit of
+// 1000 samples.
+// ---------------------------------------------------------------------------
+
+pub fn sequence_length() -> usize {
+    return 128;
+}
+
+pub fn dummy_token_id() -> u32 {
+    return 42;
+}
+
+pub fn stub_epoch_limit() -> usize {
+    return 1000;
+}
+
+// ---------------------------------------------------------------------------
+// Tests: every decision above asserted against values read from the
+// original, including the original's own three tests (lines 76-113).
+// ---------------------------------------------------------------------------
+
+test new_batch_loader_state {
+    const state = FineWebBatch_new("/dummy/path", 32);
+    assert(FineWebBatch_batch_size(state) == 32);
+    assert(FineWebBatch_offset(state) == 0);
+    assert(state.max_samples == 1000);
+}
+
+test next_batch_returns_correct_shape {
+    const outcome = FineWebBatch_next_batch(FineWebBatch_new("/dummy/path", 4));
+    assert(outcome.available);
+    assert(outcome.sequence_count == 4);
+    assert(outcome.tokens_per_sequence == 128);
+}
+
+test multiple_batches_advance_by_batch_size {
+    const first = FineWebBatch_next_batch(FineWebBatch_new("/dummy/path", 100));
+    assert(first.available);
+    assert(first.next_offset == 100);
+    const second = FineWebBatch_next_batch(FineWebBatchState {
+        batch_size: 100,
+        offset: 100,
+        max_samples: 1000,
+    });
+    assert(second.available);
+    assert(second.next_offset == 200);
+}
+
+test exhaustion_at_epoch_limit {
+    const at_limit = FineWebBatch_next_batch(FineWebBatchState {
+        batch_size: 4,
+        offset: 1000,
+        max_samples: 1000,
+    });
+    assert(!at_limit.available);
+    assert(at_limit.next_offset == 1000);
+    const before_limit = FineWebBatch_next_batch(FineWebBatchState {
+        batch_size: 4,
+        offset: 996,
+        max_samples: 1000,
+    });
+    assert(before_limit.available);
+    assert(before_limit.next_offset == 1000);
+}
+
+test reset_returns_to_start {
+    const reset = FineWebBatch_reset(FineWebBatchState {
+        batch_size: 32,
+        offset: 200,
+        max_samples: 1000,
+    });
+    assert(FineWebBatch_offset(reset) == 0);
+    assert(FineWebBatch_batch_size(reset) == 32);
+    assert(reset.max_samples == 1000);
+}
+
+test stub_content_constants {
+    assert(sequence_length() == 128);
+    assert(dummy_token_id() == 42);
+    assert(stub_epoch_limit() == 1000);
+}