From c35d15986f506e3b2b031d33d8660dd8ddb9633e Mon Sep 17 00:00:00 2001 From: Dmitrii Vasilev Date: Sun, 18 Oct 2026 17:18:21 +0700 Subject: [PATCH 1/3] spec(hslm): nn/hslm_tokenizer and ml/hslm_corpus, first cut (Refs #10372) Co-Authored-By: Claude Sonnet 5.5 --- specs/ml/hslm_corpus.t27 | 971 ++++++++++++++++++++++++++++++++++++ specs/nn/hslm_tokenizer.t27 | 782 +++++++++++++++++++++++++++++ 2 files changed, 1753 insertions(+) create mode 100644 specs/ml/hslm_corpus.t27 create mode 100644 specs/nn/hslm_tokenizer.t27 diff --git a/specs/ml/hslm_corpus.t27 b/specs/ml/hslm_corpus.t27 new file mode 100644 index 0000000000..b05c0b884e --- /dev/null +++ b/specs/ml/hslm_corpus.t27 @@ -0,0 +1,971 @@ +// SPDX-License-Identifier: Apache-2.0 +; specs/ml/hslm_corpus.t27 -- a text corpus becomes a pinned token file, in t27, no heap. +; Part of gHashTag/t27#10024 (our own model), cell 25 "t27c --backend ml"; issue #10372. + +; WHAT IT DOES: main() reads two text files, tokenizes every story with the HSLM 729-token BPE +; (nn/hslm_tokenizer.t27) and writes two token files: +; /tmp/hslm/valid.txt -> /tmp/hslm/valid.tok +; /tmp/hslm/train.txt -> /tmp/hslm/train.tok +; (main cannot take argv in the ml backend: host_open takes a string literal, so the paths are +; fixed and both files are converted in one run.) + +; THE TOKEN FILE (the contract a trainer reads): +; bytes 0..3 the magic "HTOK" (72 84 79 75) +; bytes 4..7 u32 little endian, the format version, 1 +; bytes 8..11 u32 little endian, N, the number of tokens +; bytes 12.. N tokens, each a u16 little endian, every one below 729 +; so the file is exactly 12 + 2N bytes. The ids are the ones nn/hslm_tokenizer.t27 gives. + +; STORIES. TinyStories separates its stories with a line that is exactly <|endoftext|>. A story +; is the text between two such lines; the line terminator right before the marker belongs to the +; marker, so it is not part of the story. Every story becomes its tokens followed by one EOS +; (id 2). The marker text itself is not tokenized: a line that merely CONTAINS it is ordinary +; text, and the tokenizer turns it into id 390 like any other bytes (a test pins this). Empty +; stories (two markers in a row) produce nothing. Text after the last marker, if any, is one more +; story and gets its EOS too. Newlines inside a story are kept as byte 10 (id 14, merged into a +; longer token where the table says so). +; Source line: trinity-training @ 3e9648c1817a97c5f9b8c727b0f8e2291928eead, src/hslm/data.zig, +; Dataset.addText and Dataset.loadTextFile. THIS IS NOT THE SAME CUT AS THE ZIG LOADER: the Zig +; takes one LINE as one story (lines of 10 bytes or fewer are dropped, newlines are lost), cuts it +; into 2000-byte pieces and wraps EACH piece in BOS and EOS, so a story is split and the marker +; line is tokenized as text. Here a story is one marker-delimited run and carries one EOS, no BOS. +; That is the choice this file makes, and the reason is that a model trained on it sees whole +; stories, with their paragraph breaks, ended by a single EOS. The tokenizer itself is the Zig +; tokenizer's, id for id. + +; CHUNKS. The merge scan of nn/hslm_tokenizer.t27 works on a window of at most 8192 bytes, so a +; story longer than that is tokenized in pieces cut at a line start (the newline in front of that +; line stays in the next piece), or, when a single line is longer than the window, at 8192 bytes. +; No story in TinyStories is that long (both files were checked); the tests cover the path. + +; MEMORY. No heap. The story window is a module array of 8192 bytes, tokens collect in an array of +; 32768 and are written out when it fills, the file is read 65536 bytes at a time. The token file +; needs its count in front, and there is no seek, so the tokens go to a temporary file +; .tok.body first; the final file is the header followed by a copy of the body, and the +; body is emptied (opened for writing and closed) at the end. After that the final file is read +; back and checked: magic, version, count equal to the tokens counted while writing, every id +; below 729, number of EOS equal to the number of stories, length 12 + 2N. + +; HOST CALLS: host_open(path literal, mode 0 read or 1 create), host_read, host_write, host_close, +; as in specs/ml/host_io_model.t27. Writes are retried until the buffer is out. +; phi^2 + 1/phi^2 = 3 | TRINITY + +module hslm_corpus; + +use nn::hslm_tokenizer; + +pub const HC_OK : i32 = 0; +pub const HC_ERR_OPEN : i32 = 70; +pub const HC_ERR_READ : i32 = 71; +pub const HC_ERR_WRITE : i32 = 72; +pub const HC_ERR_VERIFY : i32 = 73; +pub const HC_ERR_CAP : i32 = 74; + +pub const HEADER_BYTES : usize = 12; +pub const FORMAT_VERSION : u32 = 1; +pub const CHUNK_CAP : usize = 8192; +pub const TOK_CAP : usize = 32768; +pub const IO_CAP : usize = 65536; + +/// The story separator: a line that is exactly this. +pub const MARKER : [13]u8 = [60, 124, 101, 110, 100, 111, 102, 116, 101, 120, 116, 124, 62]; + +// ---- state ----------------------------------------------------------------------------------- + +var sbuf : [8192]u8 = undefined; +var slen : usize = 0; +var lstart : usize = 0; +var cont : bool = false; +var pending : bool = false; +var tbuf : [32768]u16 = undefined; +var tn : usize = 0; +var obuf : [65536]u8 = undefined; +var rbuf : [65536]u8 = undefined; +var sink_fd : i32 = -1; +var total : u32 = 0; +var stories : u32 = 0; +var ioerr : i32 = 0; +var hdr : [12]u8 = undefined; + +var vhdr : [12]u8 = undefined; +var vpos : u32 = 0; +var vlo : u32 = 0; +var vtoks : u32 = 0; +var veos : u32 = 0; +var vbad : i32 = 0; + +fn reset_state(sink: i32) -> void { + slen = 0; + lstart = 0; + cont = false; + pending = false; + tn = 0; + sink_fd = sink; + total = 0; + stories = 0; + ioerr = 0; +} + +// ---- little endian helpers ------------------------------------------------------------------- + +fn put32(buf: []u8, off: usize, v: u32) -> void { + var x : u32 = v; + var k : usize = 0; + while (k < 4) { + buf[off + k] = @intCast(@mod(x, 256)); + x = @divTrunc(x, 256); + k = k + 1; + } +} + +fn get32(buf: []const u8, off: usize) -> u32 { + var x : u32 = 0; + var k : usize = 4; + while (k > 0) { + k = k - 1; + const b : u32 = @intCast(buf[off + k]); + x = x * 256 + b; + } + return x; +} + +/// The 12 header bytes for a file of count tokens, into hdr. +fn make_header(count: u32) -> void { + hdr[0] = 72; + hdr[1] = 84; + hdr[2] = 79; + hdr[3] = 75; + put32(&hdr, 4, FORMAT_VERSION); + put32(&hdr, 8, count); +} + +/// tbuf[0..tn] as u16 little endian into obuf. Returns the byte count, 2 * tn. +fn pack_tokens() -> usize { + var i : usize = 0; + while (i < tn) { + const v : u16 = tbuf[i]; + obuf[2 * i] = @intCast(@mod(v, 256)); + obuf[2 * i + 1] = @intCast(@divTrunc(v, 256)); + i = i + 1; + } + return 2 * tn; +} + +fn write_all(fd: i32, buf: []const u8) -> i32 { + var done : usize = 0; + while (done < buf.len) { + const w : i32 = host_write(fd, buf[done .. buf.len]); + if (w <= 0) { + return HC_ERR_WRITE; + } + done = done + @as(usize, @intCast(w)); + } + return HC_OK; +} + +// ---- the story machine ----------------------------------------------------------------------- + +/// Write the collected tokens to the sink and empty tbuf. With no sink (the tests) it keeps them. +fn flush_tokens() -> void { + if (sink_fd < 0 or ioerr != 0) { + return; + } + const nb : usize = pack_tokens(); + const rc : i32 = write_all(sink_fd, obuf[0 .. nb]); + if (rc != HC_OK) { + ioerr = rc; + } + tn = 0; +} + +/// Tokenize sbuf[0..n] (n <= 8192) and append the ids to tbuf. +fn emit_chunk(n: usize) -> void { + if (tn + n > TOK_CAP) { + flush_tokens(); + } + if (tn + n > TOK_CAP) { + ioerr = HC_ERR_CAP; + return; + } + const m : usize = encode_window(sbuf[0 .. n], n, tbuf[tn .. tn + n]); + tn = tn + m; + total = total + @as(u32, @intCast(m)); +} + +fn emit_eos() -> void { + if (tn + 1 > TOK_CAP) { + flush_tokens(); + } + if (tn + 1 > TOK_CAP) { + ioerr = HC_ERR_CAP; + return; + } + tbuf[tn] = EOS_TOKEN; + tn = tn + 1; + total = total + 1; + stories = stories + 1; +} + +/// The window is full: tokenize all the lines before the current one and keep the current line. +/// The newline that ends the last emitted line is kept as the first byte of what stays, so that a +/// story's final newline is always still in the window when the story closes. +fn flush_full() -> void { + if (lstart >= 2) { + const cut : usize = lstart - 1; + emit_chunk(cut); + var k : usize = 0; + while (cut + k < slen) { + sbuf[k] = sbuf[cut + k]; + k = k + 1; + } + slen = slen - cut; + lstart = 1; + } else { + // one line fills the window: emit it all and carry on in the middle of that line + emit_chunk(slen); + slen = 0; + lstart = 0; + cont = true; + } + pending = true; +} + +/// A story ends: drop the line terminator before the marker, tokenize what is left, add EOS. +fn close_story() -> void { + if (slen > 0 and sbuf[slen - 1] == 10) { + slen = slen - 1; + } + if (slen > 0) { + emit_chunk(slen); + pending = true; + } + if (pending) { + emit_eos(); + } + slen = 0; + lstart = 0; + cont = false; + pending = false; +} + +/// Is the current line, sbuf[lstart..slen], exactly the marker? The caller checked its length. +fn line_is_marker() -> bool { + var k : usize = 0; + while (k < 13) { + if (sbuf[lstart + k] != MARKER[k]) { + return false; + } + k = k + 1; + } + return true; +} + +fn feed_byte(b: u8) -> void { + if (slen >= CHUNK_CAP) { + flush_full(); + } + if (b == 10) { + if (!cont and slen - lstart == 13 and line_is_marker()) { + slen = lstart; + close_story(); + } else { + sbuf[slen] = 10; + slen = slen + 1; + lstart = slen; + cont = false; + } + } else { + sbuf[slen] = b; + slen = slen + 1; + } +} + +/// End of input: a last line without a newline may still be the marker; then close the story. +fn finish() -> void { + if (!cont and slen - lstart == 13 and line_is_marker()) { + slen = lstart; + } + close_story(); + flush_tokens(); +} + +/// Run the machine over a byte slice (the tests, and anything that has its text in memory). +fn convert_bytes(data: []const u8, sink: i32) -> void { + reset_state(sink); + var i : usize = 0; + while (i < data.len) { + feed_byte(data[i]); + i = i + 1; + } + finish(); +} + +// ---- the file passes ------------------------------------------------------------------------- + +/// Pass 1: text file in_fd -> tokens (no header) on body_fd. Leaves total and stories set. +fn run_convert(in_fd: i32, body_fd: i32) -> i32 { + reset_state(body_fd); + var going : bool = true; + while (going and ioerr == 0) { + const got : i32 = host_read(in_fd, rbuf[0 .. IO_CAP]); + if (got < 0) { + ioerr = HC_ERR_READ; + } else if (got == 0) { + going = false; + } else { + const n : usize = @intCast(got); + var i : usize = 0; + while (i < n) { + feed_byte(rbuf[i]); + i = i + 1; + } + } + } + if (ioerr == 0) { + finish(); + } + return ioerr; +} + +/// Pass 2: header (with total) and then a copy of the body, onto out_fd. +fn run_assemble(body_fd: i32, out_fd: i32) -> i32 { + make_header(total); + var rc : i32 = write_all(out_fd, hdr[0 .. 12]); + if (rc != HC_OK) { + return rc; + } + var copied : u32 = 0; + var going : bool = true; + while (going) { + const got : i32 = host_read(body_fd, rbuf[0 .. IO_CAP]); + if (got < 0) { + return HC_ERR_READ; + } + if (got == 0) { + going = false; + } else { + const n : usize = @intCast(got); + rc = write_all(out_fd, rbuf[0 .. n]); + if (rc != HC_OK) { + return rc; + } + copied = copied + @as(u32, @intCast(got)); + } + } + if (copied != 2 * total) { + return HC_ERR_VERIFY; + } + return HC_OK; +} + +// ---- the verifier: a byte-at-a-time check of a finished token file ---------------------------- + +fn vreset() -> void { + vpos = 0; + vlo = 0; + vtoks = 0; + veos = 0; + vbad = 0; +} + +fn vstep(b: u8) -> void { + const bb : u32 = @intCast(b); + if (vpos < 12) { + vhdr[@intCast(vpos)] = b; + } else if (@mod(vpos, 2) == 0) { + vlo = bb; + } else { + const tok : u32 = vlo + 256 * bb; + if (tok >= 729) { + vbad = vbad + 1; + } + if (tok == 2) { + veos = veos + 1; + } + vtoks = vtoks + 1; + } + vpos = vpos + 1; +} + +fn vfinish(expect_count: u32, expect_eos: u32) -> i32 { + if (vpos < 12) { + return HC_ERR_VERIFY; + } + if (vhdr[0] != 72 or vhdr[1] != 84 or vhdr[2] != 79 or vhdr[3] != 75) { + return HC_ERR_VERIFY; + } + if (get32(&vhdr, 4) != FORMAT_VERSION) { + return HC_ERR_VERIFY; + } + if (get32(&vhdr, 8) != expect_count) { + return HC_ERR_VERIFY; + } + if (@mod(vpos, 2) != 0 or vtoks != expect_count or veos != expect_eos or vbad != 0) { + return HC_ERR_VERIFY; + } + return HC_OK; +} + +/// Pass 3: read the finished file back and check it against what was written. +fn run_verify(fd: i32, expect_count: u32, expect_eos: u32) -> i32 { + vreset(); + var going : bool = true; + while (going) { + const got : i32 = host_read(fd, rbuf[0 .. IO_CAP]); + if (got < 0) { + return HC_ERR_READ; + } + if (got == 0) { + going = false; + } else { + const n : usize = @intCast(got); + var i : usize = 0; + while (i < n) { + vstep(rbuf[i]); + i = i + 1; + } + } + } + return vfinish(expect_count, expect_eos); +} + +// ---- the two files --------------------------------------------------------------------------- + +/// /tmp/hslm/valid.txt -> /tmp/hslm/valid.tok +pub fn convert_valid() -> i32 { + const in_fd : i32 = host_open("/tmp/hslm/valid.txt", 0); + if (in_fd < 0) { + return HC_ERR_OPEN; + } + const body_w : i32 = host_open("/tmp/hslm/valid.tok.body", 1); + if (body_w < 0) { + return HC_ERR_OPEN; + } + const rc1 : i32 = run_convert(in_fd, body_w); + const c1 : i32 = host_close(in_fd); + const c2 : i32 = host_close(body_w); + if (rc1 != HC_OK) { + return rc1; + } + if (c1 != 0 or c2 != 0) { + return HC_ERR_WRITE; + } + const body_r : i32 = host_open("/tmp/hslm/valid.tok.body", 0); + if (body_r < 0) { + return HC_ERR_OPEN; + } + const out_fd : i32 = host_open("/tmp/hslm/valid.tok", 1); + if (out_fd < 0) { + return HC_ERR_OPEN; + } + const rc2 : i32 = run_assemble(body_r, out_fd); + const c3 : i32 = host_close(body_r); + const c4 : i32 = host_close(out_fd); + if (rc2 != HC_OK) { + return rc2; + } + if (c3 != 0 or c4 != 0) { + return HC_ERR_WRITE; + } + const chk : i32 = host_open("/tmp/hslm/valid.tok", 0); + if (chk < 0) { + return HC_ERR_OPEN; + } + const rc3 : i32 = run_verify(chk, total, stories); + const c5 : i32 = host_close(chk); + if (rc3 != HC_OK) { + return rc3; + } + if (c5 != 0) { + return HC_ERR_READ; + } + const z : i32 = host_open("/tmp/hslm/valid.tok.body", 1); + if (z < 0) { + return HC_ERR_OPEN; + } + const c6 : i32 = host_close(z); + if (c6 != 0) { + return HC_ERR_WRITE; + } + return HC_OK; +} + +/// /tmp/hslm/train.txt -> /tmp/hslm/train.tok +pub fn convert_train() -> i32 { + const in_fd : i32 = host_open("/tmp/hslm/train.txt", 0); + if (in_fd < 0) { + return HC_ERR_OPEN; + } + const body_w : i32 = host_open("/tmp/hslm/train.tok.body", 1); + if (body_w < 0) { + return HC_ERR_OPEN; + } + const rc1 : i32 = run_convert(in_fd, body_w); + const c1 : i32 = host_close(in_fd); + const c2 : i32 = host_close(body_w); + if (rc1 != HC_OK) { + return rc1; + } + if (c1 != 0 or c2 != 0) { + return HC_ERR_WRITE; + } + const body_r : i32 = host_open("/tmp/hslm/train.tok.body", 0); + if (body_r < 0) { + return HC_ERR_OPEN; + } + const out_fd : i32 = host_open("/tmp/hslm/train.tok", 1); + if (out_fd < 0) { + return HC_ERR_OPEN; + } + const rc2 : i32 = run_assemble(body_r, out_fd); + const c3 : i32 = host_close(body_r); + const c4 : i32 = host_close(out_fd); + if (rc2 != HC_OK) { + return rc2; + } + if (c3 != 0 or c4 != 0) { + return HC_ERR_WRITE; + } + const chk : i32 = host_open("/tmp/hslm/train.tok", 0); + if (chk < 0) { + return HC_ERR_OPEN; + } + const rc3 : i32 = run_verify(chk, total, stories); + const c5 : i32 = host_close(chk); + if (rc3 != HC_OK) { + return rc3; + } + if (c5 != 0) { + return HC_ERR_READ; + } + const z : i32 = host_open("/tmp/hslm/train.tok.body", 1); + if (z < 0) { + return HC_ERR_OPEN; + } + const c6 : i32 = host_close(z); + if (c6 != 0) { + return HC_ERR_WRITE; + } + return HC_OK; +} + +/// The program `t27c compile --backend ml` builds: valid first, then train; die loudly on any error. +pub fn main() -> void { + const v : i32 = convert_valid(); + assert v == HC_OK; + const t : i32 = convert_train(); + assert t == HC_OK; +} + +// ---- tests (byte slices in memory, no files) --------------------------------------------------- + +const ONCE_T : [16]u8 = [79, 110, 99, 101, 32, 117, 112, 111, 110, 32, 97, 32, 116, 105, 109, 101]; +const FOX_T : [19]u8 = [84, 104, 101, 32, 113, 117, 105, 99, 107, 32, 98, 114, 111, 119, 110, 32, 102, 111, 120]; +const ABC_T : [3]u8 = [97, 98, 99]; +const LINE2_T : [42]u8 = [79, 110, 99, 101, 32, 117, 112, 111, 110, 32, 97, 32, 116, 105, 109, 101, 44, 10, 116, 104, 101, 114, 101, 32, 119, 97, 115, 32, 97, 32, 108, 105, 116, 116, 108, 101, 32, 103, 105, 114, 108, 46]; + +var tin : [20100]u8 = undefined; +var tex : [20100]u8 = undefined; +var dbuf : [24576]u8 = undefined; + +fn put_bytes(dst: []u8, at: usize, src: []const u8) -> usize { + var i : usize = 0; + while (i < src.len) { + dst[at + i] = src[i]; + i = i + 1; + } + return at + src.len; +} + +fn put_nl(dst: []u8, at: usize) -> usize { + dst[at] = 10; + return at + 1; +} + +/// The collected tokens are exactly want[0..k]. +fn tokens_are(want: []const u16, k: usize) -> bool { + if (tn != k) { + return false; + } + var i : usize = 0; + while (i < k) { + if (tbuf[i] != want[i]) { + return false; + } + i = i + 1; + } + return true; +} + +/// The collected tokens are one story: its text is exp[0..n] and the last id is EOS. +fn story_is(exp: []const u8, n: usize) -> bool { + if (tn < 2) { + return false; + } + if (tbuf[tn - 1] != EOS_TOKEN) { + return false; + } + const m : usize = decode(tbuf[0 .. tn - 1], &dbuf); + if (m != n) { + return false; + } + var i : usize = 0; + while (i < n) { + if (dbuf[i] != exp[i]) { + return false; + } + i = i + 1; + } + return true; +} + +test two_stories_each_end_in_one_eos { + // "Once upon a time" "The quick brown fox" ; ids from the Zig tokenizer + var p : usize = 0; + p = put_bytes(&tin, p, &ONCE_T); + p = put_nl(&tin, p); + p = put_bytes(&tin, p, &MARKER); + p = put_nl(&tin, p); + p = put_bytes(&tin, p, &FOX_T); + p = put_nl(&tin, p); + p = put_bytes(&tin, p, &MARKER); + p = put_nl(&tin, p); + convert_bytes(tin[0 .. p], -1); + const want : [14]u16 = [463, 105, 2, 427, 683, 109, 490, 102, 397, 507, 36, 660, 124, 2]; + assert ioerr == 0; + assert tokens_are(&want, 14); + assert total == 14; + assert stories == 2; +} + +test a_newline_inside_a_story_is_kept { + var p : usize = 0; + p = put_bytes(&tin, p, &LINE2_T); + p = put_nl(&tin, p); + p = put_bytes(&tin, p, &MARKER); + p = put_nl(&tin, p); + convert_bytes(tin[0 .. p], -1); + // the Zig tokenizer on "Once upon a time,\nthere was a little girl." is 1 463 105 48 14 464 410 504 112 50 2 + const want : [10]u16 = [463, 105, 48, 14, 464, 410, 504, 112, 50, 2]; + assert tokens_are(&want, 10); + assert stories == 1; +} + +test empty_stories_make_nothing { + var p : usize = 0; + p = put_bytes(&tin, p, &MARKER); + p = put_nl(&tin, p); + p = put_bytes(&tin, p, &MARKER); + p = put_nl(&tin, p); + p = put_nl(&tin, p); + p = put_bytes(&tin, p, &MARKER); + p = put_nl(&tin, p); + p = put_bytes(&tin, p, &ABC_T); + p = put_nl(&tin, p); + p = put_bytes(&tin, p, &MARKER); + p = put_nl(&tin, p); + convert_bytes(tin[0 .. p], -1); + const want : [4]u16 = [101, 102, 103, 2]; + assert tokens_are(&want, 4); + assert stories == 1; +} + +test no_input_means_no_tokens { + convert_bytes(tin[0 .. 0], -1); + assert ioerr == 0; + assert tn == 0; + assert total == 0; + assert stories == 0; +} + +test only_blank_lines_make_nothing { + tin[0] = 10; + tin[1] = 10; + convert_bytes(tin[0 .. 2], -1); + assert tn == 0; + assert stories == 0; +} + +test the_marker_inside_a_line_is_text { + // "x<|endoftext|>\nabc": one story, the marker text tokenized (the Zig gives 1 124 391 101 102 103 2) + var p : usize = 0; + tin[p] = 120; + p = p + 1; + p = put_bytes(&tin, p, &MARKER); + p = put_nl(&tin, p); + p = put_bytes(&tin, p, &ABC_T); + convert_bytes(tin[0 .. p], -1); + const want : [6]u16 = [124, 391, 101, 102, 103, 2]; + assert tokens_are(&want, 6); + assert stories == 1; +} + +test a_marker_at_the_end_without_a_newline_still_closes { + var p : usize = 0; + p = put_bytes(&tin, p, &ABC_T); + p = put_nl(&tin, p); + p = put_bytes(&tin, p, &MARKER); + convert_bytes(tin[0 .. p], -1); + const want : [4]u16 = [101, 102, 103, 2]; + assert tokens_are(&want, 4); +} + +test a_last_story_without_a_marker_gets_its_eos { + const want : [4]u16 = [101, 102, 103, 2]; + convert_bytes(&ABC_T, -1); + assert tokens_are(&want, 4); + var p : usize = 0; + p = put_bytes(&tin, p, &ABC_T); + p = put_nl(&tin, p); + convert_bytes(tin[0 .. p], -1); + assert tokens_are(&want, 4); +} + +test blank_lines_inside_a_story_are_kept { + // "abc\n\nabc" then the marker + var p : usize = 0; + p = put_bytes(&tex, p, &ABC_T); + p = put_nl(&tex, p); + p = put_nl(&tex, p); + p = put_bytes(&tex, p, &ABC_T); + const n : usize = p; + p = put_bytes(&tin, 0, tex[0 .. n]); + p = put_nl(&tin, p); + p = put_bytes(&tin, p, &MARKER); + p = put_nl(&tin, p); + convert_bytes(tin[0 .. p], -1); + assert story_is(tex[0 .. n], n); +} + +test only_one_newline_before_the_marker_is_dropped { + // "abc\n\n" then the marker: the story is "abc\n" + var p : usize = 0; + p = put_bytes(&tex, p, &ABC_T); + p = put_nl(&tex, p); + const n : usize = p; + p = put_bytes(&tin, 0, tex[0 .. n]); + p = put_nl(&tin, p); + p = put_bytes(&tin, p, &MARKER); + p = put_nl(&tin, p); + convert_bytes(tin[0 .. p], -1); + assert story_is(tex[0 .. n], n); +} + +test a_story_longer_than_the_window_is_cut_at_line_starts_and_decodes_back { + // 200 lines of 99 letters: 19999 bytes of story, then the marker + var line : usize = 0; + var p : usize = 0; + var q : usize = 0; + while (line < 200) { + var j : usize = 0; + while (j < 99) { + const c : u8 = @intCast(97 + @mod(j + line, 26)); + tin[p] = c; + tex[q] = c; + p = p + 1; + q = q + 1; + j = j + 1; + } + tin[p] = 10; + p = p + 1; + if (line < 199) { + tex[q] = 10; + q = q + 1; + } + line = line + 1; + } + assert q == 19999; + p = put_bytes(&tin, p, &MARKER); + p = put_nl(&tin, p); + convert_bytes(tin[0 .. p], -1); + assert ioerr == 0; + assert stories == 1; + assert story_is(tex[0 .. q], q); +} + +test one_line_longer_than_the_window_is_cut_at_8192 { + var p : usize = 0; + while (p < 9000) { + const c : u8 = @intCast(98 + @mod(p, 5)); + tin[p] = c; + tex[p] = c; + p = p + 1; + } + tin[p] = 10; + p = p + 1; + p = put_bytes(&tin, p, &MARKER); + p = put_nl(&tin, p); + convert_bytes(tin[0 .. p], -1); + assert ioerr == 0; + assert stories == 1; + assert story_is(tex[0 .. 9000], 9000); +} + +test a_marker_after_a_cut_line_is_not_a_boundary { + // 8192 letters, then the marker text on the same line: the rest of that line is exactly 13 + // bytes, but it is not a line of its own, so no story ends there + var p : usize = 0; + while (p < 8192) { + tin[p] = 99; + tex[p] = 99; + p = p + 1; + } + var q : usize = 8192; + q = put_bytes(&tex, q, &MARKER); + q = put_nl(&tex, q); + tex[q] = 100; + q = q + 1; + p = put_bytes(&tin, p, &MARKER); + p = put_nl(&tin, p); + tin[p] = 100; + p = p + 1; + p = put_nl(&tin, p); + p = put_bytes(&tin, p, &MARKER); + p = put_nl(&tin, p); + convert_bytes(tin[0 .. p], -1); + assert ioerr == 0; + assert stories == 1; + assert story_is(tex[0 .. q], q); +} + +test the_header_is_htok_version_count { + make_header(16909060); + assert hdr[0] == 72 and hdr[1] == 84 and hdr[2] == 79 and hdr[3] == 75; + assert hdr[4] == 1 and hdr[5] == 0 and hdr[6] == 0 and hdr[7] == 0; + assert hdr[8] == 4 and hdr[9] == 3 and hdr[10] == 2 and hdr[11] == 1; + assert get32(&hdr, 8) == 16909060; +} + +test tokens_are_packed_as_u16_little_endian { + tbuf[0] = 463; + tbuf[1] = 2; + tbuf[2] = 728; + tn = 3; + const nb : usize = pack_tokens(); + assert nb == 6; + assert obuf[0] == 207 and obuf[1] == 1; + assert obuf[2] == 2 and obuf[3] == 0; + assert obuf[4] == 216 and obuf[5] == 2; + tn = 0; +} + +test a_converted_stream_passes_the_verifier { + var p : usize = 0; + p = put_bytes(&tin, p, &ONCE_T); + p = put_nl(&tin, p); + p = put_bytes(&tin, p, &MARKER); + p = put_nl(&tin, p); + p = put_bytes(&tin, p, &FOX_T); + p = put_nl(&tin, p); + p = put_bytes(&tin, p, &MARKER); + p = put_nl(&tin, p); + convert_bytes(tin[0 .. p], -1); + make_header(total); + const nb : usize = pack_tokens(); + vreset(); + var i : usize = 0; + while (i < 12) { + vstep(hdr[i]); + i = i + 1; + } + var j : usize = 0; + while (j < nb) { + vstep(obuf[j]); + j = j + 1; + } + assert vtoks == 14; + assert vfinish(total, stories) == HC_OK; +} + +/// A good 3-token file (463 105 2) in tin; returns its length, 18. +fn good_stream() -> usize { + make_header(3); + var i : usize = 0; + while (i < 12) { + tin[i] = hdr[i]; + i = i + 1; + } + tin[12] = 207; + tin[13] = 1; + tin[14] = 105; + tin[15] = 0; + tin[16] = 2; + tin[17] = 0; + return 18; +} + +fn verify_tin(n: usize, count: u32, eos: u32) -> i32 { + vreset(); + var i : usize = 0; + while (i < n) { + vstep(tin[i]); + i = i + 1; + } + return vfinish(count, eos); +} + +test the_verifier_accepts_a_good_file { + const n : usize = good_stream(); + assert verify_tin(n, 3, 1) == HC_OK; +} + +test the_verifier_refuses_a_bad_magic { + const n : usize = good_stream(); + tin[0] = 73; + assert verify_tin(n, 3, 1) == HC_ERR_VERIFY; +} + +test the_verifier_refuses_a_bad_version { + const n : usize = good_stream(); + tin[4] = 2; + assert verify_tin(n, 3, 1) == HC_ERR_VERIFY; +} + +test the_verifier_refuses_a_count_that_is_not_the_file { + const n : usize = good_stream(); + assert verify_tin(n, 4, 1) == HC_ERR_VERIFY; + tin[8] = 4; + assert verify_tin(n, 3, 1) == HC_ERR_VERIFY; + assert verify_tin(n, 4, 1) == HC_ERR_VERIFY; +} + +test the_verifier_refuses_a_truncated_or_odd_file { + const n : usize = good_stream(); + assert verify_tin(n - 1, 3, 1) == HC_ERR_VERIFY; + assert verify_tin(n - 2, 3, 1) == HC_ERR_VERIFY; + assert verify_tin(8, 3, 1) == HC_ERR_VERIFY; +} + +test the_verifier_refuses_an_id_outside_the_vocabulary { + const n : usize = good_stream(); + tin[12] = 217; + tin[13] = 2; + assert verify_tin(n, 3, 1) == HC_ERR_VERIFY; +} + +test the_verifier_refuses_a_wrong_story_count { + const n : usize = good_stream(); + assert verify_tin(n, 3, 2) == HC_ERR_VERIFY; + assert verify_tin(n, 3, 0) == HC_ERR_VERIFY; +} + +test the_chunk_size_is_the_tokenizer_window { + assert CHUNK_CAP == WORK_CAP; + assert TOK_CAP * 2 == IO_CAP; + assert HEADER_BYTES == 12; +} + +test the_comparison_helpers_can_fail { + // tokens_are must say no to a wrong id and a wrong length, story_is to a wrong byte + convert_bytes(&ABC_T, -1); + const right : [4]u16 = [101, 102, 103, 2]; + const wrong : [4]u16 = [101, 102, 104, 2]; + assert tokens_are(&right, 4); + assert !tokens_are(&wrong, 4); + assert !tokens_are(&right, 3); + const same : [3]u8 = [97, 98, 99]; + const other : [3]u8 = [97, 98, 100]; + assert story_is(&same, 3); + assert !story_is(&other, 3); + assert !story_is(&same, 2); +} diff --git a/specs/nn/hslm_tokenizer.t27 b/specs/nn/hslm_tokenizer.t27 new file mode 100644 index 0000000000..5d12081d3e --- /dev/null +++ b/specs/nn/hslm_tokenizer.t27 @@ -0,0 +1,782 @@ +// SPDX-License-Identifier: Apache-2.0 +; specs/nn/hslm_tokenizer.t27 -- the HSLM byte-level BPE tokenizer: 729 tokens, 469 merges, in t27. +; Part of gHashTag/t27#10024 (our own model), cell 25 "t27c --backend ml"; issue #10372. + +; SOURCE OF TRUTH: gHashTag/trinity-training @ 3e9648c1817a97c5f9b8c727b0f8e2291928eead +; src/hslm/tokenizer.zig Tokenizer.encode, Tokenizer.decode, the layout constants +; src/hslm/bpe_merges.zig merge_rules, merge_byte_lengths, merge_byte_data (469 rules, 1934 bytes) +; The four data tables below (MERGE_LEFT, MERGE_RIGHT, MERGE_LEN, MERGE_BYTES) were converted from +; bpe_merges.zig mechanically with sed and awk, nothing was retyped. MERGE_START is the running sum +; of MERGE_LEN, which tokenizer.zig computes at init (byte_offsets); here it is data too, and a test +; checks that it is that sum. + +; THE VOCABULARY (729 = 3^6): +; 0..3 specials: PAD 0, BOS 1, EOS 2, UNK 3 +; 4..259 the 256 byte values, token = byte + 4 +; 260..728 469 merges, token = 260 + rule index; a rule joins two earlier tokens +; The rule list is the priority list: encode applies rule 0 to the whole sequence, then rule 1, +; and so on (tokenizer.zig: "lower merge index = higher priority"). That is a scan per rule, not a +; pair-frequency search, and it is what is ported. + +; THE PORT IS EXACT, INCLUDING THE EDGES. encode(text, output) puts BOS first and EOS last and +; truncates the way the Zig does: the text is cut to max(len(output) - 2, 0) bytes and to the +; 8192-token merge window; the merged tokens are cut to leave room for the EOS. decode skips PAD, +; BOS and EOS, writes '?' for UNK, expands a merge token to its bytes, ignores a token >= 729, and +; stops writing when the output is full. The tests pin the ids that the Zig tokenizer returns for +; the same strings; those ids were produced by running tokenizer.zig itself (zig 0.16.0 on the +; Railway lab, a four-line main that calls Tokenizer.encode) and are copied here, not computed by +; this file. + +; WHAT IS DIFFERENT FROM THE ZIG, so nobody reads this as more: +; - The Zig builds a hash map merge_lookup at init and never reads it again; encode and decode +; do not use it. It is not ported. +; - Tokenizer.tokenToTrits and tritsToToken (the 6-trit view of an id) are not ported. +; - encode_window is the Zig's steps 1 and 2 on their own (text bytes -> merged tokens, no BOS or +; EOS), taking a caller-owned workspace; hslm_corpus uses it so that a story is not wrapped in +; BOS and EOS. encode itself is the Zig function. +; - No heap, no module state: every buffer is the caller's or a local. +; phi^2 + 1/phi^2 = 3 | TRINITY + +module hslm_tokenizer; + +pub const VOCAB_SIZE : u16 = 729; +pub const PAD_TOKEN : u16 = 0; +pub const BOS_TOKEN : u16 = 1; +pub const EOS_TOKEN : u16 = 2; +pub const UNK_TOKEN : u16 = 3; +pub const SPECIAL_COUNT : u16 = 4; +pub const BYTE_OFFSET : u16 = 4; +pub const BYTE_COUNT : u16 = 256; +pub const MERGE_OFFSET : u16 = 260; +pub const MERGE_COUNT : usize = 469; +pub const WORK_CAP : usize = 8192; + +// MERGE_LEFT[i], MERGE_RIGHT[i]: the two tokens that merge i joins. MERGE_LEN[i]: how many bytes +// the new token stands for. MERGE_START[i]: where those bytes begin in MERGE_BYTES. + +pub const MERGE_LEFT : [469]u16 = [ + 105, 104, 120, 36, 50, 120, 125, 119, 114, 120, 105, 105, 262, 48, 123, 109, + 105, 108, 269, 115, 263, 264, 108, 263, 115, 119, 101, 36, 108, 115, 109, 107, + 50, 112, 274, 101, 115, 101, 109, 270, 109, 118, 104, 109, 116, 109, 271, 109, + 101, 262, 123, 118, 119, 112, 101, 101, 264, 288, 264, 114, 36, 316, 101, 281, + 270, 275, 121, 310, 83, 115, 116, 122, 105, 281, 103, 112, 102, 293, 38, 112, + 102, 318, 273, 104, 121, 109, 125, 123, 115, 285, 284, 106, 275, 113, 119, 112, + 112, 102, 282, 276, 290, 106, 107, 121, 119, 102, 105, 295, 101, 331, 103, 115, + 106, 274, 274, 282, 276, 103, 112, 336, 361, 332, 114, 349, 128, 351, 385, 302, + 276, 128, 64, 390, 298, 105, 309, 80, 122, 118, 292, 120, 319, 304, 264, 328, + 107, 111, 373, 392, 120, 43, 112, 105, 108, 101, 262, 103, 114, 403, 275, 279, + 109, 119, 416, 118, 123, 107, 125, 88, 262, 262, 278, 271, 38, 276, 108, 119, + 123, 106, 102, 289, 304, 285, 346, 48, 121, 374, 119, 288, 102, 328, 106, 123, + 271, 77, 447, 279, 313, 320, 37, 125, 363, 327, 449, 462, 429, 402, 88, 112, + 119, 120, 106, 116, 103, 88, 118, 105, 463, 119, 119, 102, 417, 273, 269, 371, + 395, 120, 37, 119, 111, 308, 334, 320, 105, 116, 305, 113, 48, 113, 294, 438, + 119, 103, 103, 105, 107, 398, 113, 123, 106, 264, 260, 441, 309, 113, 120, 313, + 107, 289, 265, 382, 109, 81, 43, 380, 521, 104, 125, 370, 109, 50, 372, 103, + 50, 482, 282, 401, 265, 121, 375, 279, 281, 264, 106, 292, 121, 105, 312, 504, + 500, 102, 103, 358, 106, 113, 119, 335, 274, 456, 115, 275, 119, 505, 525, 88, + 101, 285, 358, 357, 264, 114, 478, 106, 289, 262, 437, 279, 88, 108, 93, 335, + 282, 262, 123, 115, 301, 302, 263, 269, 116, 119, 104, 112, 123, 105, 107, 118, + 111, 103, 289, 327, 365, 274, 119, 574, 262, 119, 87, 67, 353, 119, 104, 104, + 91, 113, 336, 371, 122, 115, 515, 116, 276, 417, 113, 262, 109, 264, 546, 354, + 292, 113, 275, 294, 121, 269, 509, 120, 493, 279, 116, 121, 524, 398, 353, 103, + 410, 120, 273, 555, 81, 450, 286, 570, 327, 105, 309, 374, 263, 88, 357, 302, + 106, 285, 115, 103, 345, 380, 107, 116, 107, 470, 292, 111, 269, 339, 121, 112, + 552, 285, 454, 285, 274, 269, 116, 117, 116, 106, 516, 292, 105, 112, 119, 121, + 115, 335, 425, 310, 567, 288, 119, 508, 438, 122, 280, 478, 276, 114, 485, 103, + 104, 264, 262, 101, 120, 395, 108, 116, 365, 118, 80, 87, 120, 398, 467, 113, + 118, 48, 313, 358, 104 +]; + +pub const MERGE_RIGHT : [469]u16 = [ + 36, 36, 108, 101, 36, 36, 36, 36, 261, 115, 118, 261, 260, 36, 101, 114, + 114, 260, 36, 121, 268, 88, 101, 36, 113, 101, 118, 272, 105, 114, 113, 36, + 14, 112, 267, 114, 118, 125, 120, 36, 104, 105, 115, 267, 112, 118, 278, 112, + 113, 105, 109, 109, 120, 115, 268, 36, 76, 266, 87, 115, 108, 260, 265, 317, + 260, 291, 114, 262, 114, 36, 116, 270, 124, 277, 111, 109, 109, 36, 36, 105, + 105, 277, 38, 297, 265, 265, 279, 36, 111, 300, 36, 120, 107, 101, 105, 260, + 261, 115, 330, 104, 36, 311, 108, 116, 108, 101, 104, 36, 266, 266, 108, 106, + 296, 119, 114, 261, 36, 260, 36, 291, 359, 120, 105, 342, 66, 381, 384, 386, + 387, 388, 389, 14, 120, 273, 266, 307, 260, 115, 391, 306, 265, 297, 77, 260, + 105, 260, 283, 355, 290, 267, 407, 119, 303, 120, 322, 286, 308, 343, 287, 356, + 261, 277, 271, 101, 324, 115, 280, 277, 270, 324, 272, 314, 14, 265, 299, 329, + 108, 105, 305, 283, 368, 123, 36, 280, 118, 399, 260, 112, 344, 377, 109, 433, + 272, 36, 116, 268, 348, 303, 36, 263, 439, 36, 460, 408, 406, 265, 360, 101, + 111, 101, 326, 115, 101, 290, 260, 101, 393, 280, 284, 260, 273, 448, 404, 36, + 266, 301, 338, 265, 36, 260, 36, 299, 120, 286, 36, 36, 283, 266, 369, 261, + 113, 115, 489, 103, 305, 476, 350, 114, 121, 70, 314, 283, 494, 284, 120, 122, + 101, 287, 272, 347, 103, 101, 265, 119, 124, 420, 273, 36, 119, 432, 36, 419, + 338, 428, 396, 271, 278, 113, 315, 362, 360, 69, 101, 38, 260, 267, 286, 378, + 307, 121, 367, 125, 455, 296, 300, 111, 125, 271, 123, 36, 269, 464, 400, 284, + 112, 420, 266, 266, 38, 104, 105, 118, 36, 299, 112, 120, 350, 115, 279, 405, + 265, 325, 419, 261, 101, 291, 114, 125, 36, 116, 368, 111, 296, 112, 118, 36, + 105, 322, 260, 287, 293, 120, 353, 265, 352, 282, 308, 338, 405, 415, 104, 260, + 108, 105, 107, 287, 271, 273, 306, 109, 120, 496, 121, 539, 362, 480, 399, 260, + 427, 295, 283, 435, 107, 115, 344, 118, 111, 265, 101, 362, 36, 14, 611, 284, + 547, 311, 272, 306, 109, 268, 114, 581, 457, 293, 113, 536, 474, 317, 124, 507, + 115, 125, 102, 413, 294, 267, 329, 344, 301, 36, 480, 319, 348, 650, 103, 266, + 315, 104, 36, 347, 591, 266, 301, 121, 121, 36, 275, 69, 331, 289, 326, 446, + 116, 616, 583, 337, 422, 459, 444, 378, 104, 299, 272, 260, 273, 522, 105, 105, + 270, 14, 36, 293, 271, 125, 270, 118, 490, 271, 674, 471, 109, 621, 639, 115, + 295, 338, 123, 426, 109 +]; + +pub const MERGE_LEN : [469]u8 = [ + 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 2, 3, 4, 2, 2, 2, + 2, 3, 3, 2, 5, 3, 2, 3, 2, 2, 2, 5, 2, 2, 2, 2, + 2, 2, 4, 2, 2, 2, 2, 3, 2, 2, 2, 3, 2, 2, 6, 2, + 2, 3, 2, 2, 2, 2, 4, 2, 3, 4, 3, 2, 2, 5, 3, 7, + 4, 4, 2, 4, 2, 2, 2, 3, 2, 6, 2, 2, 2, 3, 2, 2, + 2, 6, 3, 3, 3, 3, 3, 2, 2, 4, 3, 2, 3, 2, 2, 3, + 3, 2, 4, 3, 3, 3, 2, 2, 2, 2, 2, 3, 3, 5, 2, 2, + 3, 3, 3, 4, 3, 3, 2, 4, 6, 3, 2, 7, 2, 5, 7, 9, + 11, 12, 13, 14, 3, 3, 5, 3, 3, 2, 16, 7, 4, 4, 3, 4, + 2, 3, 6, 6, 3, 3, 7, 2, 4, 2, 5, 3, 3, 7, 7, 5, + 3, 4, 6, 2, 5, 2, 6, 4, 4, 6, 7, 7, 2, 4, 4, 3, + 2, 2, 3, 5, 5, 3, 4, 6, 2, 10, 3, 3, 4, 5, 2, 5, + 7, 2, 4, 5, 4, 5, 2, 3, 7, 5, 12, 15, 12, 5, 4, 2, + 2, 2, 3, 2, 2, 3, 3, 2, 18, 6, 3, 3, 9, 6, 4, 3, + 5, 3, 3, 3, 2, 4, 3, 5, 2, 3, 3, 2, 4, 3, 9, 5, + 2, 2, 5, 2, 3, 34, 4, 2, 2, 3, 6, 6, 6, 3, 2, 3, + 2, 7, 6, 4, 2, 2, 3, 7, 3, 4, 3, 3, 2, 3, 4, 6, + 3, 8, 5, 7, 5, 2, 6, 4, 6, 3, 2, 3, 3, 3, 4, 5, + 4, 2, 4, 5, 6, 3, 3, 3, 3, 7, 2, 3, 3, 46, 8, 3, + 2, 5, 6, 4, 3, 2, 4, 2, 3, 5, 3, 3, 4, 2, 3, 5, + 4, 6, 6, 3, 3, 4, 3, 3, 2, 2, 4, 2, 3, 2, 2, 2, + 2, 4, 4, 9, 4, 3, 3, 5, 5, 3, 3, 3, 5, 4, 2, 3, + 2, 2, 3, 7, 4, 3, 9, 2, 3, 11, 2, 6, 3, 11, 11, 4, + 6, 3, 5, 7, 2, 3, 6, 2, 4, 4, 2, 3, 4, 17, 5, 3, + 12, 3, 6, 9, 2, 5, 3, 10, 9, 3, 4, 8, 5, 5, 3, 4, + 2, 3, 2, 3, 7, 8, 3, 4, 3, 4, 11, 3, 4, 5, 2, 3, + 8, 3, 5, 4, 4, 4, 3, 2, 2, 2, 4, 3, 4, 3, 3, 4, + 2, 6, 5, 5, 10, 5, 3, 4, 4, 4, 9, 5, 4, 4, 4, 2, + 3, 3, 3, 3, 4, 4, 3, 2, 5, 4, 3, 3, 2, 27, 5, 2, + 3, 3, 3, 10, 2 +]; + +pub const MERGE_START : [469]u16 = [ + 0, 2, 4, 6, 8, 10, 12, 14, 16, 19, 21, 23, 26, 30, 32, 34, + 36, 38, 41, 44, 46, 51, 54, 56, 59, 61, 63, 65, 70, 72, 74, 76, + 78, 80, 82, 86, 88, 90, 92, 94, 97, 99, 101, 103, 106, 108, 110, 116, + 118, 120, 123, 125, 127, 129, 131, 135, 137, 140, 144, 147, 149, 151, 156, 159, + 166, 170, 174, 176, 180, 182, 184, 186, 189, 191, 197, 199, 201, 203, 206, 208, + 210, 212, 218, 221, 224, 227, 230, 233, 235, 237, 241, 244, 246, 249, 251, 253, + 256, 259, 261, 265, 268, 271, 274, 276, 278, 280, 282, 284, 287, 290, 295, 297, + 299, 302, 305, 308, 312, 315, 318, 320, 324, 330, 333, 335, 342, 344, 349, 356, + 365, 376, 388, 401, 415, 418, 421, 426, 429, 432, 434, 450, 457, 461, 465, 468, + 472, 474, 477, 483, 489, 492, 495, 502, 504, 508, 510, 515, 518, 521, 528, 535, + 540, 543, 547, 553, 555, 560, 562, 568, 572, 576, 582, 589, 596, 598, 602, 606, + 609, 611, 613, 616, 621, 626, 629, 633, 639, 641, 651, 654, 657, 661, 666, 668, + 673, 680, 682, 686, 691, 695, 700, 702, 705, 712, 717, 729, 744, 756, 761, 765, + 767, 769, 771, 774, 776, 778, 781, 784, 786, 804, 810, 813, 816, 825, 831, 835, + 838, 843, 846, 849, 852, 854, 858, 861, 866, 868, 871, 874, 876, 880, 883, 892, + 897, 899, 901, 906, 908, 911, 945, 949, 951, 953, 956, 962, 968, 974, 977, 979, + 982, 984, 991, 997, 1001, 1003, 1005, 1008, 1015, 1018, 1022, 1025, 1028, 1030, 1033, 1037, + 1043, 1046, 1054, 1059, 1066, 1071, 1073, 1079, 1083, 1089, 1092, 1094, 1097, 1100, 1103, 1107, + 1112, 1116, 1118, 1122, 1127, 1133, 1136, 1139, 1142, 1145, 1152, 1154, 1157, 1160, 1206, 1214, + 1217, 1219, 1224, 1230, 1234, 1237, 1239, 1243, 1245, 1248, 1253, 1256, 1259, 1263, 1265, 1268, + 1273, 1277, 1283, 1289, 1292, 1295, 1299, 1302, 1305, 1307, 1309, 1313, 1315, 1318, 1320, 1322, + 1324, 1326, 1330, 1334, 1343, 1347, 1350, 1353, 1358, 1363, 1366, 1369, 1372, 1377, 1381, 1383, + 1386, 1388, 1390, 1393, 1400, 1404, 1407, 1416, 1418, 1421, 1432, 1434, 1440, 1443, 1454, 1465, + 1469, 1475, 1478, 1483, 1490, 1492, 1495, 1501, 1503, 1507, 1511, 1513, 1516, 1520, 1537, 1542, + 1545, 1557, 1560, 1566, 1575, 1577, 1582, 1585, 1595, 1604, 1607, 1611, 1619, 1624, 1629, 1632, + 1636, 1638, 1641, 1643, 1646, 1653, 1661, 1664, 1668, 1671, 1675, 1686, 1689, 1693, 1698, 1700, + 1703, 1711, 1714, 1719, 1723, 1727, 1731, 1734, 1736, 1738, 1740, 1744, 1747, 1751, 1754, 1757, + 1761, 1763, 1769, 1774, 1779, 1789, 1794, 1797, 1801, 1805, 1809, 1818, 1823, 1827, 1831, 1835, + 1837, 1840, 1843, 1846, 1849, 1853, 1857, 1860, 1862, 1867, 1871, 1874, 1877, 1879, 1906, 1911, + 1913, 1916, 1919, 1922, 1932 +]; + +pub const MERGE_BYTES : [1934]u8 = [ + 101, 32, 100, 32, 116, 104, 32, 97, 46, 32, 116, 32, 121, 32, 115, 32, + 110, 100, 32, 116, 111, 101, 114, 101, 100, 32, 116, 104, 101, 32, 44, 32, + 119, 97, 105, 110, 101, 110, 104, 101, 32, 116, 111, 32, 111, 117, 32, 97, + 110, 100, 32, 46, 32, 84, 104, 97, 32, 97, 32, 111, 109, 115, 97, 97, + 114, 32, 116, 104, 101, 32, 104, 101, 111, 110, 105, 109, 103, 32, 46, 10, + 108, 108, 119, 97, 115, 32, 97, 110, 111, 114, 97, 121, 105, 116, 101, 114, + 32, 105, 100, 114, 101, 100, 111, 105, 115, 32, 112, 108, 105, 114, 101, 100, + 32, 116, 111, 32, 105, 108, 97, 109, 116, 104, 101, 119, 105, 114, 105, 115, + 116, 108, 111, 97, 110, 100, 32, 97, 32, 46, 32, 72, 104, 101, 121, 32, + 46, 32, 83, 110, 111, 32, 104, 46, 32, 72, 101, 32, 97, 116, 32, 46, + 32, 84, 104, 101, 121, 32, 101, 114, 101, 32, 105, 110, 103, 32, 117, 110, + 119, 105, 116, 104, 79, 110, 111, 32, 112, 112, 118, 101, 114, 101, 120, 46, + 32, 84, 104, 101, 32, 99, 107, 108, 105, 98, 105, 108, 108, 32, 34, 32, + 108, 101, 98, 101, 46, 32, 83, 104, 101, 32, 44, 32, 34, 100, 97, 121, + 117, 116, 32, 105, 116, 32, 121, 111, 117, 119, 32, 111, 107, 115, 97, 105, + 100, 111, 109, 32, 102, 116, 105, 110, 103, 109, 97, 115, 101, 108, 101, 32, + 108, 100, 32, 98, 111, 104, 97, 112, 112, 101, 110, 100, 105, 109, 32, 102, + 114, 105, 103, 104, 117, 112, 115, 104, 98, 97, 101, 100, 97, 110, 32, 97, + 121, 32, 118, 101, 114, 121, 32, 99, 104, 111, 102, 102, 111, 114, 119, 97, + 115, 119, 97, 110, 104, 97, 100, 32, 101, 110, 32, 99, 101, 32, 108, 32, + 98, 105, 103, 32, 102, 114, 105, 101, 110, 100, 101, 120, 116, 110, 101, 115, + 97, 105, 100, 44, 32, 34, 124, 62, 102, 116, 101, 120, 116, 102, 116, 101, + 120, 116, 124, 62, 100, 111, 102, 116, 101, 120, 116, 124, 62, 101, 110, 100, + 111, 102, 116, 101, 120, 116, 124, 62, 124, 101, 110, 100, 111, 102, 116, 101, + 120, 116, 124, 62, 60, 124, 101, 110, 100, 111, 102, 116, 101, 120, 116, 124, + 62, 60, 124, 101, 110, 100, 111, 102, 116, 101, 120, 116, 124, 62, 10, 105, + 116, 116, 101, 44, 32, 116, 104, 101, 121, 32, 76, 105, 108, 118, 101, 32, + 114, 111, 46, 10, 60, 124, 101, 110, 100, 111, 102, 116, 101, 120, 116, 124, + 62, 10, 116, 101, 100, 32, 116, 111, 32, 110, 111, 116, 32, 112, 108, 97, + 121, 46, 32, 73, 79, 110, 101, 32, 103, 101, 107, 101, 32, 119, 97, 115, + 32, 97, 32, 105, 116, 116, 108, 101, 32, 116, 105, 109, 39, 115, 32, 108, + 105, 116, 116, 108, 101, 32, 101, 115, 104, 105, 115, 32, 97, 116, 116, 104, + 97, 116, 32, 99, 97, 114, 110, 97, 109, 79, 110, 101, 32, 100, 97, 121, + 105, 110, 32, 116, 104, 101, 32, 111, 117, 108, 100, 32, 105, 100, 32, 115, + 104, 101, 32, 110, 97, 109, 101, 100, 32, 114, 97, 119, 101, 114, 101, 32, + 103, 111, 121, 32, 97, 110, 100, 32, 84, 104, 101, 32, 116, 104, 101, 114, + 116, 104, 101, 114, 101, 32, 116, 111, 32, 116, 104, 101, 32, 101, 100, 32, + 97, 110, 100, 32, 34, 10, 101, 110, 116, 32, 104, 101, 114, 32, 115, 111, + 32, 119, 104, 102, 101, 98, 105, 114, 111, 110, 32, 97, 32, 112, 108, 97, + 121, 32, 115, 97, 119, 121, 111, 117, 32, 44, 32, 97, 110, 100, 32, 117, + 114, 119, 97, 110, 116, 101, 100, 32, 116, 111, 32, 115, 101, 32, 104, 101, + 108, 98, 117, 116, 32, 79, 110, 99, 101, 32, 102, 105, 119, 101, 110, 116, + 32, 101, 100, 32, 116, 104, 101, 32, 73, 32, 104, 101, 108, 112, 111, 117, + 110, 100, 32, 108, 111, 111, 107, 32, 104, 105, 115, 32, 33, 32, 121, 32, + 97, 117, 112, 111, 110, 32, 97, 32, 119, 105, 116, 104, 32, 79, 110, 99, + 101, 32, 117, 112, 111, 110, 32, 97, 32, 79, 110, 99, 101, 32, 117, 112, + 111, 110, 32, 97, 32, 116, 105, 109, 116, 104, 101, 114, 101, 32, 119, 97, + 115, 32, 97, 32, 46, 32, 73, 116, 32, 84, 105, 109, 32, 108, 97, 115, + 107, 116, 97, 102, 117, 110, 112, 111, 99, 97, 84, 105, 109, 114, 101, 32, + 101, 97, 79, 110, 99, 101, 32, 117, 112, 111, 110, 32, 97, 32, 116, 105, + 109, 101, 44, 32, 115, 32, 97, 110, 100, 32, 115, 111, 109, 98, 101, 32, + 79, 110, 101, 32, 100, 97, 121, 44, 32, 44, 32, 98, 117, 116, 32, 116, + 111, 103, 101, 111, 102, 32, 76, 105, 108, 121, 32, 116, 114, 101, 33, 34, + 32, 115, 116, 32, 107, 32, 97, 109, 101, 32, 99, 107, 32, 32, 104, 101, + 114, 32, 101, 116, 112, 97, 114, 105, 114, 32, 109, 32, 44, 32, 97, 32, + 109, 121, 32, 119, 97, 115, 32, 118, 101, 114, 121, 32, 98, 105, 114, 100, + 32, 115, 109, 99, 111, 99, 97, 109, 101, 32, 101, 99, 103, 105, 114, 46, + 10, 60, 124, 101, 110, 100, 111, 102, 116, 101, 120, 116, 124, 62, 10, 79, + 110, 99, 101, 32, 117, 112, 111, 110, 32, 97, 32, 116, 105, 109, 101, 44, + 32, 109, 111, 109, 32, 119, 110, 102, 117, 46, 32, 66, 101, 32, 97, 110, + 100, 32, 115, 97, 119, 32, 97, 32, 116, 104, 101, 105, 114, 32, 109, 111, + 109, 116, 116, 108, 111, 118, 103, 97, 111, 110, 32, 116, 104, 101, 32, 116, + 32, 116, 104, 101, 32, 110, 101, 119, 32, 105, 99, 77, 97, 39, 116, 32, + 102, 114, 105, 101, 110, 100, 115, 77, 97, 120, 100, 105, 100, 32, 121, 44, + 32, 99, 104, 32, 105, 115, 46, 34, 10, 102, 111, 114, 32, 99, 111, 117, + 108, 100, 32, 46, 34, 32, 116, 111, 103, 101, 116, 104, 101, 114, 104, 97, + 118, 101, 32, 112, 108, 97, 121, 101, 100, 32, 116, 32, 116, 111, 32, 117, + 109, 104, 97, 100, 32, 97, 32, 111, 117, 103, 104, 46, 32, 84, 105, 109, + 32, 46, 32, 65, 102, 97, 46, 10, 34, 117, 101, 32, 101, 115, 32, 115, + 116, 97, 114, 103, 105, 114, 108, 32, 115, 109, 105, 108, 98, 117, 99, 97, + 110, 32, 104, 97, 112, 112, 121, 102, 111, 117, 110, 100, 32, 109, 111, 114, + 115, 105, 100, 108, 105, 107, 119, 97, 121, 108, 111, 111, 107, 101, 100, 32, + 111, 119, 105, 110, 32, 115, 116, 111, 46, 10, 60, 124, 101, 110, 100, 111, + 102, 116, 101, 120, 116, 124, 62, 10, 79, 110, 99, 101, 32, 117, 112, 111, + 110, 32, 97, 32, 116, 105, 109, 101, 44, 32, 116, 104, 101, 114, 101, 32, + 119, 97, 115, 32, 97, 32, 100, 105, 100, 32, 110, 111, 116, 32, 84, 111, + 109, 97, 108, 115, 97, 105, 100, 32, 104, 97, 112, 112, 121, 32, 98, 111, + 121, 32, 46, 32, 34, 110, 100, 115, 111, 109, 101, 102, 114, 111, 110, 32, + 116, 104, 101, 114, 32, 102, 101, 108, 111, 117, 116, 84, 111, 109, 32, 104, + 111, 89, 111, 117, 108, 105, 107, 101, 32, 104, 97, 116, 32, 116, 104, 105, + 110, 103, 32, 119, 111, 117, 108, 100, 32, 111, 100, 32, 114, 101, 97, 100, + 111, 103, 32, 32, 97, 110, 116, 111, 121, 112, 32, 115, 112, 100, 97, 121, + 32, 108, 107, 119, 111, 114, 101, 108, 103, 114, 114, 32, 107, 101, 99, 97, + 116, 32, 111, 110, 101, 32, 119, 105, 116, 104, 32, 116, 104, 101, 32, 98, + 97, 108, 108, 119, 97, 116, 115, 109, 97, 102, 101, 108, 116, 32, 116, 104, + 105, 110, 103, 115, 104, 97, 83, 97, 109, 63, 34, 32, 109, 97, 107, 101, + 32, 115, 99, 97, 114, 100, 100, 100, 101, 32, 87, 104, 109, 101, 98, 105, + 103, 111, 102, 32, 116, 104, 101, 32, 118, 101, 100, 32, 111, 44, 32, 108, + 111, 118, 101, 100, 32, 116, 111, 32, 112, 105, 101, 110, 116, 79, 110, 101, + 32, 100, 97, 121, 44, 32, 97, 32, 109, 117, 116, 104, 111, 117, 103, 104, + 105, 103, 104, 46, 32, 79, 110, 101, 32, 100, 97, 121, 44, 32, 115, 116, + 97, 114, 116, 101, 100, 32, 116, 111, 32, 115, 101, 101, 32, 46, 10, 84, + 104, 101, 32, 109, 97, 110, 105, 110, 32, 97, 32, 119, 97, 115, 32, 115, + 111, 32, 117, 103, 116, 111, 111, 46, 32, 66, 117, 116, 32, 116, 114, 112, + 97, 114, 107, 111, 117, 116, 32, 112, 97, 117, 103, 104, 77, 97, 120, 32, + 46, 10, 60, 124, 101, 110, 100, 111, 102, 116, 101, 120, 116, 124, 62, 10, + 10, 109, 97, 100, 101, 32, 99, 111, 109, 108, 105, 116, 116, 108, 101, 32, + 103, 105, 114, 108, 32, 116, 114, 105, 44, 32, 116, 104, 101, 32, 108, 105, + 107, 101, 100, 32, 116, 111, 32, 77, 105, 102, 105, 110, 100, 32, 97, 114, + 110, 115, 111, 109, 101, 116, 104, 105, 110, 103, 32, 119, 105, 116, 104, 32, + 104, 105, 115, 32, 101, 108, 108, 116, 104, 101, 109, 119, 97, 110, 116, 32, + 116, 111, 32, 32, 97, 114, 101, 32, 84, 104, 101, 121, 32, 98, 111, 120, + 100, 111, 119, 110, 102, 111, 115, 97, 121, 111, 98, 99, 97, 116, 105, 116, + 32, 119, 97, 115, 32, 102, 114, 105, 101, 110, 100, 115, 32, 103, 111, 32, + 112, 117, 116, 32, 103, 114, 101, 102, 117, 110, 32, 46, 10, 79, 110, 101, + 32, 100, 97, 121, 44, 32, 107, 110, 111, 116, 111, 111, 107, 108, 101, 97, + 114, 110, 117, 99, 108, 121, 32, 102, 111, 117, 110, 100, 32, 97, 32, 115, + 97, 100, 104, 101, 108, 112, 32, 115, 97, 119, 32, 119, 97, 108, 107, 116, + 111, 121, 32, 112, 114, 101, 113, 117, 112, 117, 102, 32, 103, 97, 105, 110, + 46, 10, 65, 101, 118, 101, 114, 108, 111, 110, 115, 117, 110, 117, 115, 101, + 32, 111, 112, 108, 105, 118, 101, 100, 32, 103, 111, 111, 100, 32, 119, 105, + 108, 108, 32, 98, 111, 121, 32, 110, 97, 109, 101, 100, 32, 104, 101, 121, + 32, 97, 115, 117, 114, 102, 117, 108, 32, 98, 105, 114, 100, 118, 101, 114, + 32, 32, 97, 110, 100, 32, 116, 104, 101, 32, 115, 111, 109, 101, 32, 101, + 110, 44, 32, 110, 39, 116, 32, 116, 114, 101, 101, 99, 101, 100, 101, 114, + 46, 32, 10, 116, 104, 32, 97, 108, 108, 116, 101, 100, 32, 76, 105, 108, + 121, 104, 101, 114, 112, 114, 98, 97, 99, 107, 32, 114, 101, 100, 32, 76, + 117, 99, 83, 112, 111, 116, 105, 46, 10, 60, 124, 101, 110, 100, 111, 102, + 116, 101, 120, 116, 124, 62, 10, 79, 110, 101, 32, 100, 97, 121, 44, 32, + 97, 32, 108, 97, 117, 103, 104, 109, 111, 114, 97, 110, 44, 34, 32, 108, + 111, 119, 104, 97, 112, 112, 121, 32, 97, 110, 100, 32, 100, 105 +]; + + +/// Steps 1 and 2 of Tokenizer.encode: the first n bytes of text become byte tokens in work, then +/// each of the 469 rules is applied to the whole sequence in priority order, in place. Returns the +/// merged length. The caller guarantees n <= work.len and n <= text.len. +pub fn encode_window(text: []const u8, n: usize, work: []u16) -> usize { + var i : usize = 0; + while (i < n) { + const b : u16 = @intCast(text[i]); + work[i] = BYTE_OFFSET + b; + i = i + 1; + } + var seq_len : usize = n; + var rule : usize = 0; + while (rule < MERGE_COUNT) { + if (seq_len < 2) { + return seq_len; + } + const left : u16 = MERGE_LEFT[rule]; + const right : u16 = MERGE_RIGHT[rule]; + const r16 : u16 = @intCast(rule); + const new_token : u16 = MERGE_OFFSET + r16; + var read_pos : usize = 0; + var write_pos : usize = 0; + while (read_pos < seq_len) { + if (read_pos + 1 < seq_len and work[read_pos] == left and work[read_pos + 1] == right) { + work[write_pos] = new_token; + write_pos = write_pos + 1; + read_pos = read_pos + 2; + } else { + work[write_pos] = work[read_pos]; + write_pos = write_pos + 1; + read_pos = read_pos + 1; + } + } + seq_len = write_pos; + rule = rule + 1; + } + return seq_len; +} + +/// Tokenizer.encode: BOS, the merged tokens of text, EOS, into output. Returns the count written. +/// A short output truncates exactly as the Zig does (see the header). +pub fn encode(text: []const u8, output: []u16) -> usize { + var work : [8192]u16 = undefined; + const max_out : usize = output.len; + var out_idx : usize = 0; + if (out_idx < max_out) { + output[out_idx] = BOS_TOKEN; + out_idx = out_idx + 1; + } + if (text.len == 0) { + if (out_idx < max_out) { + output[out_idx] = EOS_TOKEN; + out_idx = out_idx + 1; + } + return out_idx; + } + var room : usize = 0; + if (max_out > 2) { + room = max_out - 2; + } + var work_len : usize = text.len; + if (room < work_len) { + work_len = room; + } + if (WORK_CAP < work_len) { + work_len = WORK_CAP; + } + const seq_len : usize = encode_window(text, work_len, &work); + var avail : usize = 0; + if (max_out > out_idx + 1) { + avail = max_out - out_idx - 1; + } + var copy_len : usize = seq_len; + if (avail < copy_len) { + copy_len = avail; + } + var c : usize = 0; + while (c < copy_len) { + output[out_idx + c] = work[c]; + c = c + 1; + } + out_idx = out_idx + copy_len; + if (out_idx < max_out) { + output[out_idx] = EOS_TOKEN; + out_idx = out_idx + 1; + } + return out_idx; +} + +/// Tokenizer.decode: token ids back to bytes in output. Returns the count written. +pub fn decode(tokens: []const u16, output: []u8) -> usize { + var out_idx : usize = 0; + var t : usize = 0; + while (t < tokens.len) { + const token : u16 = tokens[t]; + t = t + 1; + // PAD, BOS and EOS (0, 1, 2) match no branch below and write nothing, as in the Zig. + if (token == UNK_TOKEN) { + if (out_idx < output.len) { + output[out_idx] = 63; + out_idx = out_idx + 1; + } + } else if (token >= MERGE_OFFSET and token < VOCAB_SIZE) { + const idx : usize = @intCast(token - MERGE_OFFSET); + const start : usize = @intCast(MERGE_START[idx]); + const len : usize = @intCast(MERGE_LEN[idx]); + const room : usize = output.len - out_idx; + var copy_len : usize = len; + if (room < copy_len) { + copy_len = room; + } + var k : usize = 0; + while (k < copy_len) { + output[out_idx + k] = MERGE_BYTES[start + k]; + k = k + 1; + } + out_idx = out_idx + copy_len; + } else if (token >= BYTE_OFFSET and token < MERGE_OFFSET) { + if (out_idx < output.len) { + output[out_idx] = @intCast(token - BYTE_OFFSET); + out_idx = out_idx + 1; + } + } + } + return out_idx; +} + +// ---- helpers for the tests ------------------------------------------------------------------- + +fn token_len(tok: usize) -> usize { + if (tok < 260) { + return 1; + } + return @intCast(MERGE_LEN[tok - 260]); +} + +fn token_byte(tok: usize, j: usize) -> u8 { + if (tok < 260) { + return @intCast(tok - 4); + } + return MERGE_BYTES[@as(usize, @intCast(MERGE_START[tok - 260])) + j]; +} + +/// encode(text[0..n]) must be exactly want[0..k], and decode of it must give text[0..n] back. +fn encodes_to(text: []const u8, n: usize, want: []const u16, k: usize) -> bool { + var out : [512]u16 = undefined; + const got : usize = encode(text[0 .. n], &out); + if (got != k) { + return false; + } + var i : usize = 0; + while (i < k) { + if (out[i] != want[i]) { + return false; + } + i = i + 1; + } + var back : [512]u8 = undefined; + const m : usize = decode(out[0 .. got], &back); + if (m != n) { + return false; + } + var j : usize = 0; + while (j < n) { + if (back[j] != text[j]) { + return false; + } + j = j + 1; + } + return true; +} + +// ---- tests: the ids the Zig tokenizer returns (BOS ... EOS) ---------------------------------- + +test zig_empty_text { + const t : [1]u8 = [0]; + const want : [2]u16 = [1, 2]; + assert encodes_to(&t, 0, &want, 2); +} + +test zig_one_byte { + const t : [1]u8 = [97]; + const want : [3]u16 = [1, 101, 2]; + assert encodes_to(&t, 1, &want, 3); +} + +test zig_once_upon_a_time { + const t : [16]u8 = [79, 110, 99, 101, 32, 117, 112, 111, 110, 32, 97, 32, 116, 105, 109, 101]; + const want : [4]u16 = [1, 463, 105, 2]; + assert encodes_to(&t, 16, &want, 4); +} + +test zig_text_with_a_newline { + const t : [43]u8 = [79, 110, 99, 101, 32, 117, 112, 111, 110, 32, 97, 32, 116, 105, 109, 101, 44, 10, 116, 104, 101, 114, 101, 32, 119, 97, 115, 32, 97, 32, 108, 105, 116, 116, 108, 101, 32, 103, 105, 114, 108, 46, 10]; + const want : [11]u16 = [1, 463, 105, 48, 14, 464, 410, 504, 112, 292, 2]; + assert encodes_to(&t, 43, &want, 11); +} + +test zig_non_ascii_utf8 { + const t : [23]u8 = [99, 97, 102, 195, 169, 32, 226, 128, 156, 113, 117, 111, 116, 101, 100, 226, 128, 157, 32, 240, 159, 152, 128]; + const want : [22]u16 = [1, 472, 106, 199, 173, 36, 230, 132, 160, 683, 115, 120, 366, 230, 132, 161, 36, 244, 163, 156, 132, 2]; + assert encodes_to(&t, 23, &want, 22); +} + +test zig_quick_brown_fox { + const t : [19]u8 = [84, 104, 101, 32, 113, 117, 105, 99, 107, 32, 98, 114, 111, 119, 110, 32, 102, 111, 120]; + const want : [12]u16 = [1, 427, 683, 109, 490, 102, 397, 507, 36, 660, 124, 2]; + assert encodes_to(&t, 19, &want, 12); +} + +test zig_hello_world { + const t : [13]u8 = [72, 101, 108, 108, 111, 44, 32, 87, 111, 114, 108, 100, 33]; + const want : [10]u16 = [1, 76, 653, 617, 91, 296, 112, 104, 37, 2]; + assert encodes_to(&t, 13, &want, 10); +} + +test zig_repeated_word { + const t : [11]u8 = [116, 104, 101, 32, 116, 104, 101, 32, 116, 104, 101]; + const want : [5]u16 = [1, 272, 272, 309, 2]; + assert encodes_to(&t, 11, &want, 5); +} + +test zig_digits { + const t : [10]u8 = [49, 50, 51, 52, 53, 54, 55, 56, 57, 48]; + const want : [12]u16 = [1, 53, 54, 55, 56, 57, 58, 59, 60, 61, 52, 2]; + assert encodes_to(&t, 10, &want, 12); +} + +test zig_endoftext_marker { + const t : [13]u8 = [60, 124, 101, 110, 100, 111, 102, 116, 101, 120, 116, 124, 62]; + const want : [3]u16 = [1, 390, 2]; + assert encodes_to(&t, 13, &want, 3); +} + +test zig_lily_sentence { + const t : [85]u8 = [79, 110, 99, 101, 32, 117, 112, 111, 110, 32, 97, 32, 116, 105, 109, 101, 32, 116, 104, 101, 114, 101, 32, 119, 97, 115, 32, 97, 32, 108, 105, 116, 116, 108, 101, 32, 103, 105, 114, 108, 32, 110, 97, 109, 101, 100, 32, 76, 105, 108, 121, 46, 32, 83, 104, 101, 32, 108, 111, 118, 101, 100, 32, 116, 111, 32, 112, 108, 97, 121, 32, 105, 110, 32, 116, 104, 101, 32, 103, 97, 114, 100, 101, 110, 46]; + const want : [17]u16 = [1, 463, 260, 464, 644, 422, 713, 341, 618, 440, 418, 107, 286, 104, 276, 50, 2]; + assert encodes_to(&t, 85, &want, 17); +} + +test zig_control_and_high_bytes { + const t : [4]u8 = [0, 1, 255, 254]; + const want : [6]u16 = [1, 4, 5, 259, 258, 2]; + assert encodes_to(&t, 4, &want, 6); +} + +test zig_overlapping_pairs { + const t : [10]u8 = [97, 97, 97, 97, 32, 97, 97, 97, 97, 97]; + const want : [11]u16 = [1, 101, 101, 101, 101, 263, 101, 101, 101, 101, 2]; + assert encodes_to(&t, 10, &want, 11); +} + +test zig_story_without_final_newline { + const t : [42]u8 = [79, 110, 99, 101, 32, 117, 112, 111, 110, 32, 97, 32, 116, 105, 109, 101, 44, 10, 116, 104, 101, 114, 101, 32, 119, 97, 115, 32, 97, 32, 108, 105, 116, 116, 108, 101, 32, 103, 105, 114, 108, 46]; + const want : [11]u16 = [1, 463, 105, 48, 14, 464, 410, 504, 112, 50, 2]; + assert encodes_to(&t, 42, &want, 11); +} + +test zig_two_bytes { + const t : [2]u8 = [104, 105]; + const want : [4]u16 = [1, 108, 109, 2]; + assert encodes_to(&t, 2, &want, 4); +} + +test zig_three_bytes { + const t : [3]u8 = [97, 98, 99]; + const want : [5]u16 = [1, 101, 102, 103, 2]; + assert encodes_to(&t, 3, &want, 5); +} + +test zig_marker_inside_a_line { + const t : [18]u8 = [120, 60, 124, 101, 110, 100, 111, 102, 116, 101, 120, 116, 124, 62, 10, 97, 98, 99]; + const want : [7]u16 = [1, 124, 391, 101, 102, 103, 2]; + assert encodes_to(&t, 18, &want, 7); +} + +// ---- tests: edges of encode and decode, also from running the Zig ---------------------------- + +test a_short_output_truncates_like_the_zig { + // Zig: encode("Once upon a time", out[6]) wrote 4 ids: only 4 bytes fit (6 - 2), "Once" + const t : [16]u8 = [79, 110, 99, 101, 32, 117, 112, 111, 110, 32, 97, 32, 116, 105, 109, 101]; + var small : [6]u16 = undefined; + const n : usize = encode(&t, &small); + assert n == 4; + assert small[0] == 1 and small[1] == 328 and small[2] == 707 and small[3] == 2; + // Zig: encode(..., out[2]) wrote BOS and EOS and no content + var tiny : [2]u16 = undefined; + const n2 : usize = encode(&t, &tiny); + assert n2 == 2; + assert tiny[0] == 1 and tiny[1] == 2; +} + +test decode_edges_match_the_zig { + // Zig: decode([0,1,2,3,4,5,259,260,728,729,1000,100]) wrote 9 bytes: 63 0 1 255 101 32 100 105 96 + // (UNK -> '?', byte tokens, two merge tokens, ids >= 729 ignored, specials silent) + const toks : [12]u16 = [0, 1, 2, 3, 4, 5, 259, 260, 728, 729, 1000, 100]; + var out : [32]u8 = undefined; + const m : usize = decode(&toks, &out); + assert m == 9; + assert out[0] == 63 and out[1] == 0 and out[2] == 1 and out[3] == 255; + assert out[4] == 101 and out[5] == 32 and out[6] == 100 and out[7] == 105 and out[8] == 96; +} + +test decode_into_a_short_buffer_stops_like_the_zig { + // Zig: decode([463, 105]) into 3 bytes wrote 79 110 99 ("Onc"); into 5 bytes, 79 110 99 101 32 + const toks : [2]u16 = [463, 105]; + var three : [3]u8 = undefined; + const a : usize = decode(&toks, &three); + assert a == 3; + assert three[0] == 79 and three[1] == 110 and three[2] == 99; + var five : [5]u8 = undefined; + const b : usize = decode(&toks, &five); + assert b == 5; + assert five[3] == 101 and five[4] == 32; +} + +test encode_window_is_encode_without_bos_and_eos { + const t : [16]u8 = [79, 110, 99, 101, 32, 117, 112, 111, 110, 32, 97, 32, 116, 105, 109, 101]; + var work : [64]u16 = undefined; + const m : usize = encode_window(&t, 16, &work); + assert m == 2; + assert work[0] == 463 and work[1] == 105; + // a prefix: the window is the first n bytes only + const m2 : usize = encode_window(&t, 4, &work); + assert m2 == 2; + assert work[0] == 328 and work[1] == 707; + const m0 : usize = encode_window(&t, 0, &work); + assert m0 == 0; +} + +test the_marker_text_is_one_token { + // the merge table was trained on TinyStories, so "<|endoftext|>" is a single id (390) + const t : [13]u8 = [60, 124, 101, 110, 100, 111, 102, 116, 101, 120, 116, 124, 62]; + var work : [16]u16 = undefined; + const m : usize = encode_window(&t, 13, &work); + assert m == 1; + assert work[0] == 390; +} + +// ---- tests: the vocabulary and the merge table itself ---------------------------------------- + +test the_layout_adds_up_to_729 { + assert SPECIAL_COUNT == BYTE_OFFSET; + assert BYTE_OFFSET + BYTE_COUNT == MERGE_OFFSET; + assert @as(usize, @intCast(MERGE_OFFSET)) + MERGE_COUNT == @as(usize, @intCast(VOCAB_SIZE)); + assert VOCAB_SIZE == 729; + assert MERGE_COUNT == 469; +} + +test merge_start_is_the_running_sum_of_merge_len { + assert MERGE_START[0] == 0; + var i : usize = 0; + var bad : i32 = 0; + while (i < 468) { + const s : u16 = MERGE_START[i]; + const l : u16 = @intCast(MERGE_LEN[i]); + if (MERGE_START[i + 1] != s + l) { + bad = bad + 1; + } + i = i + 1; + } + assert bad == 0; + // 1934 bytes in all (bpe_merges.zig: merge_byte_data: [1934]u8) + assert @as(usize, @intCast(MERGE_START[468])) + @as(usize, @intCast(MERGE_LEN[468])) == 1934; +} + +test every_rule_joins_two_earlier_tokens { + // an operand that is a special, or a merge made later, could never fire + var i : usize = 0; + var bad : i32 = 0; + while (i < 469) { + const l : usize = @intCast(MERGE_LEFT[i]); + const r : usize = @intCast(MERGE_RIGHT[i]); + if (l < 4 or r < 4 or l >= 260 + i or r >= 260 + i) { + bad = bad + 1; + } + i = i + 1; + } + assert bad == 0; +} + +test every_rule_spells_the_bytes_of_its_two_halves { + var i : usize = 0; + var bad : i32 = 0; + while (i < 469) { + const l : usize = @intCast(MERGE_LEFT[i]); + const r : usize = @intCast(MERGE_RIGHT[i]); + const ll : usize = token_len(l); + const rl : usize = token_len(r); + if (ll + rl != token_len(260 + i)) { + bad = bad + 1; + } else { + var j : usize = 0; + while (j < ll) { + if (token_byte(260 + i, j) != token_byte(l, j)) { + bad = bad + 1; + } + j = j + 1; + } + var k : usize = 0; + while (k < rl) { + if (token_byte(260 + i, ll + k) != token_byte(r, k)) { + bad = bad + 1; + } + k = k + 1; + } + } + i = i + 1; + } + assert bad == 0; +} + +test every_token_decodes_to_its_bytes { + var tok : usize = 4; + var bad : i32 = 0; + while (tok < 729) { + var one : [1]u16 = undefined; + one[0] = @intCast(tok); + var out : [64]u8 = undefined; + const m : usize = decode(&one, &out); + if (m != token_len(tok)) { + bad = bad + 1; + } else { + var j : usize = 0; + while (j < m) { + if (out[j] != token_byte(tok, j)) { + bad = bad + 1; + } + j = j + 1; + } + } + tok = tok + 1; + } + assert bad == 0; +} + +test every_byte_value_survives_a_round_trip { + var text : [256]u8 = undefined; + var i : usize = 0; + while (i < 256) { + text[i] = @intCast(i); + i = i + 1; + } + var out : [300]u16 = undefined; + const n : usize = encode(&text, &out); + var back : [300]u8 = undefined; + const m : usize = decode(out[0 .. n], &back); + assert m == 256; + var j : usize = 0; + var bad : i32 = 0; + while (j < 256) { + if (back[j] != text[j]) { + bad = bad + 1; + } + j = j + 1; + } + assert bad == 0; +} + +test the_comparison_can_fail { + // encodes_to must say no to a wrong id, a wrong length and a wrong byte count + const t : [3]u8 = [97, 98, 99]; + const right : [5]u16 = [1, 101, 102, 103, 2]; + assert encodes_to(&t, 3, &right, 5); + const wrong : [5]u16 = [1, 101, 102, 104, 2]; + assert !encodes_to(&t, 3, &wrong, 5); + assert !encodes_to(&t, 3, &right, 4); + assert !encodes_to(&t, 2, &right, 5); +} From f1a410ca736122e5956c66ebe1ff78061a553b64 Mon Sep 17 00:00:00 2001 From: Dmitrii Vasilev Date: Sun, 18 Oct 2026 17:19:44 +0700 Subject: [PATCH 2/3] spec(hslm): run the layout checks on variables so none is vacuous (Refs #10372) Co-Authored-By: Claude Sonnet 5.5 --- specs/ml/hslm_corpus.t27 | 11 ++++++++--- specs/nn/hslm_tokenizer.t27 | 17 ++++++++++++----- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/specs/ml/hslm_corpus.t27 b/specs/ml/hslm_corpus.t27 index b05c0b884e..cece47d5c8 100644 --- a/specs/ml/hslm_corpus.t27 +++ b/specs/ml/hslm_corpus.t27 @@ -950,9 +950,14 @@ test the_verifier_refuses_a_wrong_story_count { } test the_chunk_size_is_the_tokenizer_window { - assert CHUNK_CAP == WORK_CAP; - assert TOK_CAP * 2 == IO_CAP; - assert HEADER_BYTES == 12; + var chunk : usize = CHUNK_CAP; + var window : usize = WORK_CAP; + var tok_cap : usize = TOK_CAP; + var io_cap : usize = IO_CAP; + var head : usize = HEADER_BYTES; + assert chunk == window; + assert tok_cap * 2 == io_cap; + assert head == 12; } test the_comparison_helpers_can_fail { diff --git a/specs/nn/hslm_tokenizer.t27 b/specs/nn/hslm_tokenizer.t27 index 5d12081d3e..3696c513ea 100644 --- a/specs/nn/hslm_tokenizer.t27 +++ b/specs/nn/hslm_tokenizer.t27 @@ -653,11 +653,18 @@ test the_marker_text_is_one_token { // ---- tests: the vocabulary and the merge table itself ---------------------------------------- test the_layout_adds_up_to_729 { - assert SPECIAL_COUNT == BYTE_OFFSET; - assert BYTE_OFFSET + BYTE_COUNT == MERGE_OFFSET; - assert @as(usize, @intCast(MERGE_OFFSET)) + MERGE_COUNT == @as(usize, @intCast(VOCAB_SIZE)); - assert VOCAB_SIZE == 729; - assert MERGE_COUNT == 469; + // copied into variables so that the compiler cannot fold the checks away + var specials : u16 = SPECIAL_COUNT; + var bytes_at : u16 = BYTE_OFFSET; + var byte_n : u16 = BYTE_COUNT; + var merges_at : u16 = MERGE_OFFSET; + var merge_n : usize = MERGE_COUNT; + var vocab : u16 = VOCAB_SIZE; + assert specials == bytes_at; + assert bytes_at + byte_n == merges_at; + assert @as(usize, @intCast(merges_at)) + merge_n == @as(usize, @intCast(vocab)); + assert vocab == 729; + assert merge_n == 469; } test merge_start_is_the_running_sum_of_merge_len { From 479f6557c9c32f1c83329e4a59abbb759b9551e4 Mon Sep 17 00:00:00 2001 From: Dmitrii Vasilev Date: Sun, 18 Oct 2026 17:20:59 +0700 Subject: [PATCH 3/3] spec(hslm): a lone newline is an empty story, a blank line is text (Refs #10372) Co-Authored-By: Claude Sonnet 5.5 --- specs/ml/hslm_corpus.t27 | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/specs/ml/hslm_corpus.t27 b/specs/ml/hslm_corpus.t27 index cece47d5c8..d0a47251e4 100644 --- a/specs/ml/hslm_corpus.t27 +++ b/specs/ml/hslm_corpus.t27 @@ -678,12 +678,17 @@ test no_input_means_no_tokens { assert stories == 0; } -test only_blank_lines_make_nothing { +test a_lone_newline_makes_nothing_but_a_blank_line_is_text { + // "\n" is just the terminator of an empty story; "\n\n" is a story of one newline (id 14) tin[0] = 10; tin[1] = 10; - convert_bytes(tin[0 .. 2], -1); + convert_bytes(tin[0 .. 1], -1); assert tn == 0; assert stories == 0; + convert_bytes(tin[0 .. 2], -1); + const want : [2]u16 = [14, 2]; + assert tokens_are(&want, 2); + assert stories == 1; } test the_marker_inside_a_line_is_text {