diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 97e4945b..2319a90a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -50,6 +50,21 @@ jobs: python -m pip install pytest PYTHONPATH=src python -m pytest tests/clef -q + kev-export: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v5 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: CPU-only export dependencies + run: | + python -m pip install torch==2.8.0 --index-url https://download.pytorch.org/whl/cpu + python -m pip install -r recipe/kev/requirements.txt + - name: Export contract tests (no weights or GPU) + run: python -m unittest discover -s tests/kev -p 'test_*.py' -v + cuda-contract: # The CUDA backend contract and its checker. No CUDA toolkit and no GPU: both # read files as text. diff --git a/Cargo.lock b/Cargo.lock index 3dff4a6d..3cf4279f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1263,6 +1263,23 @@ dependencies = [ "tokio", ] +[[package]] +name = "omni-kev-native" +version = "0.1.0" +dependencies = [ + "anyhow", + "axum", + "omni-qwen3-5-native", + "omni-runtime", + "regex", + "safetensors 0.8.0", + "serde", + "serde_json", + "sha2 0.10.9", + "tokenizers 0.22.2", + "tokio", +] + [[package]] name = "omni-laya" version = "0.2.0" diff --git a/Cargo.toml b/Cargo.toml index 7910b3dd..1ebf8e22 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,3 +1,3 @@ [workspace] -members = ["src/frontend", "src/runtime", "src/models/clm", "src/models/cua_s1/native", "src/models/qwen3_5/native", "src/models/open_jev/native", "src/models/jev_vl/native", "src/models/omnijev/native", "src/models/laya", "src/backends/cuda", "src/models/decider/native", "src/models/jemm/native", "src/models/startlux/native", "src/models/clef/native"] +members = ["src/frontend", "src/runtime", "src/models/clm", "src/models/cua_s1/native", "src/models/qwen3_5/native", "src/models/open_jev/native", "src/models/jev_vl/native", "src/models/omnijev/native", "src/models/laya", "src/backends/cuda", "src/models/decider/native", "src/models/jemm/native", "src/models/startlux/native", "src/models/clef/native", "src/models/kev/native"] resolver = "3" diff --git a/README.md b/README.md index 1502a8fb..63aee228 100644 --- a/README.md +++ b/README.md @@ -191,6 +191,7 @@ checkpoint. Public231 concurrency-1 latency for heads `1ba07d1`, `f968e576`, | CLM-v0.1-8B | [External worker with a CPU stub encoder](recipe/clm/README.md); contract checks only, real Qwen3-8B decisions unverified by this recipe | | LFM2.5-350M | [Transformers worker](recipe/lfm2/README.md); text state and independent choice questions | | OmniJev-4B v1.1 | [Native Rust/CUDA worker](recipe/omnijev/native.md), eager: one image, Choice, Noul and Score; [reference comparison on sm_89](recipe/omnijev/native.md#comparison-with-the-reference) | +| Kev-4B | [Eager native Rust/CUDA text worker](recipe/kev/native.md); Choice/Noul/Score; [BF16 acceptance passes, including 36 Noul boundaries](recipe/kev/noul-boundary.md); author-run; review pending ([#27](https://github.com/InferMatrixLab/system1-omni/issues/27)) | | D3 (Decision 3.0) | [Python reference worker](recipe/d3/reference.md) on the checkpoint's pinned runtime; `choice`/`noul`/`score` over text, image and video state; author-run on A100 for text and media serving (see the integration and media PRs) | | Vela 2.0 0.3B | [Python reference worker](recipe/vela/reference.md) on the checkpoint's pinned runtime; all five kinds (`choice`/`noul`/`score`/`span`/`set`) for text state; CPU FP32 validated (author-run, evidence in the integration PR) | | Vela 2.0 0.8B hybrid | [Python reference worker](recipe/vela/reference.md) on the checkpoint's pinned runtime; same surface plus `span_heads`; CPU FP32 parity recorded in the integration PR | diff --git a/docs/architecture.md b/docs/architecture.md index 276143e3..436b8ed1 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -7,7 +7,7 @@ design. Concrete input/output types follow each executor's supported layout. ## Implementation status The [Rust frontend](../src/frontend/README.md) currently forwards HTTP requests -to separately running workers. Cua-S1, Open-Jev, Decider, and StartLux Instant +to separately running workers. Cua-S1, Open-Jev, Decider, Kev-4B, and StartLux Instant 4B have native Rust/CUDA workers that share the [Qwen3.5/3.8 executor](../src/models/qwen3_5/native/), which accepts single prompts and bounded packed prefill. StartLux Instant uses a @@ -81,6 +81,14 @@ FP32 adapters and framework-reference arithmetic. It computes selected BF16 label-head logits on CUDA, then calibrates typed answers on the CPU. JEMM admits one complete text or 1–4-image request and synchronizes both model streams before releasing its resources. See its [model contract](../src/models/jemm/README.md). +Kev uses merged BF16 weights and an optional Torch 2.8 numerical profile. It +recomputes state then question in eager mode, matching the pinned official +padding and projection shapes, and downloads selected option/decision hidden +rows for its FP32 pointer head. Temporary GDN/KV state belongs to one question +call and is drained before release; it is not shared between requests. Its +processor owns prompt tokens and question geometry, while complete-question +calibration and typed response reconstruction remain in postprocessing. +The existing Qwen default, fixed and reference profiles keep their own paths. Decider projects selected tied-embedding rows on CUDA with BF16 output rounding before FP32 calibration and CPU response assembly. Laya computes its scorer and action head on CUDA. Its fixed-shape @@ -138,6 +146,7 @@ and candidate identity, usage, and response metadata outside the executor. | Cua-S1 | One unpadded token-ID vector and option count per question, in request order. | One FP32 answer-letter logit vector per question. | Per-question softmax, choice/confidence, ordered answers, and token usage. | | Open-Jev | Token-ID vectors grouped by question, then independent candidate, in request order. | One FP32 learned scalar per candidate in the same grouping. | Add the `noul` false logit of zero, calibrate across each complete question, and restore typed answers, usage, and metadata. | | Decider | Complete independent token-ID rows in question order; Score expands one no/yes row per level. | One BF16-projection-rounded FP32 selected-label logit vector per row. | Per-type calibration, whole-Score normalization, ordered typed answers and unique-prefix token usage. | +| Kev-4B | Complete text rows, consecutive positions and ordered option/decision readouts, one row per question. | One FP32 pointer-logit vector per complete question. | Saved-temperature softmax over all candidates, ordered Choice/Noul/Score answers and official token billing. | | Laya | One padded request: token IDs, true lengths, question types and ordered option markers; at most 16 questions, 512 tokens per row and 2048 markers. | Per-question FP32 option logits and two action logits copied back after GPU heads. | Calibrate and decode ordered `choice`, `score` and `noul` answers, usage and metadata. | | JEMM | Ordered token IDs, three-axis positions, candidate counts and optional shared image patches for one complete request. | Per-question selected BF16 label logits copied back as FP32. | Apply checksum-covered text/image calibration and reconstruct ordered `choice`, `score` and `noul` answers and usage. | | StartLux Instant 4B | One unpadded token-ID prompt per question, or per knockout group and then a final prompt, in request order. | FP32 letter logits from a CPU dot of the tied embedding rows at the last position. | Per-type temperature, none-check split, wide-choice residual, and ordered `choice`, `noul`, and `score` answers with token usage. | @@ -149,7 +158,7 @@ Open-Jev's model-specific batch adapter packs up to 16 independent candidates an execute alone. It restores question/candidate grouping before normalization. Laya batches questions within one request. Shared runtime admission precedes blocking dispatch: Cua-S1 admits one question forward at a -time; Open-Jev, Decider, Laya, StartLux Instant, and Clef admit one complete request. Clef's admitted work is the frame exchange with its executor process. StartLux runs each prompt inside that admission one at a time. Cua-S1's CPU letter projection +time; Open-Jev, Decider, Kev, Laya, StartLux Instant, and Clef admit one complete request. Kev runs all question rows and its FP32 head inside that admission; listener readiness follows a real mixed-question warmup. Clef's admitted work is the frame exchange with its executor process. StartLux runs each prompt inside that admission one at a time. Cua-S1's CPU letter projection stays outside admission; Open-Jev's scalar heads and Laya's GPU heads and synchronized readback remain inside their request unit. Decider includes every backbone row and CUDA head projection, synchronizes both streams and retires @@ -212,6 +221,11 @@ semantics. Preserve whole-request failure behavior, error status/body, and usage fields when splitting or regrouping work. +Kev prepares one complete causal row per question, reads every candidate closing +marker plus the final decision marker through the shared Qwen selected-position +readout, and applies its FP32 pointer head. Its eager executor recomputes the state +for each question and admits one whole request through the serial scheduler. + Cua-S1 prepares one prompt per question and reads option-letter logits. Open-Jev prepares independent candidate prompts and normalizes across the complete question's candidates. Laya pads prepared questions into one request diff --git a/docs/supported-models.md b/docs/supported-models.md index ca929b0e..e323a75a 100644 --- a/docs/supported-models.md +++ b/docs/supported-models.md @@ -34,6 +34,16 @@ Models that are being added are also tracked in issues labeled [new model](https | Clef (27B) | Same `omni-clef` server and text contract as Clef-flash, selected with `--variant clef` | Contract tests only; no inference | Not run | Not supported | Same stack; pinned `Cloudflare/clef` snapshot (~55.0 GB). Weights and joint-head config differ from flash | | pplx-decider-v1.1-27b | [Python reference worker](../recipe/pplx_decider/README.md) on the checkpoint's own runtime. Text and up to four images; `choice`/`noul`/`score`. Native CUDA executor is not this worker. Lab L20X-labeled GPUs are the H200 target. The causal-conv1d build keeps sm_90 and sm_89. Capability 8.9 is ready on FLA chunk. Capability 9.0 uses all-sequence FlashInfer. The kept primary rotates 48 `cu_seqlens` buffers per device and length, passes `use_cp` false, skips the q/k repeat, and packs q and k before the L2-norm view. `801ef7b` fused the 48 linear gated RMSNorms and missed the median (p50 89.172 ms, p95 274.790 ms), so that kernel stays off. `2bbbe4e` replaced `silu(gate) * up` with FLA `swiglu_fwd` and the owner closeout rejected it (p50 97.613 ms, p95 292.795 ms), so that kernel stays off. Gate and up GEMMs stay separate. Residual RMSNorm and the gated output norm stay eager. An unknown `PPLX_GDN_PREFILL` fails closed on every device, including capability 8.9. `3ecdf5a` crashed in warmup (0/231). Always-on fusion `3afb472` and the 512-token gate `705ae089` missed the median. The 48-slot FlashInfer state ring `17a7b120` missed the gate. `PPLX_GDN_PREFILL=fla` reproduces the prior `f968e576` chunk. `PPLX_GDN_PREFILL=short` is the rejected ≤256 dispatch. SDPA stays on the checkpoint order. The fused gate-up GEMM is refused. The forward stays eager. `PPLX_COMPILE=dynamic` and `PPLX_COMPILE=dynamic-tail` are rejected opt-ins | Contract checks only; no inference | Reported load on 1× lab L20X (sm_89, ~140 GiB) at head `1ba07d1`: BF16 resident ~50333 MiB, no OOM. Historical primary: 231/231, p50 98.777 ms, p95 341.202 ms. Prior primary is the `f968e576` remeasure on the lab L20X-labeled GPU, `compute_capability=9.0`, `fla-chunk`, `flashinfer` null: 231/231, p50 94.077 ms, p95 300.200 ms (p50 −4.76%, p95 −12.02%). Prior primary `ffae182`: 231/231, p50 92.844 ms, p95 296.458 ms (p50 −1.31%, p95 −1.25%). Prior primary `4b99b73`: 231/231, p50 88.942 ms, p95 295.797 ms (p50 −4.20%, p95 −0.22%), 48-slot `cu_seqlens`, quality `choice_acc=0.9065` (139), `noul_acc=0.8784` (74), `score_mae=0.2035` (18). Kept primary is `fcf5e2b`: 231/231, p50 87.782 ms, p95 291.183 ms (p50 −1.30%, p95 −1.56%), grouped-value FlashInfer with q and k packed, same quality digits. Uncached all-sequence FlashInfer `3c72f02`: p50 88.808 ms, p95 300.751 ms, gate failed. Short-sequence FlashInfer `a50b9aa`: p50 95.446 ms, p95 300.871 ms, gate failed. cuDNN-first SDPA `0dcaa0ac`: p50 138.501 ms, p95 332.859 ms, gate failed. Fused gate-up `8b4ee326`: p50 102.437 ms, p95 304.970 ms, gate failed. Always-on dynamic compile `e36b32ed`: p50 97.751 ms, p95 263.124 ms, gate failed (p95 improved). 512-token compile `7e3b1be`: p50 100.037 ms, p95 257.084 ms, gate failed. Isolated-median compile `34ba5de`: p50 97.084 ms, p95 256.679 ms, gate failed (median probe was eager). FlashInfer state-workspace ring `17a7b120`: p50 89.165 ms, p95 295.723 ms, gate failed. Always-on FLA residual RMSNorm `3afb472`: p50 93.935 ms, p95 274.902 ms, gate failed. Length-gated FLA residual RMSNorm `705ae089`: p50 94.604 ms, p95 274.835 ms, gate failed. `use_cp=auto` `a6f4d9a`: p50 112.294 ms, p95 299.419 ms, gate failed. Grouped-value `3ecdf5a` crashed in warmup (0/231, no p50 or p95). The fused linear gated-norm candidate has no sample. Does not enter the H800 board | Not supported | Python 3.12, pinned checkpoint revision `14c9d90137962d300f3de95633f7b6535e80c5ce`, checkpoint `requirements.txt`, `causal-conv1d` 1.7.0, `flash-linear-attention` 0.5.2, optional `flashinfer-python` 0.6.16.post3. Shard sizes in the release manifest sum with the readout to about 48.6 GiB before activations | +Kev-4B has an [eager native Rust/CUDA text worker](../recipe/kev/native.md) for +Choice, Noul and Score. All frozen gates pass over 5,484 requests / 5,662 +questions from the fixed official BF16 default service: maximum probability error 0.0024709404 <=0.01 and Score error +0.00000305218 <=0.1 level. Near-boundary Noul coverage is 36/10 (Choice 28/10). +Processing, independent head, 896-question task-quality gates, real warmup and +55 HTTP response comparisons pass. Author-run evidence; maintainer review remains pending. +See the [boundary qualification and limitations](../recipe/kev/noul-boundary.md) +for [#27](https://github.com/InferMatrixLab/system1-omni/issues/27). Other sizes +and backends remain unverified. + - **Validated:** covered by the recipe on `main` or by the checks in the linked merged pull request. - **Author-run:** recorded by the recipe author on the stated hardware and not independently reproduced; the row names its owner. - **Unverified:** the worker accepts this device, but no recipe or merged pull request covers it. diff --git a/mkdocs.yml b/mkdocs.yml index 86abd35a..f83f347f 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -81,6 +81,11 @@ nav: - JEV-27B-VL experimental worker: recipe/jev_vl/README.md - JEV-27B-VL validation status: recipe/jev_vl/validation.md - OmniJev preparation: recipe/omnijev/README.md + - Kev-4B preparation: recipe/kev/README.md + - Kev-4B native worker: recipe/kev/native.md + - Kev-4B service acceptance: recipe/kev/service-acceptance.md + - Kev-4B numerical repair: recipe/kev/numerical-diagnosis.md + - Kev-4B Noul boundary qualification: recipe/kev/noul-boundary.md - OmniJev native worker: recipe/omnijev/native.md - CLM stub-encoder contract: recipe/clm/README.md - LFM2.5-350M choice worker: recipe/lfm2/README.md diff --git a/recipe/README.md b/recipe/README.md index 22179d21..8bfaf113 100644 --- a/recipe/README.md +++ b/recipe/README.md @@ -42,6 +42,9 @@ For a first real decision, follow the [complete CPU walkthrough](../docs/getting - [OmniJev-4B v1.1 preparation](omnijev/README.md): export the checkpoint and regenerate the CPU fixtures; [native worker](omnijev/native.md): serve it on CUDA and compare it with the reference. +- [Kev-4B preparation](kev/README.md): export all LoRA projections and validate + the Rust FP32 pointer head on CPU; [native worker](kev/native.md) serves eager + text Choice/Noul/Score with explicit numerical acceptance gates. - [CLM behind the frontend](clm/README.md): run CLM's own server behind the frontend on CPU with a stub encoder, and what the response comparison has to allow for. - [pplx-decider reference worker](pplx_decider/README.md): pin diff --git a/recipe/kev/README.md b/recipe/kev/README.md new file mode 100644 index 00000000..e9a17eb1 --- /dev/null +++ b/recipe/kev/README.md @@ -0,0 +1,90 @@ +# Kev-4B: checkpoint export and pointer-head validation + +This is a preparation recipe for [#27](https://github.com/InferMatrixLab/system1-omni/issues/27), +with a separate [native worker recipe](native.md). The [model contract](../../src/models/kev/README.md) +records the pinned artifacts, numerical boundaries and worker ownership. +All commands below run from the repository root. Native serving requires a CUDA +FP32 LoRA merge matching the official default service. Head/unit tests remain CPU-only. + +## Environment and downloads + +Use Python 3.12 and the stable Rust toolchain used by this repository. The +export reads about 9.5 GB of inputs and writes about 8.5 GB; reserve at least 25 GB +of free disk and 24 GB of host RAM. It processes one source shard at a time. + +```sh +python3.12 -m venv .venv-kev +.venv-kev/bin/pip install torch==2.8.0 --index-url https://download.pytorch.org/whl/cu128 +.venv-kev/bin/pip install -r recipe/kev/requirements.txt huggingface-hub==1.33.0 + +.venv-kev/bin/hf download Qwen/Qwen3.5-4B-Base \ + --revision 1001bb4d826a52d1f399e183466143f4da7b741b \ + --include 'config.json' 'tokenizer.json' 'tokenizer_config.json' \ + 'model.safetensors.index.json' '*.safetensors' --local-dir weights/kev-base +.venv-kev/bin/hf download jaredpalmer/kev-4b \ + --revision 139fdd94f1b6a6ad80cc15e08fcb99cac885a101 \ + --include 'adapter_config.json' 'adapter_model.safetensors' 'head.pt' \ + --local-dir weights/kev-adapter + +gpu run --gpus 1 --wait 2h --timeout 30m --note 'Kev pinned CUDA export' -- \ + .venv-kev/bin/python recipe/kev/export.py \ + --base weights/kev-base --checkpoint weights/kev-adapter --out weights/kev-native +``` + +For CPU-only export/head diagnosis, add `--merge-device cpu` and choose a separate +output directory. That artifact is deliberately not accepted by the worker: +nine BF16 weights can differ from official GPU merging despite both merges using +FP32. `merge_numerics` records the device and exact Torch wheel. The CUDA export +requires Torch 2.8.0+cu128 and disables TF32 for the LoRA matmul. + +Every required input must match its pinned SHA-256. The output directory must not +already exist. `kev_export.json` lists merged targets, output hashes, FP32 head +dimensions and calibration, plus the base tokenizer's five delimiter IDs. +Do not replace these artifacts with another Kev size or a bare Qwen checkpoint. +The target inventory is specific to this hash-pinned 32-layer Kev-4B base: full +attention at layers 3, 7, ..., 31 and all five GDN projections in other layers. + +## CPU checks + +```sh +.venv-kev/bin/python -m unittest discover -s tests/kev -p 'test_*.py' -v +cargo test --locked -p omni-kev-native -- --list +cargo test --locked -p omni-kev-native +KEV_HEAD="$PWD/weights/kev-native/head.safetensors" \ + cargo test --release --locked -p omni-kev-native --test head \ + -- --ignored --nocapture +``` + +The final test is opt-in because it needs the released head, but it is still +CPU-only. It compares the Rust implementation against the committed small oracle +for 1/2/7/31/255 options. Default CI executes the ordinary head tests without +downloading weights. Synthetic two-shard export tests exercise real safetensors +I/O, all 248 adapter targets, index/manifest hashes, preserved FP32 head and +calibration, final directory publication, and cleanup after a late shard failure. +Only input hash pins are substituted for generated fixture files; release-file +validation is unchanged. Full backbone inference is not part of these checks. + +To regenerate the oracle using the official class rather than this implementation: + +```sh +git clone https://github.com/jaredpalmer/kev.git /tmp/kev-reference +git -C /tmp/kev-reference checkout --detach 2074cc44256200bb37540103fa4572901faa10f3 +.venv-kev/bin/python tests/kev/generate_head_fixture.py \ + --reference-model /tmp/kev-reference/kev/model.py \ + --head weights/kev-adapter/head.pt --out /tmp/kev-head-reference.json +# Exact provenance diff requires torch==2.8.0+cpu, as recorded in head.json. +# Other wheel suffixes change metadata even if numerical values agree. +diff -u tests/kev/data/head.json /tmp/kev-head-reference.json +``` + +The generator verifies the complete source and head hashes, extracts only the +upstream `PointerHead` class, and uses CPU PyTorch. It does not need Transformers, +PEFT or the backbone. The fixture's Torch version includes the wheel's local +suffix, so a different wheel can change provenance even when numbers agree. + +## Native worker + +The [native worker recipe](native.md) builds and serves the eager Rust/CUDA worker, +reproduces exact processing fixtures, and runs the three-way full-model and HTTP +acceptance checks. Export/head checks alone do not establish full-model parity. +The export/head and complete worker remain separate review boundaries. diff --git a/recipe/kev/build_cuda.sh b/recipe/kev/build_cuda.sh new file mode 100755 index 00000000..0170a4fe --- /dev/null +++ b/recipe/kev/build_cuda.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +# Build-only dependencies: PyTorch 2.8.0 headers and its exact CUTLASS submodule. +# The resulting worker/library does not link libtorch or load Python. +set -euo pipefail +root=$(cd "$(dirname "$0")/../.." && pwd) +out=${1:?usage: build_cuda.sh OUTPUT [COMPUTE_CAPABILITY]} +arch=${2:-89} +python=${KEV_PYTHON:-python3} +cache=${KEV_BUILD_CACHE:-${TMPDIR:-/tmp}/kev-cuda-headers} +revision=ad7b2f5e84fcfa124cb02b91d5bd26d238c0459e +digest=36429479df2ab1cd1c7981c2d34e786106326ccfefc8846f902f76151a65e112 +export KEV_ATEN_INCLUDE_DIR +KEV_ATEN_INCLUDE_DIR=$("$python" - <<'PY' +from pathlib import Path +import torch +if torch.__version__.split('+')[0] != '2.8.0': + raise RuntimeError('Kev requires PyTorch 2.8.0 build headers') +print(Path(torch.__file__).resolve().parent / 'include') +PY +) +mkdir -p "$cache" +archive="$cache/cutlass-$revision.tar.gz" +if [ ! -f "$archive" ]; then + tmp=$(mktemp "$cache/download.XXXXXX") + trap 'rm -f "$tmp"' EXIT + curl --fail --location --retry 2 "https://codeload.github.com/NVIDIA/cutlass/tar.gz/$revision" -o "$tmp" + printf '%s %s\n' "$digest" "$tmp" | sha256sum --check --status + mv "$tmp" "$archive" +fi +printf '%s %s\n' "$digest" "$archive" | sha256sum --check --status +# Re-extract verified bytes into a private staging directory; do not trust a stale tree. +headers=$(mktemp -d "$cache/headers.XXXXXX") +trap 'rm -rf "$headers"' EXIT +tar -xzf "$archive" -C "$headers" +export KEV_CUTLASS_INCLUDE_DIR="$headers/cutlass-$revision/include" +bash "$root/src/backends/cuda/qwen3_5/build.sh" "$out" "$arch" +# Header instantiation embeds PyTorch/CUTLASS code in the library. Keep their +# redistribution notices next to the binary, even though neither runtime is linked. +cp "$root/src/models/qwen3_5/native/licenses/PYTORCH-2.8-LICENSE" "$out/PYTORCH-LICENSE" +cp "$root/src/models/qwen3_5/native/licenses/CUTLASS-LICENSE" "$out/CUTLASS-LICENSE" +cat > "$out/KEV_CUDA_NOTICES.txt" <<'NOTICE' +This library includes header-instantiated PyTorch 2.8.0 memory-efficient attention +(Copyright (c) Meta Platforms, Inc. and affiliates) and CUTLASS +ad7b2f5e84fcfa124cb02b91d5bd26d238c0459e. Redistribute this notice and the adjacent +PYTORCH-LICENSE and CUTLASS-LICENSE files with the library. +NOTICE diff --git a/recipe/kev/export.py b/recipe/kev/export.py new file mode 100644 index 00000000..9ac680ff --- /dev/null +++ b/recipe/kev/export.py @@ -0,0 +1,274 @@ +"""Export pinned Kev-4B language weights and pointer head (no serving).""" + +import argparse +import hashlib +import json +import math +import os +import shutil +import tempfile +from pathlib import Path + +import torch +from safetensors import safe_open +from safetensors.torch import load_file, save_file + +MODEL_ID = "jaredpalmer/kev-4b" +REVISION = "139fdd94f1b6a6ad80cc15e08fcb99cac885a101" +BASE_ID = "Qwen/Qwen3.5-4B-Base" +BASE_REVISION = "1001bb4d826a52d1f399e183466143f4da7b741b" +REFERENCE_REVISION = "2074cc44256200bb37540103fa4572901faa10f3" +SPECIAL = { + "<|fim_prefix|>": 248060, + "<|fim_middle|>": 248061, + "<|box_start|>": 248049, + "<|box_end|>": 248050, + "<|fim_suffix|>": 248062, +} +BASE_FILES = { + "config.json": "ddc63e1c717afa86c865bb5e01313d89d72bb53b97ad4a8a03ba8510c0621670", + "tokenizer.json": "fe000e3ed39ed12b8d2481d527d44f93c65d37e87645d2dcc80d1bf9d50d2927", + "tokenizer_config.json": "3891e840d7dc5fca0af33d3a25083a735e36fe06214e3f707024820cb6b9f89c", + "model.safetensors.index.json": "eae340074abb0a5f31a6621f7ae8e8248a7c1790df04a722c4e4b70c2a6d1dbb", + "model.safetensors-00001-of-00002.safetensors": "df547074dce70532a0493e5433152bd17a65efb89088cfabc2e7e2371a93d712", + "model.safetensors-00002-of-00002.safetensors": "590fbaac095dd31db886c322d9d2f7df47777966391acf306ddddc3e4e3a15ef", +} +CHECKPOINT_FILES = { + "adapter_config.json": "8a05dfd6c5e7f61a62e6db5d8de8093a8dbbd105aa7ee7b5e7b73f6249f45617", + "adapter_model.safetensors": "90e817356246e7f18bfa7ca3d31794cd4fbeb3332a66a84cb51d9ceae925f2b2", + "head.pt": "dd633435998ecc751ac538717a3742e32149500fabf7d7276287dbf0693f347c", +} + + +def expected_targets(): + """Kev-4B only: 96 MLP, 32 attention and 120 GDN targets. + + The hash-pinned base config fixes 32 layers and full attention at 3 mod 4; + this inventory is not a validator for other Kev sizes or Qwen checkpoints. + """ + targets = set() + for layer in range(32): + modules = [f"mlp.{p}_proj" for p in ("gate", "up", "down")] + if layer % 4 == 3: + modules += [f"self_attn.{p}_proj" for p in ("q", "k", "v", "o")] + else: + modules += [f"linear_attn.{p}" for p in ( + "in_proj_qkv", "in_proj_a", "in_proj_z", "in_proj_b", "out_proj" + )] + targets.update(f"model.language_model.layers.{layer}.{m}.weight" for m in modules) + return targets + + +def validate_targets(targets): + expected = expected_targets() + if set(targets) != expected: + raise ValueError( + f"LoRA target mismatch: missing={sorted(expected - set(targets))}, " + f"unexpected={sorted(set(targets) - expected)}" + ) + + +def sha256(path): + h = hashlib.sha256() + with path.open("rb") as stream: + for block in iter(lambda: stream.read(1 << 24), b""): + h.update(block) + return h.hexdigest() + + +def verify_files(directory, expected): + for name, digest in expected.items(): + if sha256(directory / name) != digest: + raise ValueError(f"{name} does not match the pinned release") + + +def validate_tokenizer(path): + tokens = { + t["content"]: t["id"] for t in json.loads(path.read_text())["added_tokens"] + } + if any(tokens.get(token) != value for token, value in SPECIAL.items()): + raise ValueError("base tokenizer has unexpected delimiter IDs") + + +def adapter_pairs(tensors): + pairs = {} + for name, value in tensors.items(): + prefix, marker, side = name.partition(".lora_") + if ( + not prefix.startswith("base_model.model.layers.") + or not marker + or side not in ("A.weight", "B.weight") + ): + raise ValueError(f"unsupported adapter tensor: {name}") + if ( + value.dtype != torch.float32 + or value.ndim != 2 + or not torch.isfinite(value).all() + ): + raise ValueError(f"invalid FP32 adapter tensor: {name}") + target = ( + "model.language_model." + + prefix.removeprefix("base_model.model.") + + ".weight" + ) + pairs.setdefault(target, {})[side[0]] = value + if any(set(pair) != {"A", "B"} for pair in pairs.values()): + raise ValueError("incomplete LoRA pair") + return pairs + + +def merge_weight(weight, a, b): + """Kev's alpha/r = 2: FP32 delta and addition, one final BF16 rounding.""" + if ( + weight.ndim != 2 + or a.shape != (16, weight.shape[1]) + or b.shape != (weight.shape[0], 16) + ): + raise ValueError("LoRA shape mismatch") + merged = (weight.float() + (b @ a) * 2.0).to(torch.bfloat16) + if not torch.isfinite(merged).all(): + raise ValueError("non-finite merged weight") + return merged.contiguous() + + +def pointer_head(meta): + expected = { + "base": BASE_ID, + "base_revision": BASE_REVISION, + "lora": 16, + "head_dim": 256, + "option_isolation": False, + "special_embeddings": False, + "weights_dtype": "fp32", + } + if any(meta.get(key) != value for key, value in expected.items()): + raise ValueError("unsupported Kev head metadata") + temperature = meta.get("temperature") + if ( + not isinstance(temperature, (int, float)) + or not math.isfinite(temperature) + or temperature <= 0 + ): + raise ValueError("invalid calibration temperature") + shapes = { + "q.weight": (256, 2560), + "k.weight": (256, 2560), + "q.bias": (256,), + "k.bias": (256,), + } + head = meta["head"] + if set(head) != set(shapes): + raise ValueError("unexpected pointer-head keys") + for name, shape in shapes.items(): + value = head[name] + if ( + value.dtype != torch.float32 + or tuple(value.shape) != shape + or not torch.isfinite(value).all() + ): + raise ValueError(f"invalid pointer-head tensor: {name}") + return {name: value.contiguous() for name, value in head.items()}, temperature + + +def export(base, checkpoint, out, merge_device="cuda"): + if out.exists(): + raise ValueError("output already exists; choose a new directory") + verify_files(base, BASE_FILES) + verify_files(checkpoint, CHECKPOINT_FILES) + validate_tokenizer(base / "tokenizer.json") + # Hash the pickle before parsing and use the restricted weights-only loader. + head, temperature = pointer_head( + torch.load(checkpoint / "head.pt", map_location="cpu", weights_only=True) + ) + pairs = adapter_pairs( + load_file(checkpoint / "adapter_model.safetensors", device="cpu") + ) + validate_targets(pairs) + if merge_device not in ("cpu", "cuda"): + raise ValueError("merge device must be cpu or cuda") + if merge_device == "cuda": + if torch.__version__ != "2.8.0+cu128": + raise ValueError("official-service export requires torch==2.8.0+cu128") + if not torch.cuda.is_available(): + raise ValueError("CUDA export requires a reserved GPU; use gpu run") + torch.set_float32_matmul_precision("highest") + torch.backends.cuda.matmul.allow_tf32 = False + out.parent.mkdir(parents=True, exist_ok=True) + staging = Path(tempfile.mkdtemp(prefix=f".{out.name}-", dir=out.parent)) + used, weight_map = set(), {} + try: + for name in BASE_FILES: + if not name.endswith(".safetensors"): + continue + weights = {} + with safe_open(base / name, framework="pt", device="cpu") as source: + for key in source.keys(): # noqa: SIM118 - safe_open exposes keys(), not iteration + if not key.startswith("model.language_model."): + continue # Kev is text-only; exclude vision and MTP. + weight = source.get_tensor(key) + if key in pairs: + pair = pairs[key] + weight = merge_weight(weight.to(merge_device), pair["A"].to(merge_device), pair["B"].to(merge_device)).cpu() + used.add(key) + else: + weight = weight.to(torch.bfloat16).contiguous() + if not torch.isfinite(weight).all(): + raise ValueError(f"non-finite base tensor: {key}") + weights[key] = weight + if key in weight_map: + raise ValueError(f"duplicate base tensor: {key}") + weight_map[key] = name + if weights: + save_file(weights, staging / name) + del weights + if used != set(pairs): + raise ValueError(f"unmerged adapter targets: {sorted(set(pairs) - used)}") + (staging / "model.safetensors.index.json").write_text( + json.dumps({"weight_map": weight_map}, indent=2) + "\n" + ) + for name in ("config.json", "tokenizer.json", "tokenizer_config.json"): + shutil.copyfile(base / name, staging / name) + save_file(head, staging / "head.safetensors") + manifest = { + "format": "kev-4b-merged/1", + "merge_numerics": {"device": merge_device, "accumulation": "float32", "torch": torch.__version__}, + "model_id": MODEL_ID, + "revision": REVISION, + "base_id": BASE_ID, + "base_revision": BASE_REVISION, + "reference_revision": REFERENCE_REVISION, + "backbone_dtype": "bfloat16", + "head_dtype": "float32", + "hidden_size": 2560, + "pointer_size": 256, + "temperature": temperature, + "special_tokens": SPECIAL, + "merged_targets": sorted(used), + "inputs": {"base": BASE_FILES, "checkpoint": CHECKPOINT_FILES}, + "files": {p.name: sha256(p) for p in sorted(staging.iterdir())}, + } + (staging / "kev_export.json").write_text(json.dumps(manifest, indent=2) + "\n") + os.rename(staging, out) + finally: + if staging.exists(): + shutil.rmtree(staging) + return manifest + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--base", required=True, type=Path) + parser.add_argument("--checkpoint", required=True, type=Path) + parser.add_argument("--out", required=True, type=Path) + parser.add_argument("--merge-device", choices=("cuda", "cpu"), default="cuda", + help="CUDA matches the default service; CPU is diagnostic/head-only") + args = parser.parse_args() + torch.set_num_threads(8) + manifest = export(args.base, args.checkpoint, args.out, args.merge_device) + print( + f"Exported {len(manifest['merged_targets'])} merged projections to {args.out}" + ) + + +if __name__ == "__main__": + main() diff --git a/recipe/kev/native.md b/recipe/kev/native.md new file mode 100644 index 00000000..a0ee3419 --- /dev/null +++ b/recipe/kev/native.md @@ -0,0 +1,213 @@ +# Kev-4B eager native text worker + +The worker implements Kev's `choice`, `noul`, and `score` contract in Rust and runs +its Qwen3.5 language backbone on CUDA. It uses the [preparation export](README.md), +the shared Qwen selected-position readout, and Kev's trained FP32 pointer head. +No Python runs in the worker. Other Kev sizes, media encoding, prefix reuse, +CUDA Graphs, and additional kernel optimization are outside this baseline. + +Run commands from the repository root. Prepare the CUDA-merged checkpoint first +using the export recipe; old CPU-only exports are refused. Install the pinned +[reference requirements](reference-requirements.txt) for build-time Torch headers. +Build the backend for the actual GPU's +compute capability; the build itself does not use a GPU. The acceptance below +used the `89` build (including its PTX); see the hardware note before generalizing: + +```sh +KEV_PYTHON=.venv-kev-reference/bin/python recipe/kev/build_cuda.sh target/release 89 +cargo build --release --locked -p omni-kev-native -p omni-jev +# On hosts requiring reservations, keep gpu run outside the worker process. +gpu run --gpus 1 --nonblock --timeout 2h --note 'Kev eager worker' -- \ + env KEV_MODEL="$PWD/weights/kev-native" \ + KEV_CUDA_LIB="$PWD/target/release/libqwen3_5_cuda.so" \ + KEV_PORT=8000 target/release/omni-kev-native +``` + +`KEV_HOST` defaults to `127.0.0.1`; `KEV_PORT` defaults to 8000. The library defaults +to `libqwen3_5_cuda.so` beside the executable. Startup verifies the pinned manifest +and every exported file, loads the model, and runs a real request containing all +three question types. Only then does it bind the listener. `/health` reports +`ready`, the checkpoint ID, eager execution, and disabled prefix reuse. + +In a second terminal, start the model-independent frontend: + +```sh +OMNI_JEV_BIND=127.0.0.1:8080 OMNI_JEV_BACKEND_URL=http://127.0.0.1:8000 \ + target/release/omni-jev +curl -fsS http://127.0.0.1:8080/health +curl -fsS http://127.0.0.1:8080/v1/systemone \ + -H 'Content-Type: application/json' -d '{ + "model":"kev-4b", "state":"The light is green. The door is open.", + "questions":{ + "color":{"type":"choice","instructions":"What color is the light?", + "criteria":{"red":null,"green":null,"blue":null}}, + "open":{"type":"noul","instructions":"The door is open."}, + "safety":{"type":"score","instructions":"How safe is entering?", + "criteria":["unsafe","uncertain","safe"]} + }}' +``` + +The worker accepts `jaredpalmer/kev-4b`, `kev-4b`, and `kev-latest` (default), and +returns the requested alias. It preserves question/candidate insertion order, +rounds probabilities/derived scalars to four decimals, and counts the serialized +answers for `output_tokens`, as the pinned reference does. `input_tokens` counts +the state once plus every question branch, even though this eager baseline +recomputes state for each question. It generates no text tokens. + +Limits: 4 MiB request body; 64 questions; 1–255 Choice candidates or Score levels; +8192 state tokens including its marker; 16384 tokens per complete question row; +65536 summed computation-row tokens per request; 16 admitted requests. A request +is validated completely before GPU dispatch. There is no silent truncation. +These context/request budgets are deliberately below the official server's 64k +state allowance. JSON state/instructions/descriptions use the official text +rendering, including booleans, numbers, null, arrays, and objects. Media URLs or +objects are rendered as text; this worker has no image/audio/video encoder. + +Malformed JSON (including duplicate keys) returns 400; oversized bodies 413; +non-JSON content types 415; invalid/unsupported requests 422; capacity exhaustion +503 with `Retry-After: 1`; execution failure 500 without internal details. Unknown +fields are ignored as in the reference. Unlike the official server, arbitrary +model names are refused and timing fields are omitted. + +The optional CUDA build emits `PYTORCH-LICENSE`, `CUTLASS-LICENSE` and +`KEV_CUDA_NOTICES.txt` beside the library. Keep them with redistributed binaries; +see the [third-party notices](../../src/models/qwen3_5/native/THIRD_PARTY_NOTICES.md). + +## Primary acceptance status + +**All frozen acceptance gates pass in the author-run campaign; review pending.** The fixed +BF16 default-service comparison over 5,484 requests / 5,662 questions has maximum probability +error 0.0024709404 (gate 0.01), maximum Score error 0.00000305218 levels (gate 0.1), +and no decision disagreements. Noul boundary coverage is 36/10; all 36 qualify +again on a fresh service. The 896-task quality gates, 55 HTTP response comparisons +and 8 error checks pass. See the [boundary report](noul-boundary.md), +[causal diagnosis and repair](numerical-diagnosis.md) and +[unchanged acceptance protocol](service-acceptance.md). + +The processor follows the effective tokenizer loaded by the pinned Transformers +5.17 Qwen2Tokenizer, including its split regex for Unicode marks. Merely loading +the base tokenizer JSON yields different tokens for some emoji sequences. + +## Reproduce diagnostic correctness checks + +Ordinary checks require no checkpoint or GPU: + +```sh +cargo test --locked -p omni-kev-native -- --list +cargo test --locked -p omni-kev-native +python -m unittest discover -s tests/kev -p 'test_*.py' -v +``` + +Pinned tokenizer and released-head checks are CPU-only: + +```sh +KEV_EXPORT="$PWD/weights/kev-native" cargo test --locked -p omni-kev-native \ + --test processing -- --ignored --nocapture +KEV_HEAD="$PWD/weights/kev-native/head.safetensors" cargo test --release --locked \ + -p omni-kev-native --test head -- --ignored --nocapture +``` + +Create a separate Python 3.12 reference environment with +[`reference-requirements.txt`](reference-requirements.txt). Keep the CPU export +environment separate if it uses a CPU-only Torch wheel. The requirements pin +`torch==2.8.0+cu128` and include its CUDA wheel index; install them directly: + +```sh +python3.12 -m venv .venv-kev-reference +.venv-kev-reference/bin/pip install -r recipe/kev/reference-requirements.txt +``` + +Check out +`jaredpalmer/kev@2074cc44256200bb37540103fa4572901faa10f3` as `kev-reference`. +The fixture generator verifies the official source hashes before importing it: + +```sh +.venv-kev-reference/bin/python tests/kev/generate_processing_fixture.py \ + --reference kev-reference --base weights/kev-base --out /tmp/kev-processing.json +cmp tests/kev/data/processing.json /tmp/kev-processing.json +``` + +Reserve at least 40 GiB free host RAM, 24 GiB device memory and space for the base, +merged export, reference environment, and compiled artifacts. The FP32 unmerged +reference is larger than the BF16 worker. Use a new result directory for each run; +preserve failures and the exact source diff. This is a correctness campaign, +not a latency/throughput benchmark. + +```sh +gpu run --gpus 1 --nonblock --timeout 2h --note 'Kev reference parity' -- \ + .venv-kev-reference/bin/python recipe/kev/validate.py reference \ + --reference kev-reference --base weights/kev-base --adapter weights/kev-adapter \ + --export weights/kev-native --out /tmp/kev-run-01 + +gpu run --gpus 1 --nonblock --timeout 2h --note 'Kev native parity' -- \ + env KEV_EXPORT="$PWD/weights/kev-native" \ + KEV_CUDA_LIB="$PWD/target/release/libqwen3_5_cuda.so" \ + KEV_NATIVE_OUT=/tmp/kev-run-01/native.json \ + cargo test --release --locked -p omni-kev-native --lib \ + native_corpus_capture -- --ignored --nocapture +python recipe/kev/validate.py compare --out /tmp/kev-run-01 + +gpu run --gpus 1 --nonblock --timeout 2h --note 'Kev direct/frontend parity' -- \ + python recipe/kev/validate_http.py \ + --worker-bin "$PWD/target/release/omni-kev-native" \ + --frontend-bin "$PWD/target/release/omni-jev" \ + --export "$PWD/weights/kev-native" \ + --cuda-lib "$PWD/target/release/libqwen3_5_cuda.so" \ + --native /tmp/kev-run-01/native.json --out /tmp/kev-http-01 +``` + +The comparator first isolates pure FP32 PEFT merge error, then BF16 export/compute +drift. It also reports official unmerged → PyTorch merged, PyTorch +merged → native, and official unmerged → native. The corpus covers 1/2/7/31/255 +candidates, 1/3/255 Score levels, mixed question types, Unicode/control-token +spellings, structured values, reordering, and three additional repeated mixed +requests. HTTP acceptance compares complete direct/frontend/native responses, +checks the first external request after warmup, and checks error status/body. + +Fixed gates: exact rendering/tokens/readout positions; isolated head logit error +`<= 1e-4 + 1e-5*abs(reference)` and probability error `<= 1e-5`; full-model maximum +probability absolute error `<= 0.01`; matching Choice decisions when the reference +top-two margin is `> 0.02`; matching Noul decisions at threshold 0.5 when the +reference is farther than 0.01 from it. Every near-boundary disagreement is listed. +A failed merge baseline is reported separately and never justifies widening a gate. + +## Historical FP32 eager diagnostics + +Local author-run at source `57e239b681f84d56609ed7cef45889a115381172`, over 13 +requests / 23 questions (10 Choice, 7 Noul, 6 Score): + +| Comparison | Maximum probability absolute error | +| --- | ---: | +| Official FP32 unmerged → FP32 merged | 0.00000298023 | +| FP32 merged → exported BF16 PyTorch | 0.00543183 | +| Official FP32 unmerged → exported BF16 PyTorch | 0.00543272 | +| Exported BF16 PyTorch → native | 0.00607389 | +| Official FP32 unmerged → native | 0.00553775 | + +The original small-corpus probability/decision gates passed. This historical result does not qualify the BF16 default service. All 10 Choice and 7 Noul decisions +agree; three additional native mixed requests are identical. There were no +near-boundary disagreements; this corpus's smallest nontrivial Choice margin is +0.121829 and smallest Noul threshold distance is 0.023158, so it does not establish +real near-boundary coverage. Synthetic checker tests verify boundary reporting. + +For the 255-level Score, the expected level is 96.7645 officially versus 96.6496 +natively (absolute difference 0.1149 on a 0–254 scale). Its maximum probability +error is 0.000169017. That historical protocol had no scalar Score threshold. The current primary BF16 +service gate is 0.1 ordinal level, following merged Decider #110. Other datasets and longer contexts need their own +qualification; this small deterministic corpus is not a model-quality benchmark. + +The exact processing/head CPU gates passed. Workspace tests: 397 passed, 91 +ignored, zero failed; relevant ignored tokenizer, released-head and native +capture checks were explicitly run. Real warmup and the first external request +passed. Sixteen paired HTTP requests matched direct worker, frontend and native +capture exactly; eight error cases matched status/body. No performance claim is +made from these checks. + +The lab device is labelled NVIDIA L20X: the reserved Torch runtime reports +capability 9.0 while `nvidia-smi` reports 8.9. The tested backend was built with +CUDA 13.0 for sm_89 plus compute_89 PTX; its SHA-256 is +`9a29aae12a35368481f61ccbe9ecf318230b658224ee1d46211bbe2a30703c71`. +Reference Torch uses CUDA 12.8. Do not infer another hardware model from the label +or claim the untested sm_90 build passed. Exact package pins are above; raw JSON, +commands, hashes, and the retained outer-wrapper interruption are supplied in +the local handoff archive. No public artifact upload was performed. diff --git a/recipe/kev/noul-boundary.md b/recipe/kev/noul-boundary.md new file mode 100644 index 00000000..7eea052f --- /dev/null +++ b/recipe/kev/noul-boundary.md @@ -0,0 +1,136 @@ +# Kev Noul boundary qualification (v4) + +The fixed official BF16 default-service acceptance now passes all frozen gates: +**5,484 requests / 5,662 questions**, including **36 Noul** and **28 Choice** +near-boundary questions (minimum 10 each). Every observed decision agrees. +The [v3 numerical repair](numerical-diagnosis.md) and +[v2 protocol and failures](service-acceptance.md) remain available. This is author-run +qualification evidence; contributor/maintainer review remains pending. + +## Evidence availability after environment recovery + +On 2026-10-31 the previous `/tmp/system1-issue-27` directory was unavailable. +The v1-v5 raw captures, layer dumps and local handoff archive referenced below +have not been recovered. The committed reports and fixed fixtures preserve the +recorded results and gates, but are not a replacement for those raw artifacts. +The [review follow-up in #197](https://github.com/InferMatrixLab/system1-omni/pull/197) +distinguishes fresh checks after the main sync from the historical campaign. +Full-checkpoint BF16/HTTP and prefix-cache requalification remain unverified at +that updated head; the old measurements must not be labeled as fresh passes. + +## Discovery frozen before inference + +The additional pool reads only the pinned official repository's 29 +`development.jsonl` files, in sorted path and original row/question order. +`tests/kev/data/boundary-sources.json` pins every source hash. +`tests/kev/boundary_discovery.py` selects clean Noul questions, one per request, +without consulting model outputs or native correctness. It removes prior and +repeated source-record/question identities and identical actual token rows. +Public dataset IDs containing `test` describe source provenance; no locked Kev +evaluation partition was read. + +Before inference, the campaign limited state and branch to 1,024 tokens each. +The exclusion ledger retains 6,607 prior/repeated source identities, 309 length +exclusions and 120 duplicate token rows. The remaining **2,269** requests are +all retained and compared, including those far from the decision threshold. +The length budget limits this discovery claim; it does not qualify longer inputs. + +Run the unchanged default service first: official `2074cc4`, base `1001bb4`, +adapter/head `139fdd9`, temperature `2.406050072164233`, Torch 2.8.0+cu128, +Transformers 5.17.0 and PEFT 0.21.0. The backbone is BF16, head FP32, LoRA merged, +SDPA/Graph/prefix cache defaults enabled, optional FLA/causal-conv1d absent. +Save actual unrounded probabilities and freeze membership before native capture. +The criterion remains **`abs(p(true) - 0.5) <= 0.01`**. + +This yields **33 new** qualifying Noul questions, plus the original 3. All 36 +have distinct source-record/question identities and token rows. None was selected +using native agreement, and none was dropped after comparison. The new discovery +population does not change the original 896-question labeled task population. + +## Results under unchanged gates + +| Check | Result | Gate | +| --- | ---: | ---: | +| Aggregate probability max absolute error | 0.0024709404 | <=0.01 | +| New 2,269-question probability max error | 0.00000059604645 | <=0.01 | +| All 36 near-boundary Noul probability max error | 0.00000047683716 | <=0.01 | +| Score raw maximum absolute error | 0.00000305218 levels | <=0.1 levels | +| Score returned-scalar maximum error | 0 | <=0.1 levels | +| Choice / Noul boundary coverage | 28 / 36 | >=10 each | +| Far-boundary decision disagreements | 0 | 0 | +| Near-boundary raw / wire disagreements | 0 / 0 | report every item | +| Original labeled task accuracy, official / native | 76.5625% / 76.5625% | aggregate drop <=1 pp, per type <=2 pp | + +Brier increase remains `8.76751e-9`; normalized Score MAE change remains +`-2.57951e-9`. All original quality gates and the paired record bootstrap remain +unchanged. This is qualification on a fixed population, not a general model +quality or statistical noninferiority claim. + +The closest Noul question is `flakeflagger/hector-client-hector/` +`me.prettyprint.hector.api.ApiV2SystemTest#testSuperCountQuery`, question `flaky`: +official `p(true)=0.5000787973403931`, distance `0.00007879734`. +The complete 36-question table is retained in the local handoff's +`evidence/noul-boundary-v4/boundary-cases.md`; raw, unrounded values remain in JSON. + +## Independent replay and HTTP + +A fresh official service and native process replay all 36 qualifying Noul cases +plus 16 contract/repeat requests: 52 requests / 75 questions. **36/36 still +qualify**, with zero probability drift within each implementation between the +first and repeat captures. All near-boundary decisions agree again. The replay +records 13 Graph captures, 37 replays, 6 prefix-cache hits and zero capture failures; +no official defaults were disabled. + +Real worker warmup, the first mixed request after readiness, **55** +direct-worker/frontend/native response comparisons, and **8** error-response +comparisons pass. This HTTP set covers every qualified Noul case, not only ten. +All child services terminate, and all GPU execution uses `gpu run`. + +## Reproduction and preserved evidence + +The local handoff contains `reproduce-bf16.sh`, `protocol.md`, source/corpus hashes, +all requests and responses, official-only membership, native captures, +`comparison-all.json`, `comparison-replay.json`, the complete boundary table and +checksummed artifact manifest. Run the script from the handoff with a fresh output +directory under `gpu run --gpus 1 --wait 2h --timeout 2h --note 'Kev BF16 reproduction' --`. +The discovery generator and repeatable supplement comparator are in `tests/kev/`. + +Production source, exported checkpoint, worker/capture binaries and CUDA library +are unchanged from v3. The 399 Rust tests, independent head checks and 83 CUDA +operator comparisons from v3 remain applicable; this round reruns 22 Python +checks, Ruff, strict documentation checks, new native captures and HTTP validation. +The comparator also reproduces the old v3 report exactly, including its failed +Noul coverage gate. The v2 probability/Score failures, v3 intermediate failures, +FP32 controls and unsuccessful earlier boundary sweep are preserved. + +Other model sizes, optional FLA setups, arbitrary concurrent official batches, +other GPU architectures, cross-request native prefix reuse and performance remain +outside this evidence. The previous lab L20X capability-reporting caveat remains. + +## Shared-backend precheck (v5) + +A fresh ordinary CUDA build and a fresh Kev-profile build each pass 27 existing +Qwen kernel tests, 3 shared-prefix tests, 3 selected-readout tests and both Graph +switch settings. Full-checkpoint shared-engine tests use the same pinned merged +Kev weights with the default Qwen execution profile, including packed projection +options, prefix lifetime/reset, Graph replay and readout invariance. These are +shared-engine regressions, not Cua-S1/Open-Jev/JEMM task-accuracy results. + +All 85 previously exported `cs1_*` symbols remain present. The legacy/default +entry points and the new Torch 2.8 profile pass 99 operator comparisons, including +5120-wide reference norms used by JEMM: zero differing elements under the +unchanged assertions. Ordinary builds need no PyTorch/CUTLASS headers. The +optional build emits redistribution notices alongside its library and has no +Python/libtorch runtime dependency. + +The rebuilt Kev library reproduces the complete 52-request boundary/contract +capture byte for byte and passes all 55 HTTP comparisons and 8 error checks. +The 2,269 new discovery requests also pass exact CPU prompt/token/readout checks. +Fresh workspace validation passes 399 Rust tests (92 opt-in tests ignored by +that ordinary command), 22 Python tests, fmt, Clippy, release and strict docs. +Relevant GPU and released-head checks are separately executed. The independent +export/head slice passes its 7 Python tests, Rust/head checks and strict docs. +No functional model/kernel change was needed during this precheck, so the full +5,662-question result above remains the frozen v4 evidence, not a newly rerun +whole-corpus campaign. Raw v5 logs and source/binary hashes remain in the local +handoff under `evidence/precheck-v5/`. diff --git a/recipe/kev/numerical-diagnosis.md b/recipe/kev/numerical-diagnosis.md new file mode 100644 index 00000000..3deb0d71 --- /dev/null +++ b/recipe/kev/numerical-diagnosis.md @@ -0,0 +1,110 @@ +# Kev BF16 service parity repair (v3) + +This records the v3 repair and its then-incomplete coverage. The subsequent +[v4 qualification](noul-boundary.md) adds 33 Noul boundary questions and passes +all frozen gates without changing production code or discarding these results. + +The same 3,215 requests / 3,393 questions now pass the probability, Score and +far-boundary decision gates. Maximum probability error is **0.0024709404 <=0.01** +(`choice-255`); maximum raw Score error is **0.00000305218 <=0.1 level** +(`score-255`), with zero returned-Score difference. All observed decisions, +including the 31 near-boundary questions, agree. The 896 labeled questions have +identical 76.5625% accuracy; all frozen task-quality gates pass. + +**At the end of v3, acceptance remained incomplete:** the fixed pool still contains only +3 near-boundary Noul questions against the requirement of 10. Choice has 28. +The [v2 protocol and failures](service-acceptance.md) are retained. No gate, +reference output, request, candidate order or label changed during this repair. +No additional boundary search or performance campaign was run. + +## Where the error starts + +The diagnostic starts from the original worst probability case +`hard-v1:hard-v1/probability/development/00023` and `score-255`. +Official observers retain the default BF16 service's real state/branch passes, +padding, cache and graph behavior. An additional complete-row forward isolates +arithmetic from segmentation; it never replaces the primary service reference. + +| Isolated difference | Evidence and correction | +| --- | --- | +| GDN accumulation | Same-input projections and convolution matched. The original BF16 GDN core differed from the official no-FLA FP32 core (relative RMS about 0.002). Use FP32 normalized Q/K, solve and recurrent state, then round its output to BF16. | +| Framework reduction order | The existing FP32 path followed Torch 2.14. Torch 2.8 uses different lane assignments at dimension 128 and ascending shuffle offsets. Match its RMSNorm, gated norm, Q/K norm and GDN normalization with optional symbols; preserve existing consumers' arithmetic. | +| Projection shape | A fixed 64-row readout GEMM selected different arithmetic from the reference's actual row count. Use the reference GEMM profile. Multi-question requests additionally require the official length groups, padding, batch count and row offset. | +| Rotary buffer precision | Official checkpoint loading casts the backbone, including `inv_freq`, to BF16. Rounding this buffer before constructing angles makes the first ten complete-row layers exact. Native's prior FP32 frequencies encoded a different model computation. | +| Export device | CPU and official GPU FP32 LoRA merge differ in nine BF16 scalar weights across nine of 426 tensors. The next divergence begins at layer 10. GPU FP32 merge of the same pinned inputs eliminates all 426 tensor differences. This is precision alignment, not a new quantization scheme. | +| State/branch attention | A complete causal row and official padded state/continuation passes select different SDPA kernels and GDN chunks. Execute those segments eagerly, with temporary per-call state, and instantiate the exact Torch 2.8 CUTLASS efficient-attention kernel for the masked path. | + +With corrected export and arithmetic, both original worst samples have **exact +embeddings, all 32 residual layer outputs and final normalization** against the +corresponding official default-service state/question passes. Their probability +errors are 1.19209e-7 and 1.02445e-8 after the separate FP32 pointer head. + +An intermediate full regression still found six probability failures, max +0.01862866. All were multi-question requests. Equalizing question padding alone +fixed two. In the remaining worst request, every layer-0 projection, GDN output +and gated norm matched exactly; the output projection was the first difference +(341 BF16 elements, max 0.00048828125). Official GEMM had 160 rows; native had 80. +Keeping the official matrix shape and each row's position fixes all six, max +probability difference 9.23872e-7. Other rows of the temporary projection matrix +are zero: matrix rows are independent, and only the current row's result is read. +This conservative baseline spends extra computation; it is not an optimization. + +## Implementation and build boundary + +`planning.rs` belongs to Kev: it reproduces official question length grouping, +padding and mapping. The shared Qwen executor provides an opt-in segmented +selected-position readout. A call computes state then its question, retains +attention KV / GDN state only until that question finishes, and synchronizes +before freeing temporary allocations on success or failure. There is no retained +cross-request prefix and no native CUDA Graph. The scheduler and HTTP API remain +unchanged. The trained FP32 head and calibration remain unchanged. + +The optional profile requires `recipe/kev/build_cuda.sh`. It compiles a +header-only instance of PyTorch 2.8.0's BSD-licensed attention kernel with the +exact CUTLASS submodule `ad7b2f5e84fcfa124cb02b91d5bd26d238c0459e` (BSD-3-Clause). +The archive SHA-256 is checked before extraction. Torch headers are a build-time +dependency; the worker and CUDA library have no libtorch/Python runtime dependency. +The ordinary Qwen build still works without these headers. Missing service +symbols fail at Kev startup, rather than silently selecting another precision. + +The worker now requires the CUDA FP32 merge metadata. A CPU diagnostic export is +still available for head tests but is rejected for serving. Re-export to a new +directory; do not overwrite historical artifacts. See [export](README.md) and +[worker](native.md) commands. The original export/head review boundary is retained; +the CUDA export correction can be reviewed separately from worker execution. + +## Validation and limits + +- All 3,215 requests assert exact token IDs, positions, readouts and input usage. + Three repeated mixed responses remain identical. +- Real startup/warmup and 32 direct-worker/frontend/native responses match exactly; + eight malformed/unsupported-input HTTP cases match. +- 83 operator checks pass exactly: Torch 2.8 norms, GDN chunk boundaries and seeded + state, masked SDPA including non-aligned keys, and old/new legacy CUDA entry + points. The GDN test retains its 1e-4 relative-L2 gate; observed error is zero. +- The labeled population remains 336 Choice, 280 Noul, 280 Score. Brier delta is + +8.76751e-9 and normalized Score MAE delta -2.57951e-9. Paired record-bootstrap + intervals and per-source metrics remain in the machine-readable report. +- Existing independent head gates and unmerged/merged FP32 diagnostics remain + separate. No claim is made that arbitrary optional FLA installations, GPU + architectures or concurrent official batching are qualified by this run. +- The pinned official server can differ between cold eager fallback and a warmed + state cache. Native uses deterministic state buckets for short states; the + fixed original request order remains the primary comparator. The residual + `choice-255` error is below the unchanged gate; complete bitwise identity for + every possible serving history is not claimed. + +Local evidence root: `/tmp/system1-issue-27/evidence/layer-diagnosis-v3/`. +`comparison-25.json` is the complete fixed-corpus result; the intermediate +`comparison-22.json`, original v2 reports, failed probes and all layer tensors +are preserved. `layer-comparison-service-20.json` has all 32-layer comparisons; +`service-kernels-27.json` has operator/legacy regression results. The handoff +records final source/build hashes and verification commands. + +The retained FP32 unmerged control is a different computation from the official +BF16 default service. Comparing the corrected native outputs with that historical +13-case control gives probability max 0.00680214 and Score-255 difference +0.28167720 levels (FP32 about 96.76453 versus official BF16 about 96.48286). +This is separately recorded in `fp32-controls-v3.json`; the unchanged 0.1 Score +gate applies to the primary official BF16 service. No FP32 results are relabeled +as BF16 acceptance, and the original 0.1149 diagnostic remains preserved. diff --git a/recipe/kev/pins.json b/recipe/kev/pins.json new file mode 100644 index 00000000..2272c0f8 --- /dev/null +++ b/recipe/kev/pins.json @@ -0,0 +1,290 @@ +{ + "format": "kev-4b-merged/1", + "model_id": "jaredpalmer/kev-4b", + "revision": "139fdd94f1b6a6ad80cc15e08fcb99cac885a101", + "base_id": "Qwen/Qwen3.5-4B-Base", + "base_revision": "1001bb4d826a52d1f399e183466143f4da7b741b", + "reference_revision": "2074cc44256200bb37540103fa4572901faa10f3", + "backbone_dtype": "bfloat16", + "head_dtype": "float32", + "hidden_size": 2560, + "pointer_size": 256, + "temperature": 2.406050072164233, + "special_tokens": { + "<|fim_prefix|>": 248060, + "<|fim_middle|>": 248061, + "<|box_start|>": 248049, + "<|box_end|>": 248050, + "<|fim_suffix|>": 248062 + }, + "merged_targets": [ + "model.language_model.layers.0.linear_attn.in_proj_a.weight", + "model.language_model.layers.0.linear_attn.in_proj_b.weight", + "model.language_model.layers.0.linear_attn.in_proj_qkv.weight", + "model.language_model.layers.0.linear_attn.in_proj_z.weight", + "model.language_model.layers.0.linear_attn.out_proj.weight", + "model.language_model.layers.0.mlp.down_proj.weight", + "model.language_model.layers.0.mlp.gate_proj.weight", + "model.language_model.layers.0.mlp.up_proj.weight", + "model.language_model.layers.1.linear_attn.in_proj_a.weight", + "model.language_model.layers.1.linear_attn.in_proj_b.weight", + "model.language_model.layers.1.linear_attn.in_proj_qkv.weight", + "model.language_model.layers.1.linear_attn.in_proj_z.weight", + "model.language_model.layers.1.linear_attn.out_proj.weight", + "model.language_model.layers.1.mlp.down_proj.weight", + "model.language_model.layers.1.mlp.gate_proj.weight", + "model.language_model.layers.1.mlp.up_proj.weight", + "model.language_model.layers.10.linear_attn.in_proj_a.weight", + "model.language_model.layers.10.linear_attn.in_proj_b.weight", + "model.language_model.layers.10.linear_attn.in_proj_qkv.weight", + "model.language_model.layers.10.linear_attn.in_proj_z.weight", + "model.language_model.layers.10.linear_attn.out_proj.weight", + "model.language_model.layers.10.mlp.down_proj.weight", + "model.language_model.layers.10.mlp.gate_proj.weight", + "model.language_model.layers.10.mlp.up_proj.weight", + "model.language_model.layers.11.mlp.down_proj.weight", + "model.language_model.layers.11.mlp.gate_proj.weight", + "model.language_model.layers.11.mlp.up_proj.weight", + "model.language_model.layers.11.self_attn.k_proj.weight", + "model.language_model.layers.11.self_attn.o_proj.weight", + "model.language_model.layers.11.self_attn.q_proj.weight", + "model.language_model.layers.11.self_attn.v_proj.weight", + "model.language_model.layers.12.linear_attn.in_proj_a.weight", + "model.language_model.layers.12.linear_attn.in_proj_b.weight", + "model.language_model.layers.12.linear_attn.in_proj_qkv.weight", + "model.language_model.layers.12.linear_attn.in_proj_z.weight", + "model.language_model.layers.12.linear_attn.out_proj.weight", + "model.language_model.layers.12.mlp.down_proj.weight", + "model.language_model.layers.12.mlp.gate_proj.weight", + "model.language_model.layers.12.mlp.up_proj.weight", + "model.language_model.layers.13.linear_attn.in_proj_a.weight", + "model.language_model.layers.13.linear_attn.in_proj_b.weight", + "model.language_model.layers.13.linear_attn.in_proj_qkv.weight", + "model.language_model.layers.13.linear_attn.in_proj_z.weight", + "model.language_model.layers.13.linear_attn.out_proj.weight", + "model.language_model.layers.13.mlp.down_proj.weight", + "model.language_model.layers.13.mlp.gate_proj.weight", + "model.language_model.layers.13.mlp.up_proj.weight", + "model.language_model.layers.14.linear_attn.in_proj_a.weight", + "model.language_model.layers.14.linear_attn.in_proj_b.weight", + "model.language_model.layers.14.linear_attn.in_proj_qkv.weight", + "model.language_model.layers.14.linear_attn.in_proj_z.weight", + "model.language_model.layers.14.linear_attn.out_proj.weight", + "model.language_model.layers.14.mlp.down_proj.weight", + "model.language_model.layers.14.mlp.gate_proj.weight", + "model.language_model.layers.14.mlp.up_proj.weight", + "model.language_model.layers.15.mlp.down_proj.weight", + "model.language_model.layers.15.mlp.gate_proj.weight", + "model.language_model.layers.15.mlp.up_proj.weight", + "model.language_model.layers.15.self_attn.k_proj.weight", + "model.language_model.layers.15.self_attn.o_proj.weight", + "model.language_model.layers.15.self_attn.q_proj.weight", + "model.language_model.layers.15.self_attn.v_proj.weight", + "model.language_model.layers.16.linear_attn.in_proj_a.weight", + "model.language_model.layers.16.linear_attn.in_proj_b.weight", + "model.language_model.layers.16.linear_attn.in_proj_qkv.weight", + "model.language_model.layers.16.linear_attn.in_proj_z.weight", + "model.language_model.layers.16.linear_attn.out_proj.weight", + "model.language_model.layers.16.mlp.down_proj.weight", + "model.language_model.layers.16.mlp.gate_proj.weight", + "model.language_model.layers.16.mlp.up_proj.weight", + "model.language_model.layers.17.linear_attn.in_proj_a.weight", + "model.language_model.layers.17.linear_attn.in_proj_b.weight", + "model.language_model.layers.17.linear_attn.in_proj_qkv.weight", + "model.language_model.layers.17.linear_attn.in_proj_z.weight", + "model.language_model.layers.17.linear_attn.out_proj.weight", + "model.language_model.layers.17.mlp.down_proj.weight", + "model.language_model.layers.17.mlp.gate_proj.weight", + "model.language_model.layers.17.mlp.up_proj.weight", + "model.language_model.layers.18.linear_attn.in_proj_a.weight", + "model.language_model.layers.18.linear_attn.in_proj_b.weight", + "model.language_model.layers.18.linear_attn.in_proj_qkv.weight", + "model.language_model.layers.18.linear_attn.in_proj_z.weight", + "model.language_model.layers.18.linear_attn.out_proj.weight", + "model.language_model.layers.18.mlp.down_proj.weight", + "model.language_model.layers.18.mlp.gate_proj.weight", + "model.language_model.layers.18.mlp.up_proj.weight", + "model.language_model.layers.19.mlp.down_proj.weight", + "model.language_model.layers.19.mlp.gate_proj.weight", + "model.language_model.layers.19.mlp.up_proj.weight", + "model.language_model.layers.19.self_attn.k_proj.weight", + "model.language_model.layers.19.self_attn.o_proj.weight", + "model.language_model.layers.19.self_attn.q_proj.weight", + "model.language_model.layers.19.self_attn.v_proj.weight", + "model.language_model.layers.2.linear_attn.in_proj_a.weight", + "model.language_model.layers.2.linear_attn.in_proj_b.weight", + "model.language_model.layers.2.linear_attn.in_proj_qkv.weight", + "model.language_model.layers.2.linear_attn.in_proj_z.weight", + "model.language_model.layers.2.linear_attn.out_proj.weight", + "model.language_model.layers.2.mlp.down_proj.weight", + "model.language_model.layers.2.mlp.gate_proj.weight", + "model.language_model.layers.2.mlp.up_proj.weight", + "model.language_model.layers.20.linear_attn.in_proj_a.weight", + "model.language_model.layers.20.linear_attn.in_proj_b.weight", + "model.language_model.layers.20.linear_attn.in_proj_qkv.weight", + "model.language_model.layers.20.linear_attn.in_proj_z.weight", + "model.language_model.layers.20.linear_attn.out_proj.weight", + "model.language_model.layers.20.mlp.down_proj.weight", + "model.language_model.layers.20.mlp.gate_proj.weight", + "model.language_model.layers.20.mlp.up_proj.weight", + "model.language_model.layers.21.linear_attn.in_proj_a.weight", + "model.language_model.layers.21.linear_attn.in_proj_b.weight", + "model.language_model.layers.21.linear_attn.in_proj_qkv.weight", + "model.language_model.layers.21.linear_attn.in_proj_z.weight", + "model.language_model.layers.21.linear_attn.out_proj.weight", + "model.language_model.layers.21.mlp.down_proj.weight", + "model.language_model.layers.21.mlp.gate_proj.weight", + "model.language_model.layers.21.mlp.up_proj.weight", + "model.language_model.layers.22.linear_attn.in_proj_a.weight", + "model.language_model.layers.22.linear_attn.in_proj_b.weight", + "model.language_model.layers.22.linear_attn.in_proj_qkv.weight", + "model.language_model.layers.22.linear_attn.in_proj_z.weight", + "model.language_model.layers.22.linear_attn.out_proj.weight", + "model.language_model.layers.22.mlp.down_proj.weight", + "model.language_model.layers.22.mlp.gate_proj.weight", + "model.language_model.layers.22.mlp.up_proj.weight", + "model.language_model.layers.23.mlp.down_proj.weight", + "model.language_model.layers.23.mlp.gate_proj.weight", + "model.language_model.layers.23.mlp.up_proj.weight", + "model.language_model.layers.23.self_attn.k_proj.weight", + "model.language_model.layers.23.self_attn.o_proj.weight", + "model.language_model.layers.23.self_attn.q_proj.weight", + "model.language_model.layers.23.self_attn.v_proj.weight", + "model.language_model.layers.24.linear_attn.in_proj_a.weight", + "model.language_model.layers.24.linear_attn.in_proj_b.weight", + "model.language_model.layers.24.linear_attn.in_proj_qkv.weight", + "model.language_model.layers.24.linear_attn.in_proj_z.weight", + "model.language_model.layers.24.linear_attn.out_proj.weight", + "model.language_model.layers.24.mlp.down_proj.weight", + "model.language_model.layers.24.mlp.gate_proj.weight", + "model.language_model.layers.24.mlp.up_proj.weight", + "model.language_model.layers.25.linear_attn.in_proj_a.weight", + "model.language_model.layers.25.linear_attn.in_proj_b.weight", + "model.language_model.layers.25.linear_attn.in_proj_qkv.weight", + "model.language_model.layers.25.linear_attn.in_proj_z.weight", + "model.language_model.layers.25.linear_attn.out_proj.weight", + "model.language_model.layers.25.mlp.down_proj.weight", + "model.language_model.layers.25.mlp.gate_proj.weight", + "model.language_model.layers.25.mlp.up_proj.weight", + "model.language_model.layers.26.linear_attn.in_proj_a.weight", + "model.language_model.layers.26.linear_attn.in_proj_b.weight", + "model.language_model.layers.26.linear_attn.in_proj_qkv.weight", + "model.language_model.layers.26.linear_attn.in_proj_z.weight", + "model.language_model.layers.26.linear_attn.out_proj.weight", + "model.language_model.layers.26.mlp.down_proj.weight", + "model.language_model.layers.26.mlp.gate_proj.weight", + "model.language_model.layers.26.mlp.up_proj.weight", + "model.language_model.layers.27.mlp.down_proj.weight", + "model.language_model.layers.27.mlp.gate_proj.weight", + "model.language_model.layers.27.mlp.up_proj.weight", + "model.language_model.layers.27.self_attn.k_proj.weight", + "model.language_model.layers.27.self_attn.o_proj.weight", + "model.language_model.layers.27.self_attn.q_proj.weight", + "model.language_model.layers.27.self_attn.v_proj.weight", + "model.language_model.layers.28.linear_attn.in_proj_a.weight", + "model.language_model.layers.28.linear_attn.in_proj_b.weight", + "model.language_model.layers.28.linear_attn.in_proj_qkv.weight", + "model.language_model.layers.28.linear_attn.in_proj_z.weight", + "model.language_model.layers.28.linear_attn.out_proj.weight", + "model.language_model.layers.28.mlp.down_proj.weight", + "model.language_model.layers.28.mlp.gate_proj.weight", + "model.language_model.layers.28.mlp.up_proj.weight", + "model.language_model.layers.29.linear_attn.in_proj_a.weight", + "model.language_model.layers.29.linear_attn.in_proj_b.weight", + "model.language_model.layers.29.linear_attn.in_proj_qkv.weight", + "model.language_model.layers.29.linear_attn.in_proj_z.weight", + "model.language_model.layers.29.linear_attn.out_proj.weight", + "model.language_model.layers.29.mlp.down_proj.weight", + "model.language_model.layers.29.mlp.gate_proj.weight", + "model.language_model.layers.29.mlp.up_proj.weight", + "model.language_model.layers.3.mlp.down_proj.weight", + "model.language_model.layers.3.mlp.gate_proj.weight", + "model.language_model.layers.3.mlp.up_proj.weight", + "model.language_model.layers.3.self_attn.k_proj.weight", + "model.language_model.layers.3.self_attn.o_proj.weight", + "model.language_model.layers.3.self_attn.q_proj.weight", + "model.language_model.layers.3.self_attn.v_proj.weight", + "model.language_model.layers.30.linear_attn.in_proj_a.weight", + "model.language_model.layers.30.linear_attn.in_proj_b.weight", + "model.language_model.layers.30.linear_attn.in_proj_qkv.weight", + "model.language_model.layers.30.linear_attn.in_proj_z.weight", + "model.language_model.layers.30.linear_attn.out_proj.weight", + "model.language_model.layers.30.mlp.down_proj.weight", + "model.language_model.layers.30.mlp.gate_proj.weight", + "model.language_model.layers.30.mlp.up_proj.weight", + "model.language_model.layers.31.mlp.down_proj.weight", + "model.language_model.layers.31.mlp.gate_proj.weight", + "model.language_model.layers.31.mlp.up_proj.weight", + "model.language_model.layers.31.self_attn.k_proj.weight", + "model.language_model.layers.31.self_attn.o_proj.weight", + "model.language_model.layers.31.self_attn.q_proj.weight", + "model.language_model.layers.31.self_attn.v_proj.weight", + "model.language_model.layers.4.linear_attn.in_proj_a.weight", + "model.language_model.layers.4.linear_attn.in_proj_b.weight", + "model.language_model.layers.4.linear_attn.in_proj_qkv.weight", + "model.language_model.layers.4.linear_attn.in_proj_z.weight", + "model.language_model.layers.4.linear_attn.out_proj.weight", + "model.language_model.layers.4.mlp.down_proj.weight", + "model.language_model.layers.4.mlp.gate_proj.weight", + "model.language_model.layers.4.mlp.up_proj.weight", + "model.language_model.layers.5.linear_attn.in_proj_a.weight", + "model.language_model.layers.5.linear_attn.in_proj_b.weight", + "model.language_model.layers.5.linear_attn.in_proj_qkv.weight", + "model.language_model.layers.5.linear_attn.in_proj_z.weight", + "model.language_model.layers.5.linear_attn.out_proj.weight", + "model.language_model.layers.5.mlp.down_proj.weight", + "model.language_model.layers.5.mlp.gate_proj.weight", + "model.language_model.layers.5.mlp.up_proj.weight", + "model.language_model.layers.6.linear_attn.in_proj_a.weight", + "model.language_model.layers.6.linear_attn.in_proj_b.weight", + "model.language_model.layers.6.linear_attn.in_proj_qkv.weight", + "model.language_model.layers.6.linear_attn.in_proj_z.weight", + "model.language_model.layers.6.linear_attn.out_proj.weight", + "model.language_model.layers.6.mlp.down_proj.weight", + "model.language_model.layers.6.mlp.gate_proj.weight", + "model.language_model.layers.6.mlp.up_proj.weight", + "model.language_model.layers.7.mlp.down_proj.weight", + "model.language_model.layers.7.mlp.gate_proj.weight", + "model.language_model.layers.7.mlp.up_proj.weight", + "model.language_model.layers.7.self_attn.k_proj.weight", + "model.language_model.layers.7.self_attn.o_proj.weight", + "model.language_model.layers.7.self_attn.q_proj.weight", + "model.language_model.layers.7.self_attn.v_proj.weight", + "model.language_model.layers.8.linear_attn.in_proj_a.weight", + "model.language_model.layers.8.linear_attn.in_proj_b.weight", + "model.language_model.layers.8.linear_attn.in_proj_qkv.weight", + "model.language_model.layers.8.linear_attn.in_proj_z.weight", + "model.language_model.layers.8.linear_attn.out_proj.weight", + "model.language_model.layers.8.mlp.down_proj.weight", + "model.language_model.layers.8.mlp.gate_proj.weight", + "model.language_model.layers.8.mlp.up_proj.weight", + "model.language_model.layers.9.linear_attn.in_proj_a.weight", + "model.language_model.layers.9.linear_attn.in_proj_b.weight", + "model.language_model.layers.9.linear_attn.in_proj_qkv.weight", + "model.language_model.layers.9.linear_attn.in_proj_z.weight", + "model.language_model.layers.9.linear_attn.out_proj.weight", + "model.language_model.layers.9.mlp.down_proj.weight", + "model.language_model.layers.9.mlp.gate_proj.weight", + "model.language_model.layers.9.mlp.up_proj.weight" + ], + "inputs": { + "base": { + "config.json": "ddc63e1c717afa86c865bb5e01313d89d72bb53b97ad4a8a03ba8510c0621670", + "tokenizer.json": "fe000e3ed39ed12b8d2481d527d44f93c65d37e87645d2dcc80d1bf9d50d2927", + "tokenizer_config.json": "3891e840d7dc5fca0af33d3a25083a735e36fe06214e3f707024820cb6b9f89c", + "model.safetensors.index.json": "eae340074abb0a5f31a6621f7ae8e8248a7c1790df04a722c4e4b70c2a6d1dbb", + "model.safetensors-00001-of-00002.safetensors": "df547074dce70532a0493e5433152bd17a65efb89088cfabc2e7e2371a93d712", + "model.safetensors-00002-of-00002.safetensors": "590fbaac095dd31db886c322d9d2f7df47777966391acf306ddddc3e4e3a15ef" + }, + "checkpoint": { + "adapter_config.json": "8a05dfd6c5e7f61a62e6db5d8de8093a8dbbd105aa7ee7b5e7b73f6249f45617", + "adapter_model.safetensors": "90e817356246e7f18bfa7ca3d31794cd4fbeb3332a66a84cb51d9ceae925f2b2", + "head.pt": "dd633435998ecc751ac538717a3742e32149500fabf7d7276287dbf0693f347c" + } + }, + "merge_numerics": { + "device": "cuda", + "accumulation": "float32", + "torch": "2.8.0+cu128" + } +} diff --git a/recipe/kev/reference-requirements.txt b/recipe/kev/reference-requirements.txt new file mode 100644 index 00000000..daf9dbc9 --- /dev/null +++ b/recipe/kev/reference-requirements.txt @@ -0,0 +1,12 @@ +# CUDA parity environment; Python 3.12. The production worker has no Python dependency. +# Pin the measured CUDA wheel; a CPU or another CUDA build is not equivalent. +--extra-index-url https://download.pytorch.org/whl/cu128 +torch==2.8.0+cu128 +transformers==5.17.0 +peft==0.21.0 +accelerate==1.15.0 +pydantic==2.12.5 +safetensors==0.8.0 +numpy==2.5.4 +huggingface-hub==1.33.0 +tokenizers==0.23.1 diff --git a/recipe/kev/requirements.txt b/recipe/kev/requirements.txt new file mode 100644 index 00000000..ec97238b --- /dev/null +++ b/recipe/kev/requirements.txt @@ -0,0 +1,3 @@ +torch==2.8.0 +safetensors==0.8.0 +numpy==2.2.6 diff --git a/recipe/kev/service-acceptance.md b/recipe/kev/service-acceptance.md new file mode 100644 index 00000000..8ae398cd --- /dev/null +++ b/recipe/kev/service-acceptance.md @@ -0,0 +1,223 @@ +# Kev-4B default-service acceptance protocol v2 + +Declared before the first v2 model run. This supplements the original FP32 eager +diagnostics; it does not replace, discard or retroactively qualify those runs. +The recorded outcome and the separately frozen supplemental experiment follow +the original protocol below. **Current status: all frozen gates pass after the [v3 repair](numerical-diagnosis.md) +and [v4 boundary qualification](noul-boundary.md), with 36 Noul boundary questions. +The failed v2 outcomes below remain historical evidence.** + +## Primary reference and controls + +- Official Kev code: `2074cc44256200bb37540103fa4572901faa10f3`. +- Base: `1001bb4d826a52d1f399e183466143f4da7b741b`. +- Adapter/head: `139fdd94f1b6a6ad80cc15e08fcb99cac885a101`. +- Run the pinned `kev.serve.main` on CUDA with no `KEV_*` environment overrides: + BF16 backbone, FP32 pointer head, merged LoRA, saved temperature + `2.406050072164233`, SDPA, prefix cache and CUDA Graphs enabled by default. +- The fixed environment has Torch 2.8.0, Transformers 5.17.0, PEFT 0.21.0, + FastAPI 0.141.1, Starlette 1.6.0 and Uvicorn 0.53.0. Optional FLA and + causal-conv1d are absent. Consequently the official default does not enable + fused_qwen35. This qualifies that environment, not every optional-kernel setup. +- The observer calls the original server response method unchanged, saves its + already-computed unrounded probabilities, and counts original graph replays. + It does not replace loading, forward, precision, response formatting or kernels. +- Freeze the full package list, source hashes, command, effective runtime, + checkpoint hashes, native binary/library hashes, corpus and request order. + A failure to start the official default is a failed/unverified reference run; + do not silently disable Graphs or change its dtype to obtain a pass. +- Native remains the existing eager worker. Exact prompt/token/readout equality + is checked against official CPU preprocessing for every retained request. + +## Gates fixed before execution + +- Every unrounded candidate probability: maximum absolute error <= **0.01**. +- Choice: identical first-max decision when official top-two margin > **0.02**. +- Noul: identical `p(true) >= 0.5` decision when official distance from 0.5 > + **0.01**. Record all raw and response-rounded threshold disagreements. +- Score: maximum absolute difference <= **0.1 ordinal level**, for both the + unrounded expectation and the returned scalar, on every question. Single-level + scores must be zero. Report the raw error and error divided by `L-1` (zero for + `L=1`). No unapproved alternative normalized gate is introduced. +- Precedent: merged [Decider #110](https://github.com/InferMatrixLab/system1-omni/pull/110) + uses Score drift 0.1 on the same 0..L-1 ordinal scale. Its probability/winner + gates 0.02/0.05 do not replace Kev's stricter existing 0.01/0.02 gates. + [Open-Jev #102](https://github.com/InferMatrixLab/system1-omni/pull/102)'s 0.001 + checks a native optimization against native, not initial framework parity. + [Cua-S1 #19](https://github.com/InferMatrixLab/system1-omni/pull/19) has no Score + support. The earlier proposed 0.25-level gate is not adopted. +- All values must be finite, each distribution in [0,1] and sum within 1e-4 of + one; counts, IDs and candidate order must match. Compare unrounded values; + four-decimal wire rounding never hides a numerical failure. +- Boundary coverage: at least **10 actual model-scored Choice** questions with + margin <=0.02 and **10 actual model-scored Noul** questions with distance <=0.01. + Keep every qualifying item and every disagreement, including exact ties. + Distinguish public labeled examples from explicitly synthetic inputs. +- Task-quality gates on the fixed labeled population: observed aggregate + accuracy drop <=1 percentage point; per-type drop <=2 points; aggregate Brier + increase <=0.01; normalized Score MAE increase <=0.005. Also report NLL, raw + Score MAE, per-source results and record-clustered paired 95% bootstrap + intervals (2,000 resamples, seed 27). These are finite-population qualification + gates, not a general statistical noninferiority or model-quality claim. +- Direct worker and frontend responses must match exactly. Compare an explicit + subset of contract and official-selected boundary cases through HTTP; record + the subset. Input usage and response schema follow the official API. Timing + fields and numerically dependent output billing tokens are not equated across + implementations; each response's billing is checked against its own answers. +- Graph capture/replay and prefix cache hits must actually occur in the official + server; report captures, failures, replay count and cache stats. Native repeated + inputs must be identical. Official repetitions retain all observed variation. + +## Frozen corpus and selection + +The generator verifies development-file hashes before reading. It never opens +locked test partitions or changes labels, and makes no selection using model +correctness or native outputs. + +1. Original 13 contract requests, including 1/2/many/255 candidates and Score + 1/3/255; three additional mixed repeats. +2. Every `clean` development record of official `evals/v4/transfer-v4`: 656 + questions (336 Choice, 280 Noul, 40 Score), including six held-out public + sources and held-out synthetic policy structures. +3. Every Score question from `clean` development records of `evals/v4/decision-v4`: 240 + questions from SST5, Yelp and Amazon. These are in-distribution labeled + sentiment tasks; do not describe them as unseen-source evaluation. +4. Boundary discovery: every `hard_ambiguous` and `hard_probability` development + record of `hard-v1`, plus a declared 101-input lottery-probability sweep from + 45.0% to 55.0% with Choice and Noul questions. This last sweep is synthetic, + has no accuracy labels and is excluded from task-quality metrics. + +All rows are prepared and frozen before inference. All selected rows are run +through both implementations, whether or not they are close to a boundary. +Boundary membership is determined only from the official unrounded outputs, +then saved before native inference. No native disagreement can remove a row. +If the fixed pool produces too few boundary examples, report a coverage failure; +do not call distant or fabricated probability vectors near-boundary examples. + +## Run budget and evidence + +One official default-service pass and one native pass over the complete frozen +corpus, then the declared HTTP subset. Preserve all startup/harness/numerical +failures. A harness repair may resume only unexecuted work or repeat an affected +stage in a fresh directory with its reason recorded; never relax a threshold. +One additional diagnostic run is allowed for a concrete failure. All GPU child +processes remain inside `gpu run`, with a two-hour execution cap and at most the +previously authorized two-hour queue wait. Stop owned services in `finally`. + +The original `run-01` FP32-unmerged / FP32-merged / BF16-export / native evidence +and its unchanged original gates remain diagnostic evidence. The v2 Score gate +applies to the primary BF16 service comparison, not retroactively to FP32 drift. +Do not claim throughput or latency improvement from this correctness campaign. + +## Reproduction + +Commands run from the repository root; use the pinned reference environment and +the CUDA-merged export produced by the current export recipe. Install [service-requirements.txt](service-requirements.txt) +in a separate Python 3.12 environment. Build the native opt-in test before reserving +a GPU. Write every run to a fresh output directory. + +```sh +python tests/kev/service_acceptance.py prepare --reference /path/to/kev \ + --base /path/to/base --out /path/to/corpus.json +gpu run --gpus 1 --wait 2h --timeout 2h --note 'Kev BF16 service acceptance' -- \ + python tests/kev/service_acceptance.py reference --reference /path/to/kev \ + --adapter /path/to/adapter --corpus /path/to/corpus.json --out /path/to/service-run +``` + +The reference may use an offline Hub snapshot cache of the pinned base. Point +`HF_HUB_CACHE` to it and set `HF_HUB_OFFLINE=1`; this changes file lookup only. +No `KEV_*` overrides may be present in the official server's environment. + +```sh +# CPU-only full token/readout/billing check before device execution: +KEV_EXPORT=/path/to/export KEV_CORPUS=/path/to/corpus.json \ + cargo test --release --locked -p omni-kev-native --test processing \ + exact_tokens_readouts_and_billing_match_official -- --ignored --nocapture +gpu run --gpus 1 --wait 2h --timeout 2h --note 'Kev native service parity' -- \ + env KEV_EXPORT=/path/to/export KEV_CUDA_LIB=/path/to/libqwen3_5_cuda.so \ + KEV_CORPUS=/path/to/corpus.json KEV_NATIVE_OUT=/path/to/native.jsonl \ + cargo test --release --locked -p omni-kev-native --lib \ + native_service_capture -- --ignored --nocapture +python tests/kev/compare_service.py --corpus /path/to/corpus.json \ + --official /path/to/service-run/official-service.json --native /path/to/native.jsonl \ + --reference /path/to/kev --base /path/to/base --out /path/to/comparison.json +``` + +The comparator returns nonzero on a failed gate and retains the complete report. +For HTTP, use `recipe/kev/validate_http.py` as in the [worker recipe](native.md), +adding `--corpus /path/to/corpus.json` and +`--boundaries /path/to/service-run/boundary-membership.json`; `--native` accepts +the JSONL capture. It selects contract/repeat requests and the first ten official +boundary questions of each type, retaining the actual subset in `http.json`. + +## Recorded v2 result and preserved failures + +Historical results below remain unchanged. The [v3 report](numerical-diagnosis.md) records the repair under the same gates. + +The original pool has 1,213 requests / 1,391 questions. Its 896 labeled questions +are 336 Choice, 280 Noul and 280 Score. Initial execution found a real tokenization +bug at request 233 (`tweet_offensive/test/356`, a development-partition record): +the pinned Transformers Qwen2Tokenizer rebuilds the split regex, differing from +the base JSON on Unicode marks such as emoji variation selectors. The processor +now installs that effective regex. A maintained Unicode oracle and CPU comparison +of every corpus row verify the repair. The partial failed capture is preserved; +the official run is reused and the repaired native run uses a fresh output file. +Regenerate the Unicode oracle with `tests/kev/generate_processing_fixture.py` +and the original reference/base arguments, adding `--unicode-only`; compare the +output with `tests/kev/data/unicode-processing.json`. + +The first pool yielded 28 near-boundary Choice and 3 Noul, failing the ten-Noul +coverage gate. Before further inference, a supplemental diagnostic froze 2,002 +Noul inputs: two random-event templates, tickets and balls, each at 0.0..100.0% +in steps of 0.1%. All were evaluated, with no native-dependent selection. No +supplemental Noul falls within 0.01 of the threshold (closest: 0.02489048). +The coverage gap remains; these synthetic inputs are excluded from task accuracy. +Generate this fixed supplement with `boundary-supplement` in place of `prepare`. +Run the same reference/native commands with fresh paths; add +`--supplement-corpus`, `--supplement-official` and `--supplement-native` to the +comparator to retain both populations in one report. + +| Gate | Result | +| --- | --- | +| Exact tokens, positions, readouts, input/output usage | Pass for all 3,215 requests | +| Probability max absolute error <=0.01 | **Fail:** 0.06492195; 55 original + 84 supplemental questions fail | +| Score raw and wire absolute error <=0.1 level | **Fail:** 255-level raw error 0.16675543, wire error 0.1667 | +| Choice agreement above margin 0.02; Noul above distance 0.01 | Pass; all three decision flips are near-boundary | +| Ten actual near-boundary questions per type | Choice 28; **Noul 3/10, incomplete** | +| Fixed-population task-quality gates | Pass; figures below are not universal model-quality claims | +| Real warmup, direct worker/frontend/native equality | Pass, 32 paired requests and 8 HTTP error cases | + +Official Score-255 is 96.48285554, native 96.64961097 (range 0..254). The difference +normalized by 254 is 0.00065652; the absolute gate still fails. The earlier 0.1149 +was against the FP32 eager reference and is retained as precision diagnosis. + +| Labeled metric | Official BF16 service | Native | +| --- | ---: | ---: | +| Accuracy, all 896 questions | 76.5625% | 76.6741% | +| Choice accuracy, 336 | 81.25% | 81.25% | +| Noul accuracy, 280 | 83.9286% | 84.2857% | +| Score argmax accuracy, 280 | 63.5714% | 63.5714% | +| Brier, all | 0.30749647 | 0.30729482 | +| NLL, all | 0.54907485 | 0.54867828 | +| Raw Score MAE | 0.45892220 | 0.45831086 | +| Normalized Score MAE | 0.12694017 | 0.12674938 | + +The paired record-bootstrap accuracy difference interval is [-0.3223, +0.5470] +percentage points; normalized Score MAE difference interval is +[-0.00040033, +0.00001614]. These intervals do not establish general noninferiority. +Per-source results and all individual failures are retained in the local report. +The three raw/wire decision disagreements are: + +| Request / question | Official | Native | Official distance | +| --- | --- | --- | ---: | +| `tweet_offensive/test/80` / offensive | Noul 0.49786508, false | 0.50078470, true | 0.00213492 | +| `boundary-probe-060` / outcome | lose | win | top-two 0.00460693 | +| `boundary-probe-095` / outcome | lose | win | top-two 0.01417685 | + +Original/supplemental official passes captured 43/4 graphs and replayed 2,246/3,990 +times, with no capture failures; the original pass had 30 prefix-cache hits. +Native repeated mixed inputs are identical. The actual execution is BF16 backbone, +FP32 head, SDPA, no remaining LoRA modules and no optional fused Qwen kernels. +Production Qwen/CUDA math is unchanged. Numerical attribution beyond the existing +FP32/merged/BF16 diagnostics requires a separate investigation; neither task +accuracy nor the small normalized Score error overrides a failed parity gate. diff --git a/recipe/kev/service-requirements.txt b/recipe/kev/service-requirements.txt new file mode 100644 index 00000000..b0ebe444 --- /dev/null +++ b/recipe/kev/service-requirements.txt @@ -0,0 +1,4 @@ +-r reference-requirements.txt +fastapi==0.141.1 +starlette==1.6.0 +uvicorn==0.53.0 diff --git a/recipe/kev/validate.py b/recipe/kev/validate.py new file mode 100644 index 00000000..667501bc --- /dev/null +++ b/recipe/kev/validate.py @@ -0,0 +1,170 @@ +"""Pinned three-way Kev acceptance: reference, exported-weight baseline, native. + +GPU subcommands must be invoked through the execution host's reservation tool. +Never changes thresholds, chooses easier cases, or overwrites previous outputs. +""" +import argparse +import gc +import hashlib +import importlib.metadata +import importlib.util +import json +import math +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +spec = importlib.util.spec_from_file_location('kev_fixture', ROOT / 'tests/kev/generate_processing_fixture.py') +fixture = importlib.util.module_from_spec(spec) +spec.loader.exec_module(fixture) + + +def save(path, value): + path.write_text(json.dumps(value, ensure_ascii=False, indent=2) + '\n') + + +def reference(args): + import torch + from peft import PeftModel + from safetensors.torch import load_file + sys.path.insert(0, str(Path(__file__).parent)) + import export + export.verify_files(args.base, export.BASE_FILES) + export.verify_files(args.adapter, export.CHECKPOINT_FILES) + manifest = json.loads((args.export / 'kev_export.json').read_text()) + export.verify_files(args.export, manifest['files']) + api, model = fixture.official(args.reference) + tok = model.load_tokenizer(args.base) + torch.set_num_threads(8) + torch.backends.cuda.matmul.allow_tf32 = False + torch.backends.cudnn.allow_tf32 = False + torch.manual_seed(0) + meta = torch.load(args.adapter / 'head.pt', weights_only=True, map_location='cpu') + export.pointer_head(meta) + provenance = {'source_hashes': fixture.SOURCE_HASHES, + 'runtime_revision': subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=ROOT, text=True).strip(), + 'runtime_diff_sha256': hashlib.sha256(subprocess.check_output(['git', 'diff', 'HEAD'], cwd=ROOT)).hexdigest(), 'reference_revision': export.REFERENCE_REVISION, + 'base_revision': export.BASE_REVISION, 'adapter_revision': export.REVISION, + 'temperature': meta['temperature'], 'torch_cuda': torch.version.cuda, + 'device': torch.cuda.get_device_name(), 'capability': torch.cuda.get_device_capability(), + 'dependencies': {n: importlib.metadata.version(n) for n in + ('torch', 'transformers', 'peft', 'accelerate', 'pydantic', 'tokenizers', 'safetensors', 'numpy')}} + corpus = json.loads((ROOT / 'tests/kev/data/processing.json').read_text())['cases'] + m = None + for kind in ('official', 'fp32-merged', 'merged'): + path = args.out / f'{kind}.json' + if path.exists(): + raise ValueError(f'preserve previous evidence: {path} exists') + print(f'Loading {kind}', flush=True) + if kind == 'fp32-merged': + # Isolate PEFT merge arithmetic from BF16 export/compute precision. + assert m is not None, 'the unmerged capture must precede FP32 merge' + m.lm = m.lm.merge_and_unload(safe_merge=True) + else: + m = model.DecisionModel(str(args.base), tok, 'cuda', dtype=torch.float32 if kind == 'official' else torch.bfloat16, attn='eager') + if kind == 'official': + m.lm = PeftModel.from_pretrained(m.lm, str(args.adapter), torch_device='cuda').to('cuda') + elif kind == 'merged': + # Exact exporter values, including BF16 norms. No PEFT merge is rerun here. + weights = {} + for name in manifest['files']: + if name.startswith('model.') and name.endswith('.safetensors'): + weights.update({k.removeprefix('model.language_model.'): v for k, v in load_file(args.export / name).items()}) + m.lm.load_state_dict(weights, strict=True) + del weights + m.head.load_state_dict(meta['head']) + m.head.temperature = meta['temperature'] + m.eval() + results = [] + with torch.inference_mode(): + for case in corpus: + req = api.SystemOneRequest.model_validate(case['request']) + rec, mapping = api.to_record(req) + enc = model.encode(tok, rec, max_state=8192, max_branch=16384, strict=True) + assert len(enc['ids']) == case['input_tokens'] + logits = m(enc) # official independent complete causal rows, no prefix reuse + ps = [torch.softmax(z, -1).double().cpu().tolist() for z in logits] + answers = api.to_answers(ps, mapping) + results.append({'name': case['name'], 'probabilities': ps, + 'calibrated_logits': [z.double().cpu().tolist() for z in logits], + 'body': {'model': req.model, 'answers': answers, + 'usage': {'input_tokens': len(enc['ids']), 'output_tokens': api.output_tokens(tok, answers)}}}) + save(path, {'kind': kind, 'provenance': provenance, 'cases': results}) + print(kind, case['name'], 'questions', len(ps), flush=True) + if kind != 'official': + m = None + gc.collect() + torch.cuda.empty_cache() + + +def compare(args): + corpus = json.loads((ROOT / 'tests/kev/data/processing.json').read_text())['cases'] + outputs = {kind: json.loads((args.out / f'{kind}.json').read_text()) for kind in ('official', 'fp32-merged', 'merged', 'native')} + report = {'gates': {'probability_max_abs': 0.01, 'choice_margin': 0.02, 'noul_distance': 0.01}, + 'comparisons': {}} + for left, right in (('official', 'fp32-merged'), ('fp32-merged', 'merged'), ('official', 'merged'), ('merged', 'native'), ('official', 'native')): + records, failures, boundary = [], [], [] + a, b = outputs[left]['cases'], outputs[right]['cases'] + assert len(a) == len(corpus) and len(b) >= len(corpus), 'incomplete corpus' + for expected, ca, cb in zip(corpus, a, b): + assert expected['name'] == ca['name'] == cb['name'] + assert ca['body']['usage']['input_tokens'] == cb['body']['usage']['input_tokens'] == expected['input_tokens'] + assert len(ca['probabilities']) == len(cb['probabilities']) == len(expected['meta']) + for meta, pa, pb in zip(expected['meta'], ca['probabilities'], cb['probabilities']): + assert len(pa) == len(pb) == len(meta['keys']) + for probabilities in (pa, pb): + assert all(math.isfinite(v) and 0 <= v <= 1 for v in probabilities), 'invalid probabilities' + assert abs(sum(probabilities)-1) < 1e-4, 'unnormalized probabilities' + error = max(abs(x-y) for x, y in zip(pa, pb)) + row = {'case': expected['name'], 'question': meta['id'], 'type': meta['type'], 'probability_max_abs': error} + if error > 0.01: + failures.append({**row, 'gate': 'probability'}) + if meta['type'] == 'choice': + top = sorted(range(len(pa)), key=lambda i: -pa[i]) + margin = pa[top[0]] - (pa[top[1]] if len(top) > 1 else 0) + agree = top[0] == max(range(len(pb)), key=lambda i: pb[i]) + row.update(margin=margin, decision_agrees=agree) + if not agree: + dest = failures if margin > 0.02 else boundary + dest.append({**row, 'gate': 'choice', 'left': pa, 'right': pb}) + if meta['type'] == 'noul': + distance = abs(pa[1]-0.5) + agree = (pa[1] >= 0.5) == (pb[1] >= 0.5) + row.update(threshold_distance=distance, decision_agrees=agree) + if not agree: + dest = failures if distance > 0.01 else boundary + dest.append({**row, 'gate': 'noul', 'left': pa[1], 'right': pb[1]}) + if meta['type'] == 'score': + row['score_abs_error'] = abs(sum(i*(x-y) for i, (x, y) in enumerate(zip(pa, pb)))) + records.append(row) + report['comparisons'][f'{left}-to-{right}'] = {'max_abs': max(x['probability_max_abs'] for x in records), + 'questions': len(records), 'failures': failures, 'near_boundary_disagreements': boundary, 'records': records} + native = outputs['native']['cases'] + assert len(native) == len(corpus)+3, 'missing repeat captures' + first = next(c for c in native if c['name'] == 'mixed') + report['repeat_exact'] = all(c['body'] == first['body'] and c['probabilities'] == first['probabilities'] for c in native[len(corpus):]) + save(args.out / 'comparison.json', report) + for name, result in report['comparisons'].items(): + print(name, 'max_abs', result['max_abs'], 'failures', len(result['failures']), 'boundary', len(result['near_boundary_disagreements'])) + return 0 if report['repeat_exact'] and all(not c['failures'] for c in report['comparisons'].values()) else 1 + + +def main(): + p = argparse.ArgumentParser(description=__doc__) + sub = p.add_subparsers(dest='command', required=True) + ref = sub.add_parser('reference') + for flag in ('reference', 'base', 'adapter', 'export', 'out'): + ref.add_argument('--'+flag, type=Path, required=True) + cmp = sub.add_parser('compare') + cmp.add_argument('--out', type=Path, required=True) + args = p.parse_args() + args.out.mkdir(parents=True, exist_ok=True) + if args.command == 'reference': + reference(args) + return 0 + return compare(args) + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/recipe/kev/validate_http.py b/recipe/kev/validate_http.py new file mode 100644 index 00000000..ea56f884 --- /dev/null +++ b/recipe/kev/validate_http.py @@ -0,0 +1,121 @@ +"""Start the real Kev worker and frontend; test readiness, parity and HTTP failures. + +Run the entire script under gpu run; child processes are always stopped before exit. +""" +import argparse +import json +import os +import socket +import subprocess +import time +import urllib.error +import urllib.request +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +HTTP = urllib.request.build_opener(urllib.request.ProxyHandler({})) + + +def port(): + with socket.socket() as sock: + sock.bind(('127.0.0.1', 0)) + return sock.getsockname()[1] + + +def request(url, value=None, raw=None, content_type='application/json'): + data = raw if raw is not None else (json.dumps(value, ensure_ascii=False).encode() if value is not None else None) + req = urllib.request.Request(url, data=data, headers={'Content-Type': content_type}) + try: + with HTTP.open(req, timeout=180) as response: + return response.status, json.load(response) + except urllib.error.HTTPError as error: + return error.code, json.loads(error.read()) + + +def ready(process, url): + deadline = time.monotonic() + 300 + while time.monotonic() < deadline: + if process.poll() is not None: + raise RuntimeError(f'server exited {process.returncode}; see captured log') + try: + status, body = request(url+'/health') + if status == 200: + return body + except (OSError, urllib.error.URLError): + pass + time.sleep(0.25) + raise TimeoutError('worker readiness deadline') + + +def main(): + p = argparse.ArgumentParser(description=__doc__) + for flag in ('worker-bin', 'frontend-bin', 'export', 'cuda-lib', 'out', 'native'): + p.add_argument('--'+flag, type=Path, required=True) + p.add_argument('--corpus', type=Path, default=ROOT/'tests/kev/data/processing.json') + p.add_argument('--boundaries', type=Path, help='official-only boundary membership; select first ten per type') + args = p.parse_args() + args.out.mkdir(parents=True, exist_ok=False) + worker_port, frontend_port = port(), port() + worker_url, frontend_url = f'http://127.0.0.1:{worker_port}', f'http://127.0.0.1:{frontend_port}' + env = {**os.environ, 'KEV_MODEL': str(args.export), 'KEV_CUDA_LIB': str(args.cuda_lib), 'KEV_PORT': str(worker_port), + 'KEV_HOST': '127.0.0.1', 'OMNI_JEV_BIND': f'127.0.0.1:{frontend_port}', 'OMNI_JEV_BACKEND_URL': worker_url} + corpus = json.loads(args.corpus.read_text())['cases'] + if args.boundaries: + boundary = json.loads(args.boundaries.read_text())['items'] + selected = {r['case'] for kind in ('choice', 'noul') for r in [b for b in boundary if b['type'] == kind][:10]} + corpus = [c for c in corpus if c['population'] in ('contract', 'repeat') or c['name'] in selected] + captured = ([json.loads(line) for line in args.native.read_text().splitlines() if line.strip()] + if args.native.suffix == '.jsonl' else json.loads(args.native.read_text())['cases']) + native = {case['name']: case['body'] for case in captured} + processes = [] + report = {'cases': [], 'errors': [], 'selected': [c['name'] for c in corpus]} + started = time.monotonic() + with (args.out/'worker.log').open('w') as wlog, (args.out/'frontend.log').open('w') as flog: + try: + worker = subprocess.Popen([str(args.worker_bin)], env=env, stdout=wlog, stderr=subprocess.STDOUT) + processes.append(worker) + report['worker_health'] = ready(worker, worker_url) + report['load_and_real_warmup_seconds'] = time.monotonic()-started + # The very first external request after readiness includes all question types. + mixed = next(c for c in corpus if c['name'] == 'mixed') + status, first = request(worker_url+'/v1/systemone', mixed['request']) + assert status == 200 and first == native['mixed'], (status, first) + report['first_after_ready_exact'] = True + frontend = subprocess.Popen([str(args.frontend_bin)], env=env, stdout=flog, stderr=subprocess.STDOUT) + processes.append(frontend) + report['frontend_health'] = ready(frontend, frontend_url) + for case in corpus + [mixed]*3: + ds, direct = request(worker_url+'/v1/systemone', case['request']) + fs, front = request(frontend_url+'/v1/systemone', case['request']) + assert ds == fs == 200 and direct == front == native[case['name']], case['name'] + report['cases'].append({'name': case['name'], 'direct': direct, 'frontend': front}) + invalid = [ + ('malformed', 400, b'{', 'application/json'), + ('duplicate', 400, b'{"state":1,"state":2}', 'application/json'), + ('media-type', 415, b'{}', 'text/plain'), + ('oversize', 413, b' '*(4*1024*1024+1), 'application/json'), + ('unknown-model', 422, json.dumps({**mixed['request'], 'model': 'other'}).encode(), 'application/json'), + ('256-options', 422, json.dumps({'state': '', 'questions': {'q': {'type': 'score', 'criteria': ['x']*256}}}).encode(), 'application/json'), + ('empty-questions', 422, b'{"state":"", "questions":{}}', 'application/json'), + ('oversize-state', 422, json.dumps({'state': 'x '*9000, 'questions': {'q': {'type': 'noul'}}}).encode(), 'application/json'), + ] + for name, expected, raw, content_type in invalid: + ds, direct = request(worker_url+'/v1/systemone', raw=raw, content_type=content_type) + fs, front = request(frontend_url+'/v1/systemone', raw=raw, content_type=content_type) + assert ds == fs == expected and direct == front, (name, ds, fs, direct, front) + report['errors'].append({'name': name, 'status': ds, 'body': direct}) + report['passed'] = True + finally: + for process in reversed(processes): + process.terminate() + try: + process.wait(timeout=30) + except subprocess.TimeoutExpired: + process.kill() + process.wait() + (args.out/'http.json').write_text(json.dumps(report, ensure_ascii=False, indent=2)+'\n') + print('direct/frontend/native equality:', len(report['cases']), 'requests; HTTP errors:', len(report['errors'])) + + +if __name__ == '__main__': + main() diff --git a/src/backends/cuda/qwen3_5/README.md b/src/backends/cuda/qwen3_5/README.md index 090e54fb..0352ab5e 100644 --- a/src/backends/cuda/qwen3_5/README.md +++ b/src/backends/cuda/qwen3_5/README.md @@ -195,3 +195,12 @@ run the matched projection regression in `tests/qwen3_5/kernels.rs` and the [Cua screenshot qualification](../../../../recipe/cua_s1/native_screenshot.md#full-prompt-matched-algorithms) for each supported GPU/library/model combination. Other numerical profiles, vision GEMMs and the existing default/fixed policies stay on their own handles. + +## Kev default-service profile + +`recipe/kev/build_cuda.sh` enables optional Torch 2.8 reductions, stateful FP32 +GDN and masked CUTLASS SDPA. The existing ABI entry points retain their original +arithmetic; the profile uses additive symbols and is selected only by Kev. +Ordinary builds need no Torch/CUTLASS headers. The Kev build pins and verifies +CUTLASS and uses PyTorch 2.8.0 headers only at compile time, with no libtorch or +Python runtime linkage. See [numerical diagnosis](../../../../recipe/kev/numerical-diagnosis.md). diff --git a/src/backends/cuda/qwen3_5/build.sh b/src/backends/cuda/qwen3_5/build.sh index b17e33a0..752889c8 100755 --- a/src/backends/cuda/qwen3_5/build.sh +++ b/src/backends/cuda/qwen3_5/build.sh @@ -25,9 +25,15 @@ extra=() if [ -n "${CUDNN_INCLUDE_DIR:-}" ] || [ -n "${CUDNN_LIB_DIR:-}" ]; then : "${CUDNN_INCLUDE_DIR:?set CUDNN_INCLUDE_DIR and CUDNN_LIB_DIR together}" : "${CUDNN_LIB_DIR:?set CUDNN_INCLUDE_DIR and CUDNN_LIB_DIR together}" - extra=(-DCS1_REFERENCE_CUDNN -I"$CUDNN_INCLUDE_DIR") + extra+=(-DCS1_REFERENCE_CUDNN -I"$CUDNN_INCLUDE_DIR") link+=(-L"$CUDNN_LIB_DIR" -l:libcudnn.so.9 -Xlinker -rpath -Xlinker "$CUDNN_LIB_DIR") fi +# Optional Kev default-service SDPA; headers are needed only at build time. +if [ -n "${KEV_ATEN_INCLUDE_DIR:-}" ] || [ -n "${KEV_CUTLASS_INCLUDE_DIR:-}" ]; then + : "${KEV_ATEN_INCLUDE_DIR:?set both Kev header paths; use recipe/kev/build_cuda.sh}" + : "${KEV_CUTLASS_INCLUDE_DIR:?set both Kev header paths; use recipe/kev/build_cuda.sh}" + extra+=(-DCS1_KEV_SDPA -I"$KEV_ATEN_INCLUDE_DIR" -I"$KEV_CUTLASS_INCLUDE_DIR" --expt-relaxed-constexpr) +fi mkdir -p "$out" "$nvcc" -O3 -std=c++17 -gencode "arch=compute_${arch},code=[sm_${arch},compute_${arch}]" \ -shared -Xcompiler -fPIC -Xcompiler -Wall,-Wextra -I"$here" "$here"/*.cu \ diff --git a/src/backends/cuda/qwen3_5/gdn_fp32.cu b/src/backends/cuda/qwen3_5/gdn_fp32.cu index ba7baa41..f42172f9 100644 --- a/src/backends/cuda/qwen3_5/gdn_fp32.cu +++ b/src/backends/cuda/qwen3_5/gdn_fp32.cu @@ -15,6 +15,7 @@ struct Context { size_t bytes = 0; cudaStream_t stream = nullptr; int device = 0; + bool torch28 = false; }; struct Layout { size_t q, k, kbeta, qd, kd, u, w, a, p, cum, state, vnew, temp, out, plus, @@ -112,6 +113,7 @@ __global__ void cumulative(const float *g, float *out, int tokens, int heads, __syncthreads(); } } +template __global__ void prepare(const cs1::bf16 *q, const cs1::bf16 *k, const cs1::bf16 *v, const cs1::bf16 *beta, Work ws, int tokens, int heads, int key_heads, int nc, @@ -122,7 +124,7 @@ __global__ void prepare(const cs1::bf16 *q, const cs1::bf16 *k, float qv[4], kv[4], qs[4] = {}, ks[4] = {}; #pragma unroll for (int j = 0; j < 4; j++) { - int d = lane * 4 + j; + int d = Torch28 ? lane + 32*j : lane*4+j; qv[j] = token < tokens ? cs1::f32(q[((size_t)token * key_heads + kh) * D + d]) : 0.f; @@ -134,7 +136,7 @@ __global__ void prepare(const cs1::bf16 *q, const cs1::bf16 *k, } float qsum = ((qs[0] + qs[1]) + qs[2]) + qs[3], ksum = ((ks[0] + ks[1]) + ks[2]) + ks[3]; - for (int step = 16; step > 0; step /= 2) { + for (int step = Torch28 ? 1 : 16; Torch28 ? step < 32 : step > 0; step = Torch28 ? step*2 : step/2) { qsum += __shfl_down_sync(0xffffffff, qsum, step); ksum += __shfl_down_sync(0xffffffff, ksum, step); } @@ -146,7 +148,7 @@ __global__ void prepare(const cs1::bf16 *q, const cs1::bf16 *k, float ec = expf(cum), er = expf(last - cum); #pragma unroll for (int j = 0; j < 4; j++) { - int d = lane * 4 + j; + int d = Torch28 ? lane + 32*j : lane*4+j; size_t index = ((size_t)head * nc * C + row) * D + d; float qn = __fmul_rn(__fmul_rn(qv[j], qi), scale), kn = __fmul_rn(kv[j], ki), kb = __fmul_rn(kn, b); @@ -259,11 +261,11 @@ extern "C" size_t cs1_gdn_fp32_workspace_floats(int tokens, int heads) { return 0; return Layout(tokens, heads).total / 4; } -extern "C" int cs1_gdn_prefill_fp32(void *opaque, const void *q, const void *k, +extern "C" int cs1_gdn_prefill_fp32_x(void *opaque, const void *q, const void *k, const void *v, const float *g, const void *beta, void *out, float *workspace, int tokens, int heads, - int key_heads, float scale, void *stream) { + int key_heads, float scale, const void *initial, void *final_state, void *stream) { auto *c = (Context *)opaque; if (!c || !workspace || !q || !k || !v || !g || !beta || !out || !cs1_gdn_fp32_workspace_floats(tokens, heads) || key_heads <= 0 || @@ -287,9 +289,15 @@ extern "C" int cs1_gdn_prefill_fp32(void *opaque, const void *q, const void *k, int width = 1 << max(4, min(9, log_width)); cumulative<<stream>>>(g, ws.cum, tokens, heads, nc, width); - prepare<<stream>>>( + if(c->torch28) { + prepare<<stream>>>( (const cs1::bf16 *)q, (const cs1::bf16 *)k, (const cs1::bf16 *)v, (const cs1::bf16 *)beta, ws, tokens, heads, key_heads, nc, scale); + } else { + prepare<<stream>>>( + (const cs1::bf16 *)q, (const cs1::bf16 *)k, (const cs1::bf16 *)v, + (const cs1::bf16 *)beta, ws, tokens, heads, key_heads, nc, scale); + } rc = cudaGetLastError(); if (rc) return rc; @@ -318,7 +326,7 @@ extern "C" int cs1_gdn_prefill_fp32(void *opaque, const void *q, const void *k, BLAS(cublasStrsmBatched(c->blas, CUBLAS_SIDE_RIGHT, CUBLAS_FILL_MODE_UPPER, CUBLAS_OP_N, CUBLAS_DIAG_UNIT, D, C, &one, ws.pa, C, ws.pw, D, batches)); - rc = cudaMemsetAsync(ws.state, 0, (size_t)heads * D * D * 4, c->stream); + rc = initial ? cudaMemcpyAsync(ws.state, initial, (size_t)heads*D*D*4, cudaMemcpyDeviceToDevice, c->stream) : cudaMemsetAsync(ws.state, 0, (size_t)heads*D*D*4, c->stream); if (rc) return rc; for (int chunk = 0; chunk < nc; chunk++) { @@ -348,5 +356,13 @@ extern "C" int cs1_gdn_prefill_fp32(void *opaque, const void *q, const void *k, return rc; } #undef BLAS + if(final_state) return cudaMemcpyAsync(final_state,ws.state,(size_t)heads*D*D*4,cudaMemcpyDeviceToDevice,c->stream); return cudaSuccess; } + +extern "C" void* cs1_gdn_fp32_create_torch28(size_t bytes) { + auto* c=(Context*)cs1_gdn_fp32_create(bytes);if(c)c->torch28=true;return c; +} +extern "C" int cs1_gdn_prefill_fp32(void* context,const void* q,const void* k,const void* v,const float* g,const void* beta,void* out,float* work,int t,int h,int hk,float scale,void* stream) { + return cs1_gdn_prefill_fp32_x(context,q,k,v,g,beta,out,work,t,h,hk,scale,nullptr,nullptr,stream); +} diff --git a/src/backends/cuda/qwen3_5/norm.cu b/src/backends/cuda/qwen3_5/norm.cu index 5e1c210d..6973aef7 100644 --- a/src/backends/cuda/qwen3_5/norm.cu +++ b/src/backends/cuda/qwen3_5/norm.cu @@ -74,27 +74,34 @@ __global__ void __launch_bounds__(NORM_THREADS) // Qwen3_5RMSNormGated with D = 128, one warp per row: the normalized value is // rounded to bfloat16, multiplied by w in bfloat16, then by silu(z) in float32. +template __global__ void gated_rms_norm_kernel(const bf16* __restrict__ x, const bf16* __restrict__ z, int ldz, const bf16* __restrict__ w, bf16* __restrict__ out, int rows, int H, float eps) { constexpr int D = 128, PER = D / 32; const int row = blockIdx.x * (blockDim.x / 32) + threadIdx.x / 32, lane = threadIdx.x & 31; if (row >= rows) return; - const size_t base = (size_t)row * D + lane * PER; - const size_t zbase = (size_t)(row / H) * ldz + (row % H) * D + lane * PER; + constexpr int STRIDE=Torch28?32:1; + const size_t base = (size_t)row * D + (Torch28?lane:lane*PER); + const size_t zbase = (size_t)(row / H) * ldz + (row % H) * D + (Torch28?lane:lane*PER); float v[PER]; float ss = 0.f; #pragma unroll for (int i = 0; i < PER; i++) { - v[i] = f32(x[base + i]); + v[i] = f32(x[base + i * STRIDE]); ss += v[i] * v[i]; } - const float inv = rsqrtf(warp_sum(ss) / D + eps); + float sum; + if constexpr(Torch28) { + for(int step=1;step<32;step*=2) ss=__fadd_rn(ss,__shfl_down_sync(0xffffffff,ss,step)); + sum=__shfl_sync(0xffffffff,ss,0); + } else {sum=warp_sum(ss);} + const float inv=rsqrtf(sum/D+eps); #pragma unroll for (int i = 0; i < PER; i++) { const float normed = round_bf16(v[i] * inv); - const float scaled = round_bf16(f32(w[lane * PER + i]) * normed); - out[base + i] = to_bf16(scaled * silu(f32(z[zbase + i]))); + const float scaled = round_bf16(f32(w[(Torch28?lane:lane*PER) + i * STRIDE]) * normed); + out[base + i * STRIDE] = to_bf16(scaled * silu(f32(z[zbase + i * STRIDE]))); } } @@ -130,14 +137,22 @@ extern "C" int cs1_add_rms_norm(void* residual, const void* delta, const void* w return cudaGetLastError(); } -extern "C" int cs1_gated_rms_norm(const void* x, const void* z, int ldz, const void* w, void* out, int T, int H, +template int gated_rms_norm_run(const void* x, const void* z, int ldz, const void* w, void* out, int T, int H, int D, float eps, void* stream) { if (D != 128 || ldz < H * D) return cudaErrorInvalidValue; const int rows = T * H; if (rows <= 0) return cudaSuccess; constexpr int WARPS = 8; - gated_rms_norm_kernel<<<(rows + WARPS - 1) / WARPS, WARPS * 32, 0, static_cast(stream)>>>( + gated_rms_norm_kernel<<<(rows + WARPS - 1) / WARPS, WARPS * 32, 0, static_cast(stream)>>>( static_cast(x), static_cast(z), ldz, static_cast(w), static_cast(out), rows, H, eps); return cudaGetLastError(); } + +extern "C" int cs1_gated_rms_norm(const void* x,const void* z,int ld,const void* w,void* out,int t,int h,int d,float eps,void* stream) { + return gated_rms_norm_run(x,z,ld,w,out,t,h,d,eps,stream); +} + +extern "C" int cs1_torch28_gated_rms_norm(const void* x,const void* z,int ld,const void* w,void* out,int t,int h,int d,float eps,void* stream) { + return gated_rms_norm_run(x,z,ld,w,out,t,h,d,eps,stream); +} diff --git a/src/backends/cuda/qwen3_5/ops.h b/src/backends/cuda/qwen3_5/ops.h index 9ad9c739..2a945bee 100644 --- a/src/backends/cuda/qwen3_5/ops.h +++ b/src/backends/cuda/qwen3_5/ops.h @@ -206,6 +206,26 @@ int cs1_vision_to_float(const void* x, float* out, size_t n, void* stream); int cs1_vision_lora_add(void* x, const float* delta, size_t n, float scale, void* stream); +// ---- Optional Kev Torch 2.8 default-service profile (additive; old ABI unchanged) ---- +// Torch 2.8 reduction order; only 4B hidden width 2560 / GDN dimension 128. +int cs1_torch28_rms_norm(const void* x,const void* w,void* out,int rows,int D,float eps,void* stream); +int cs1_torch28_add_rms_norm(void* residual,const void* delta,const void* w,void* out,int rows,int D,float eps,void* stream); +int cs1_torch28_gated_rms_norm(const void* x,const void* z,int ldz,const void* w,void* out,int T,int H,int D,float eps,void* stream); +int cs1_torch28_attn_prep(const void* qg,const void* kr,int ld,const void* qw,const void* kw,const void* cos,const void* sin, + void* q,void* gate,void* k,int T,int Hq,int Hk,int Dh,int half,float eps,void* stream); +void* cs1_gdn_fp32_create_torch28(size_t workspace_bytes); +// Null initial state means zeros; optional final_state receives [H,128,128] FP32. +// Context and every allocation remain alive until the supplied stream drains. +int cs1_gdn_prefill_fp32_x(void* context,const void* q,const void* k,const void* v,const float* g,const void* beta, + void* out,float* workspace,int T,int H,int HK,float scale,const void* initial,void* final_state,void* stream); +// Q/gate/out [nq,H,256]; K [nk,HK,256]; V [nk,ld]. prefix = nk-nq. +// Mask allows real prefix keys [left,prefix) and causal question keys [prefix,prefix+real), +// plus each query's own key. Workspace bytes include a BF16 bias with 8-element row alignment. +size_t cs1_service_attention_bytes(int nq,int nk,int H); +int cs1_service_attention(const void* q,const void* k,const void* v,int ld,const void* gate,void* out, + int nq,int nk,int H,int HK,int prefix,int left,int real,void* workspace,void* stream); +int cs1_service_zero(void* ptr,size_t bytes,void* stream); + // ---- Optional JEMM framework numerics (ABI 6 additive) ---- void* cs1_reference_gemm_create(size_t workspace_bytes); int cs1_reference_rms_norm(const void* x, const void* w, void* out, int rows, int D, float eps, void* stream); diff --git a/src/backends/cuda/qwen3_5/qwen3_5.backend.json b/src/backends/cuda/qwen3_5/qwen3_5.backend.json index 0f11ccff..23334d45 100644 --- a/src/backends/cuda/qwen3_5/qwen3_5.backend.json +++ b/src/backends/cuda/qwen3_5/qwen3_5.backend.json @@ -21,7 +21,8 @@ "reference_patch.cu", "reference_prep.cu", "reference_vision.cu", - "reference_vision_attention.cu" + "reference_vision_attention.cu", + "service_readout.cu" ], "models": [ "src/models/cua_s1/native/", @@ -30,7 +31,8 @@ "src/models/decider/native/", "src/models/jemm/native/", "src/models/startlux/native/", - "src/models/omnijev/native/" + "src/models/omnijev/native/", + "src/models/kev/native/" ], "build": { "script": "build.sh", diff --git a/src/backends/cuda/qwen3_5/reference_norm.cu b/src/backends/cuda/qwen3_5/reference_norm.cu index c4f13891..33f87800 100644 --- a/src/backends/cuda/qwen3_5/reference_norm.cu +++ b/src/backends/cuda/qwen3_5/reference_norm.cu @@ -1,6 +1,6 @@ #include "common.cuh" using namespace cs1; -template +template __global__ void reference_norm(const bf16* x,const bf16* delta,const bf16* w,bf16* out,int D,float eps) { __shared__ float shared[512]; int lane=threadIdx.x;size_t row=blockIdx.x;x+=row*D;if(delta)delta+=row*D;out+=row*D;float sum[4]={}; @@ -11,24 +11,34 @@ __global__ void reference_norm(const bf16* x,const bf16* delta,const bf16* w,bf1 float value=__fadd_rn(__fadd_rn(__fadd_rn(sum[0],sum[1]),sum[2]),sum[3]); shared[lane]=value; for(int step=blockDim.x/2;step>=32;step/=2) {__syncthreads();if(lane0;step/=2)value=__fadd_rn(value,__shfl_down_sync(0xffffffff,value,step));if(lane==0)shared[0]=rsqrtf(__fadd_rn(__fmul_rn(value,1.f/D),eps));} + if(lane<32){if constexpr(Torch28){for(int step=1;step<32;step*=2)value=__fadd_rn(value,__shfl_down_sync(0xffffffff,value,step));}else{for(int step=16;step>0;step/=2)value=__fadd_rn(value,__shfl_down_sync(0xffffffff,value,step));}if(lane==0)shared[0]=rsqrtf(__fadd_rn(__fmul_rn(value,1.f/D),eps));} __syncthreads();float inverse=shared[0]; for(int i=lane;i int reference_norm_run(const void* x,const void* delta,const void* w,void* out,int rows,int D,float eps,void* stream) { int powRows=1,powD=1;while(powRows*2<=rows&&powRows<512)powRows*=2;while(powD*2<=D/4&&powD<512)powD*=2; int width=min(powD,32),height=min(powRows,512/width);width=min(powD,512/height); int threads=width;if((D+width-1)/width>=min(height*16,256))threads*=height; - if(delta)reference_norm<<>>((const bf16*)x,(const bf16*)delta,(const bf16*)w,(bf16*)out,D,eps); - else reference_norm<<>>((const bf16*)x,nullptr,(const bf16*)w,(bf16*)out,D,eps); + if(delta)reference_norm<<>>((const bf16*)x,(const bf16*)delta,(const bf16*)w,(bf16*)out,D,eps); + else reference_norm<<>>((const bf16*)x,nullptr,(const bf16*)w,(bf16*)out,D,eps); return cudaGetLastError(); } extern "C" int cs1_reference_rms_norm(const void* x,const void* w,void* out,int rows,int d,float eps,void* stream) { if(rows<=0 || (d!=5120 && d!=2560)) return cudaErrorInvalidValue; - return reference_norm_run(x,nullptr,w,out,rows,d,eps,stream); + return reference_norm_run(x,nullptr,w,out,rows,d,eps,stream); } extern "C" int cs1_reference_add_rms_norm(void* x,const void* delta,const void* w,void* out,int rows,int d,float eps,void* stream) { if(rows<=0 || (d!=5120 && d!=2560)) return cudaErrorInvalidValue; - return reference_norm_run(x,delta,w,out,rows,d,eps,stream); + return reference_norm_run(x,delta,w,out,rows,d,eps,stream); +} + +// PyTorch 2.8 Reduce.cuh visits shuffle offsets in ascending order. +extern "C" int cs1_torch28_rms_norm(const void* x,const void* w,void* out,int rows,int d,float eps,void* stream) { + if(rows<=0 || d!=2560) return cudaErrorInvalidValue; + return reference_norm_run(x,nullptr,w,out,rows,d,eps,stream); +} +extern "C" int cs1_torch28_add_rms_norm(void* x,const void* delta,const void* w,void* out,int rows,int d,float eps,void* stream) { + if(rows<=0 || d!=2560) return cudaErrorInvalidValue; + return reference_norm_run(x,delta,w,out,rows,d,eps,stream); } diff --git a/src/backends/cuda/qwen3_5/reference_prep.cu b/src/backends/cuda/qwen3_5/reference_prep.cu index 138d99a9..ee3c72c6 100644 --- a/src/backends/cuda/qwen3_5/reference_prep.cu +++ b/src/backends/cuda/qwen3_5/reference_prep.cu @@ -5,6 +5,7 @@ constexpr int PER = DH / 32; // values per lane // One warp per (token, head), q heads first, then k heads. Each lane holds 8 // consecutive dims, so the rotary partner of dim i < 32 (dim i + 32) sits in lane ^ 4. +template __global__ void attn_prep_kernel(const bf16* __restrict__ qg, const bf16* __restrict__ kr, int ld, const bf16* __restrict__ qw, const bf16* __restrict__ kw, const bf16* __restrict__ cos, const bf16* __restrict__ sin, @@ -29,7 +30,7 @@ __global__ void attn_prep_kernel(const bf16* __restrict__ qg, const bf16* __rest sums[j]=__fadd_rn(__fmul_rn(a,a),__fmul_rn(b,b)); } float ss=__fadd_rn(__fadd_rn(__fadd_rn(sums[0],sums[1]),sums[2]),sums[3]); - for(int step=16;step>0;step/=2)ss=__fadd_rn(ss,__shfl_down_sync(0xffffffff,ss,step)); + if constexpr(Torch28){for(int step=1;step<32;step*=2)ss=__fadd_rn(ss,__shfl_down_sync(0xffffffff,ss,step));}else{for(int step=16;step>0;step/=2)ss=__fadd_rn(ss,__shfl_down_sync(0xffffffff,ss,step));} ss=__shfl_sync(0xffffffff,ss,0); const float inv=rsqrtf(__fadd_rn(__fmul_rn(ss,1.f/DH),eps)); float wv[PER]; @@ -66,7 +67,7 @@ __global__ void attn_prep_kernel(const bf16* __restrict__ qg, const bf16* __rest } } using namespace cs1; using namespace cs1::reference_prep; -extern "C" int cs1_reference_attn_prep(const void* qg, const void* kr, int ld, const void* qw, const void* kw, +template int reference_attn_prep_run(const void* qg, const void* kr, int ld, const void* qw, const void* kw, const void* cos, const void* sin, void* q, void* gate, void* k, int T, int Hq, int Hk, int Dh, int half, float eps, void* stream) { if (ld % 8 != 0 || T < 0) return cudaErrorInvalidValue; @@ -75,9 +76,17 @@ extern "C" int cs1_reference_attn_prep(const void* qg, const void* kr, int ld, c const int warps = T * (Hq + Hk); if (warps == 0) return cudaSuccess; constexpr int WARPS = 8; - attn_prep_kernel<<<(warps + WARPS - 1) / WARPS, WARPS * 32, 0, static_cast(stream)>>>( + attn_prep_kernel<<<(warps + WARPS - 1) / WARPS, WARPS * 32, 0, static_cast(stream)>>>( static_cast(qg), static_cast(kr), ld, static_cast(qw), static_cast(kw), static_cast(cos), static_cast(sin), static_cast(q), static_cast(gate), static_cast(k), T, Hq, Hk, half, eps); return cudaGetLastError(); } + +extern "C" int cs1_reference_attn_prep(const void* qg,const void* kr,int ld,const void* qw,const void* kw,const void* cos,const void* sin,void* q,void* gate,void* k,int t,int hq,int hk,int d,int half,float eps,void* stream) { + return reference_attn_prep_run(qg,kr,ld,qw,kw,cos,sin,q,gate,k,t,hq,hk,d,half,eps,stream); +} + +extern "C" int cs1_torch28_attn_prep(const void* qg,const void* kr,int ld,const void* qw,const void* kw,const void* cos,const void* sin,void* q,void* gate,void* k,int t,int hq,int hk,int d,int half,float eps,void* stream) { + return reference_attn_prep_run(qg,kr,ld,qw,kw,cos,sin,q,gate,k,t,hq,hk,d,half,eps,stream); +} diff --git a/src/backends/cuda/qwen3_5/service_readout.cu b/src/backends/cuda/qwen3_5/service_readout.cu new file mode 100644 index 00000000..b284f4eb --- /dev/null +++ b/src/backends/cuda/qwen3_5/service_readout.cu @@ -0,0 +1,65 @@ +// Kev's masked SDPA: PyTorch 2.8.0 + CUTLASS ad7b2f5 (see recipe/kev/build_cuda.sh). +// Header-only kernel instantiation; no libtorch or Python runtime dependency. +// Other workers retain their existing attention path and build dependencies. +#ifdef CS1_KEV_SDPA +#include +#include +using Kernel=PyTorchMemEffAttention::AttentionKernel; +static int cutlass_attention(const void* q,const void* k,const void* v,const void* mask,void* out,void* accum,int nq,int nk,int heads,void* stream) { + Kernel::Params p; + p.query_ptr=(const cutlass::bfloat16_t*)q; + p.key_ptr=(const cutlass::bfloat16_t*)k; + p.value_ptr=(const cutlass::bfloat16_t*)v; + p.attn_bias_ptr=(const cutlass::bfloat16_t*)mask; + p.output_ptr=(cutlass::bfloat16_t*)out; + p.output_accum_ptr=(float*)accum; + p.scale=.0625f;p.head_dim=256;p.head_dim_value=256; + p.num_queries=nq;p.num_keys=nk;p.num_keys_absolute=nk; + p.q_strideM=p.k_strideM=p.v_strideM=p.o_strideM=heads*256; + p.q_strideH=p.k_strideH=p.v_strideH=256; + p.bias_strideM=(nk+7)/8*8;p.bias_strideH=0;p.bias_strideB=0; + p.q_strideB=(int64_t)nq*heads*256;p.k_strideB=p.v_strideB=(int64_t)nk*heads*256; + p.num_batches=1;p.num_heads=heads; + auto fn=PyTorchMemEffAttention::attention_kernel_batched_impl; + constexpr int smem=sizeof(Kernel::SharedStorage); + auto err=cudaFuncSetAttribute(fn,cudaFuncAttributeMaxDynamicSharedMemorySize,smem); + if(err) return err; + fn<<>>(p); + return cudaGetLastError(); +} +#include "common.cuh" +#include "ops.h" +namespace { +__global__ void expand_kv(const cs1::bf16* k,const cs1::bf16* v,int ld,cs1::bf16* kr,cs1::bf16* vr,int nk,int h,int hk) { + size_t i=(size_t)blockIdx.x*blockDim.x+threadIdx.x,n=(size_t)nk*h*256; + if(i>=n)return;int d=i%256,head=i/256%h,t=i/(256*h),kh=head/(h/hk); + kr[i]=k[((size_t)t*hk+kh)*256+d];vr[i]=v[(size_t)t*ld+kh*256+d]; +} +__global__ void mask_rows(cs1::bf16* mask,int nq,int nk,int prefix,int left,int real) { + int maskld=(nk+7)/8*8; + size_t i=(size_t)blockIdx.x*blockDim.x+threadIdx.x;if(i>=(size_t)nq*maskld)return; + int q=i/maskld,k=i%maskld; + bool visible=(k=left && k<=prefix+q && (k65536 || nk65536 || h<=0 || h>32)return 0; + return (size_t)nq*h*256*4+2*(size_t)nk*h*256*2+(size_t)nq*((nk+7)/8*8)*2; +} +extern "C" int cs1_service_attention(const void* q,const void* k,const void* v,int ld,const void* gate,void* out,int nq,int nk,int h,int hk,int prefix,int left,int real,void* workspace,void* stream) { + if(!q || !k || !v || !gate || !out || !workspace || !cs1_service_attention_bytes(nq,nk,h) + || hk<=0 || h%hk || ldnk + || real<0 || real>nq)return cudaErrorInvalidValue; + char* bytes=(char*)workspace;size_t accum=(size_t)nq*h*256*4,kv=(size_t)nk*h*256*2; + auto kr=(cs1::bf16*)(bytes+accum);auto vr=(cs1::bf16*)(bytes+accum+kv);auto mask=(cs1::bf16*)(bytes+accum+2*kv); + expand_kv<<<((size_t)nk*h*256+255)/256,256,0,(cudaStream_t)stream>>>((const cs1::bf16*)k,(const cs1::bf16*)v,ld,kr,vr,nk,h,hk); + mask_rows<<<((size_t)nq*((nk+7)/8*8)+255)/256,256,0,(cudaStream_t)stream>>>(mask,nq,nk,prefix,left,real); + auto error=cudaGetLastError();if(error)return error; + int rc=cutlass_attention(q,kr,vr,mask,out,workspace,nq,nk,h,stream); + if(rc)return rc; + return cs1_sigmoid_gate(out,gate,(size_t)nq*h*256,stream); +} +extern "C" int cs1_service_zero(void* ptr,size_t n,void* stream) {return cudaMemsetAsync(ptr,0,n,(cudaStream_t)stream);} + +#endif // CS1_KEV_SDPA diff --git a/src/models/kev/README.md b/src/models/kev/README.md new file mode 100644 index 00000000..3e8731ea --- /dev/null +++ b/src/models/kev/README.md @@ -0,0 +1,125 @@ +# Kev-4B native text decisions + +Kev serves `choice`, `noul`, and `score` with its own processor and FP32 pointer +head over the shared Qwen3.5 CUDA backbone. The [preparation recipe](../../../recipe/kev/README.md) +remains independently reviewable; the [native recipe](../../../recipe/kev/native.md) +covers the complete eager worker. Other sizes and prefix reuse are deferred. +The historical FP32 diagnostics used a 13-request / 23-question corpus; +[results and limits](../../../recipe/kev/native.md#historical-fp32-eager-diagnostics) include +merge drift, hardware provenance, Score scale effects and HTTP checks. + +## Pinned artifacts + +| Artifact | Immutable revision | +| --- | --- | +| [jaredpalmer/kev-4b](https://huggingface.co/jaredpalmer/kev-4b/tree/139fdd94f1b6a6ad80cc15e08fcb99cac885a101), adapter and head | `139fdd94f1b6a6ad80cc15e08fcb99cac885a101` | +| [Qwen/Qwen3.5-4B-Base](https://huggingface.co/Qwen/Qwen3.5-4B-Base/tree/1001bb4d826a52d1f399e183466143f4da7b741b) | `1001bb4d826a52d1f399e183466143f4da7b741b` | +| [Kev reference code](https://github.com/jaredpalmer/kev/tree/2074cc44256200bb37540103fa4572901faa10f3) | `2074cc44256200bb37540103fa4572901faa10f3` | + +All are Apache-2.0; the adapted model/API code's license is retained in +[`native/LICENSE.kev`](native/LICENSE.kev). No model weights are committed. +The source hash in the fixture generator covers the complete pinned `kev/model.py`. + +## Export contract + +[`export.py`](../../../recipe/kev/export.py) verifies SHA-256 for every required +base/checkpoint file **before deserializing `head.pt`**, which uses PyTorch's +weights-only CPU loader. It merges all 496 FP32 LoRA tensors into 248 language +projections as `BF16(FP32(W) + 2 * (B @ A))`. This includes all five GDN projection +types: `in_proj_qkv`, `in_proj_a`, `in_proj_z`, `in_proj_b` and `out_proj`. +Missing pairs or unused targets fail the export. The output excludes vision/MTP +weights and preserves the original language tensor names for the shared Qwen loader. +Other language tensors are converted to BF16, including originally FP32 norms. + +The four pointer-head tensors remain FP32: `q.weight`/`k.weight` are `[256, 2560]` +and their biases are `[256]`. The manifest records temperature +`2.406050072164233`, dimensions, delimiter IDs, exact input hashes, every merged +target and all output hashes. It is written after the artifacts, and the private +staging directory is renamed only on success. Existing outputs are refused. + +The tokenizer comes from the **base**: `<|fim_prefix|>` = 248060 (state), +`<|fim_middle|>` = 248061 (question), `<|box_start|>` = 248049 (option start), +`<|box_end|>` = 248050 (option readout), `<|fim_suffix|>` = 248062 (decision +readout). The adapter's older added-token metadata is not copied. + +## Executor and postprocessing boundary + +`head::PointerHead` owns the trained CPU weights and accepts one final-normalized +decision hidden state plus 1–255 option hidden states, each with 2560 FP32 values +for this checkpoint. It computes `k(h_option) dot q(h_decide) / sqrt(256)` in FP32, +including both learned biases, and preserves candidate order. Inputs, shapes, +weights and results must be finite. Returned logits own their storage. +`postprocess::probabilities` divides these logits by the saved temperature and +normalizes across the **whole question**. This follows the eval-mode reference; +temperature must not be applied twice. Projection and calibration have separate +modules even though both currently run on CPU. + +## Request flow and ownership + +`http::decide` admits a bounded request and prepares it on a blocking CPU task. +`contract::compile` follows the pinned `kev/api.py`: recursive text rendering, +insertion order, Choice `name: description`, Noul `no`/`yes` candidates and ordered +Score levels. `processing::Processor` owns the base tokenizer, rewrites caller +`<|name|>` spellings to `<¦name¦>`, and encodes each fragment separately, without a chat template. Each +prepared question owns one complete causal row and its candidate/decision indices. +Question positions restart at the state length. Prepared rows are unpadded; the +executor applies reference bucket padding, without truncation. + +`Engine` owns a shared `SerialScheduler` and one mutex-protected `Executor`. +One admitted execution unit is a whole request. Questions run sequentially; +`planning` reproduces official question groups and projection shapes. +`Model::forward_segmented_readout` consumes text-only state/question segments +with identical T/H/W positions and returns final-normalized readout rows. The +executor owns CUDA weights/scratch and the FP32 pointer head; only owned logits +leave it. Every question recomputes its state and continues from that temporary GDN/KV +state. No prefix is retained across questions or requests. The stream is synchronized +before execution resources and admission permits can be released, including on +errors. Cancelled callers cannot free buffers while dispatched GPU work uses them. + +`ResponseContext` retains original question/candidate mapping and the tokenizer. +Postprocessing calibrates once, normalizes across each entire question, chooses +the first candidate on ties, and restores the reference's confidence, legends, +four-decimal scalars, and billing token counts. State counts once in input usage; +compute cost separately counts each complete row. The native recipe documents +stricter context/request limits and deliberate HTTP differences. + +The export loader verifies every input pin in `recipe/kev/pins.json`, the complete +merged target inventory, all seven output hashes, shard references, head dimensions +and base tokenizer markers before CUDA loading. Startup runs all three question +types before binding its HTTP listener. Shared Qwen/kernel/runtime code is reused +without adding a Kev-specific backend or scheduling policy. + +## Validation and remaining gates + +Tests live under [`tests/kev/`](../../../tests/kev/). Ordinary Rust tests cover +learned biases, scale, candidate ordering, one/255 candidates, malformed weights, +hidden states and temperature. CPU Python tests cover pinned-input rejection, +no overwrite, all five GDN target names, one-rounding merge arithmetic, incomplete +pairs, delimiter IDs and head metadata. + +The opt-in **CPU** test loads the real exported head and compares 1, 2, 7, 31 and +255 synthetic hidden-state candidates with the actual pinned upstream +`PointerHead` class (PyTorch 2.8.0 CPU). Logit tolerance is +`1e-4 + 1e-5 * abs(reference)`; probability absolute tolerance is `1e-5`, declared +before comparison. The small committed fixture contains expected outputs and +provenance, not weights; its deterministic binary-fraction inputs are regenerated +by the generator and Rust test. + +The maintained `tests/kev/data/processing.json` oracle is generated by the +hash-verified official API and `encode`/`rows_of` functions. CPU checks compare +rendering and answers; the opt-in tokenizer check compares every token and readout, +plus input/output usage. The native capture and HTTP harness are explicit GPU +checks, never run by ordinary workspace tests. + +Primary acceptance uses the pinned official BF16 default service, with Score +scalar error at most 0.1 ordinal level and maximum probability error 0.01. +The [expanded qualification](../../../recipe/kev/service-acceptance.md) currently +passes all frozen gates after the [v3 repair](../../../recipe/kev/numerical-diagnosis.md) +and [v4 boundary qualification](../../../recipe/kev/noul-boundary.md): 36 Noul +and 28 Choice boundary questions, with no decision disagreements. Contributor and maintainer review remain pending. +Decision gates require Choice agreement +for reference top-two margins above 0.02, and Noul agreement at threshold 0.5 when +the reference distance exceeds 0.01. The comparator reports all near-boundary +flips. Diagnostic comparisons retain unmerged official FP32, exported BF16 PyTorch and +native execution. Failure never widens these limits. See the native recipe for +commands and current validation status. No performance claim is made. diff --git a/src/models/kev/native/Cargo.toml b/src/models/kev/native/Cargo.toml new file mode 100644 index 00000000..0cd889a3 --- /dev/null +++ b/src/models/kev/native/Cargo.toml @@ -0,0 +1,31 @@ +[package] +name = "omni-kev-native" +version = "0.1.0" +edition = "2024" +publish = false +description = "Kev-4B eager Rust/CUDA text worker" + +[dependencies] +anyhow = "1.0.100" +axum = "0.8.8" +omni-qwen3-5-native = { path = "../../qwen3_5/native" } +omni-runtime = { path = "../../../runtime" } +regex = "1" +safetensors = "0.8.0" +serde = { version = "1", features = ["derive"] } +serde_json = { version = "1.0.149", features = ["float_roundtrip", "preserve_order"] } +sha2 = "0.10" +tokenizers = { version = "=0.22.2", default-features = false, features = ["onig"] } +tokio = { version = "1.49.0", features = ["macros", "net", "rt-multi-thread", "sync"] } + +[[test]] +name = "head" +path = "../../../../tests/kev/head.rs" + +[[test]] +name = "processing" +path = "../../../../tests/kev/processing.rs" + +[[test]] +name = "checkpoint" +path = "../../../../tests/kev/checkpoint.rs" diff --git a/src/models/kev/native/LICENSE.kev b/src/models/kev/native/LICENSE.kev new file mode 100644 index 00000000..e1ece45b --- /dev/null +++ b/src/models/kev/native/LICENSE.kev @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2026 Jared Palmer + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/src/models/kev/native/src/checkpoint.rs b/src/models/kev/native/src/checkpoint.rs new file mode 100644 index 00000000..88c2f14f --- /dev/null +++ b/src/models/kev/native/src/checkpoint.rs @@ -0,0 +1,110 @@ +//! Verify the complete preparation contract before loading any CUDA library. +use anyhow::{Context, Result, ensure}; +use serde_json::Value; +use sha2::{Digest, Sha256}; +use std::{collections::BTreeSet, fs::File, io::Read, path::Path}; + +pub const PINS: &str = include_str!("../../../../../recipe/kev/pins.json"); +const SHARDS: [&str; 2] = [ + "model.safetensors-00001-of-00002.safetensors", + "model.safetensors-00002-of-00002.safetensors", +]; +const FILES: [&str; 7] = [ + "config.json", + "head.safetensors", + "model.safetensors.index.json", + "tokenizer.json", + "tokenizer_config.json", + SHARDS[0], + SHARDS[1], +]; + +pub fn validate_manifest(manifest: &Value) -> Result<()> { + let pins: Value = serde_json::from_str(PINS)?; + for (key, value) in pins.as_object().unwrap() { + ensure!( + manifest.get(key) == Some(value), + "unsupported Kev export field: {key}" + ); + } + let files = manifest["files"] + .as_object() + .context("missing export files")?; + ensure!( + files.keys().map(String::as_str).collect::>() == FILES.into_iter().collect(), + "unexpected or missing export files" + ); + for (name, digest) in files { + let digest = digest.as_str().context("export hash must be a string")?; + ensure!( + digest.len() == 64 && digest.bytes().all(|b| b.is_ascii_hexdigit()), + "invalid hash: {name}" + ); + } + for name in ["config.json", "tokenizer.json", "tokenizer_config.json"] { + ensure!( + files[name] == pins["inputs"]["base"][name], + "expected pinned base {name}" + ); + } + ensure!( + files["head.safetensors"] + == "65698f2568fc1b6ebb60a1c2a24c7bb6622dcc23e1c6d9efe8d3274ad918a9a7", + "expected the pinned FP32 head" + ); + Ok(()) +} + +pub fn sha256(path: &Path) -> Result { + let mut file = File::open(path).with_context(|| format!("reading {}", path.display()))?; + let mut hash = Sha256::new(); + let mut buffer = vec![0u8; 1 << 20]; + loop { + let n = file.read(&mut buffer)?; + if n == 0 { + break; + } + hash.update(&buffer[..n]); + } + Ok(format!("{:x}", hash.finalize())) +} + +pub fn verify(dir: &Path) -> Result<()> { + let manifest: Value = serde_json::from_slice( + &std::fs::read(dir.join("kev_export.json")).context("run recipe/kev/export.py first")?, + )?; + validate_manifest(&manifest)?; + for (name, digest) in manifest["files"].as_object().unwrap() { + ensure!( + sha256(&dir.join(name))? == digest.as_str().unwrap(), + "export hash mismatch: {name}" + ); + } + ensure!( + !dir.join("model.safetensors").exists(), + "unlisted model.safetensors would shadow the verified shards" + ); + let index: Value = + serde_json::from_slice(&std::fs::read(dir.join("model.safetensors.index.json"))?)?; + let map = index["weight_map"] + .as_object() + .context("weight_map must be an object")?; + ensure!(!map.is_empty(), "empty weight map"); + for (name, shard) in map { + ensure!( + name.starts_with("model.language_model.") + && shard.as_str().is_some_and(|s| SHARDS.contains(&s)), + "invalid language weight map entry" + ); + } + // Check shapes without accepting a differently sized pointer head. + let bytes = std::fs::read(dir.join("head.safetensors"))?; + let tensors = safetensors::SafeTensors::deserialize(&bytes)?; + for name in ["q.weight", "k.weight"] { + ensure!( + tensors.tensor(name)?.shape() == [256, 2560], + "expected Kev-4B {name}" + ); + } + Ok(()) +} diff --git a/src/models/kev/native/src/contract.rs b/src/models/kev/native/src/contract.rs new file mode 100644 index 00000000..f45f1359 --- /dev/null +++ b/src/models/kev/native/src/contract.rs @@ -0,0 +1,171 @@ +//! Kev's API mapping, adapted from kev/api.py at the pinned reference revision. +//! See ../LICENSE.kev. Preserve insertion order and Python scalar rendering. +use anyhow::{Context, Result, bail, ensure}; +use omni_qwen3_5_native::json; +use serde_json::{Map, Value}; + +pub const MODEL: &str = "jaredpalmer/kev-4b"; +pub const MAX_QUESTIONS: usize = 64; +pub const MAX_STATE: usize = 8192; +pub const MAX_ROW: usize = 16384; +pub const MAX_REQUEST_TOKENS: usize = 65536; +pub const MAX_BODY: usize = 4 << 20; +pub const TEMPERATURE: f64 = 2.406050072164233; +pub const SPECIAL: [(&str, u32); 5] = [ + ("<|fim_prefix|>", 248060), + ("<|fim_middle|>", 248061), + ("<|box_start|>", 248049), + ("<|box_end|>", 248050), + ("<|fim_suffix|>", 248062), +]; + +#[derive(Debug, Clone, PartialEq)] +pub enum Kind { + Choice, + Noul, + Score, +} + +#[derive(Debug, Clone)] +pub struct Question { + pub id: String, + pub kind: Kind, + pub instructions: String, + pub options: Vec, + pub keys: Vec, + pub legend: Option, +} + +pub struct Request { + pub model: String, + pub state: String, + pub questions: Vec, +} + +pub fn render(value: &Value, indent: usize) -> String { + let pad = " ".repeat(indent); + match value { + Value::Null => String::new(), + Value::String(s) => s.clone(), + Value::Bool(b) => if *b { "True" } else { "False" }.into(), + Value::Number(_) => json::dumps(value), + Value::Array(items) => items + .iter() + .map(|v| { + format!( + "{pad}- {}", + render(v, indent + 1).trim_start_matches(|c: char| { + c.is_whitespace() || ('\u{1c}'..='\u{1f}').contains(&c) + }) + ) + }) + .collect::>() + .join("\n"), + Value::Object(items) => items + .iter() + .map(|(k, v)| { + if v.is_array() || v.is_object() { + format!("{pad}{k}:\n{}", render(v, indent + 1)) + } else { + format!("{pad}{k}: {}", render(v, 0)) + } + }) + .collect::>() + .join("\n"), + } +} + +fn option(name: &str, value: &Value) -> String { + if value.is_null() || value.as_str() == Some("") { + name.into() + } else { + format!("{name}: {}", render(value, 0)) + } +} + +pub fn compile(request: &Map) -> Result { + let model = match request.get("model") { + None => "kev-latest", + Some(value) => value.as_str().context("model must be a string")?, + }; + ensure!( + [MODEL, "kev-latest", "kev-4b"].contains(&model), + "requested model is not loaded" + ); + let state = render(request.get("state").context("request requires state")?, 0); + let definitions = request + .get("questions") + .and_then(Value::as_object) + .context("questions must be an object")?; + ensure!( + (1..=MAX_QUESTIONS).contains(&definitions.len()), + "questions must contain 1..={MAX_QUESTIONS} entries" + ); + let mut questions = Vec::with_capacity(definitions.len()); + for (id, value) in definitions { + let d = value + .as_object() + .context("question definition must be an object")?; + let criteria = d.get("criteria").unwrap_or(&Value::Null); + let (kind, keys, options, legend) = match d.get("type").and_then(Value::as_str) { + Some("choice") => { + let c = criteria + .as_object() + .context("Choice requires an object criteria")?; + ( + Kind::Choice, + c.keys().cloned().collect(), + c.iter().map(|(k, v)| option(k, v)).collect::>(), + None, + ) + } + Some("score") => { + let c = criteria + .as_array() + .context("Score requires an array criteria")?; + let keys: Vec = (0..c.len()).map(|i| i.to_string()).collect(); + let options: Vec = c.iter().map(|v| render(v, 0)).collect(); + let legend: Map = keys + .iter() + .cloned() + .zip(options.iter().cloned().map(Value::String)) + .collect(); + (Kind::Score, keys, options, Some(Value::Object(legend))) + } + Some("noul") => { + ensure!( + criteria.is_null() || criteria.is_object(), + "Noul criteria must be null or an object" + ); + // The pinned reference ignores other keys and fills absent descriptions. + let options = [("no", "false"), ("yes", "true")] + .map(|(name, key)| option(name, criteria.get(key).unwrap_or(&Value::Null))) + .to_vec(); + ( + Kind::Noul, + vec!["false".into(), "true".into()], + options, + None, + ) + } + _ => bail!("question type must be choice, noul, or score"), + }; + ensure!( + (1..=255).contains(&options.len()), + "question {id}: expected 1..=255 candidates" + ); + questions.push(Question { + id: id.clone(), + kind, + keys, + options, + legend, + instructions: render(d.get("instructions").unwrap_or(&Value::Null), 0), + }); + } + Ok(Request { + model: model.into(), + state, + questions, + }) +} diff --git a/src/models/kev/native/src/engine.rs b/src/models/kev/native/src/engine.rs new file mode 100644 index 00000000..99d41843 --- /dev/null +++ b/src/models/kev/native/src/engine.rs @@ -0,0 +1,64 @@ +//! Model ownership and request-sized serial FIFO execution. +use crate::{ + checkpoint, + executor::Executor, + processing::{Prepared, Processor}, +}; +use anyhow::Result; +use omni_runtime::{SerialScheduler, control::ExecutionContext}; +use serde_json::Value; +use std::{ + path::Path, + sync::{Arc, Mutex}, +}; + +pub struct Engine { + pub processor: Processor, + scheduler: SerialScheduler, + executor: Arc>, +} +impl Engine { + pub async fn load(dir: &Path, library: &Path) -> Result { + let (dir, library) = (dir.to_owned(), library.to_owned()); + let (processor, executor) = tokio::task::spawn_blocking(move || { + checkpoint::verify(&dir)?; + let processor = Processor::load(&dir)?; + let executor = Executor::load(&dir, &library)?; + Ok::<_, anyhow::Error>((processor, executor)) + }) + .await??; + Ok(Self { + processor, + scheduler: SerialScheduler::default(), + executor: Arc::new(Mutex::new(executor)), + }) + } + pub async fn infer(&self, prepared: Prepared, context: ExecutionContext) -> Result { + let executor = self.executor.clone(); + self.scheduler + .run_with_context(context, move || { + let logits = executor + .lock() + .map_err(|_| anyhow::anyhow!("poisoned Kev executor"))? + .execute(&prepared.rows)?; + prepared.context.finish(logits) + }) + .await + } + pub async fn warmup(&self) -> Result<()> { + let request = serde_json::json!({ + "state": "The light is green.", + "questions": { + "c": {"type":"choice", "criteria":{"red":null,"green":null}}, + "n": {"type":"noul", "instructions":"The light is green."}, + "s": {"type":"score", "criteria":["stop","wait","go"]} + } + }); + self.infer( + self.processor.prepare(request.as_object().unwrap())?, + ExecutionContext::default(), + ) + .await?; + Ok(()) + } +} diff --git a/src/models/kev/native/src/executor.rs b/src/models/kev/native/src/executor.rs new file mode 100644 index 00000000..0098fddc --- /dev/null +++ b/src/models/kev/native/src/executor.rs @@ -0,0 +1,112 @@ +//! State/question eager execution, shared Qwen backbone, Kev FP32 head. +use crate::{contract, head::PointerHead, processing::Row}; +use anyhow::{Context, Result, ensure}; +use omni_qwen3_5_native::{inputs::MultimodalInput, model::Model}; +use std::path::Path; + +pub struct Executor { + model: Model, + head: PointerHead, +} + +pub fn validate_row(row: &Row) -> Result<()> { + ensure!( + !row.ids.is_empty() && row.ids.len() <= contract::MAX_ROW, + "invalid row length" + ); + ensure!( + row.ids.iter().all(|&id| id < 248320), + "token outside Kev vocabulary" + ); + ensure!( + row.positions.iter().copied().eq(0..row.ids.len() as i64), + "expected consecutive text positions" + ); + ensure!( + (2..=256).contains(&row.readouts.len()), + "expected candidate and decision readouts" + ); + ensure!( + row.readouts.windows(2).all(|p| p[0] < p[1]) + && row.readouts.last() == Some(&(row.ids.len() - 1)), + "invalid readout order" + ); + for &i in &row.readouts[..row.readouts.len() - 1] { + ensure!( + row.ids.get(i) == Some(&248050), + "option readout must be at box_end" + ); + } + ensure!( + row.ids.last() == Some(&248062), + "decision readout must be at fim_suffix" + ); + Ok(()) +} + +impl Executor { + /// Caller has already verified the export; eager selection is explicit. + pub(crate) fn load(dir: &Path, library: &Path) -> Result { + let head = PointerHead::load(&dir.join("head.safetensors"))?; + let mut model = Model::load_with_graph(dir, library, false)?; + model.enable_service_readout_numerics()?; + Ok(Self { model, head }) + } + + pub fn execute(&mut self, rows: &[Row]) -> Result>> { + ensure!( + (1..=contract::MAX_QUESTIONS).contains(&rows.len()), + "invalid question count" + ); + for row in rows { + validate_row(row)?; + } + ensure!( + rows.iter().map(|r| r.ids.len()).sum::() <= contract::MAX_REQUEST_TOKENS, + "too many computed tokens" + ); + let plans = crate::planning::plan(rows)?; + let result = rows + .iter() + .enumerate() + .map(|(index, row)| { + let input = MultimodalInput { + token_ids: &row.ids, + image_token_indices: &[], + image_embeddings: &[], + position_ids: [&row.positions, &row.positions, &row.positions], + }; + let plan = plans[index]; + let read = self.model.forward_segmented_readout( + &input, + plan.split, + plan.state - plan.split, + plan.question - (row.ids.len() - plan.split), + plan.prefix - plan.split, + 248044, + plan.count, + plan.index, + &row.readouts, + )?; + let (decide, options) = read.hidden.split_last().context("empty readout")?; + self.head.logits( + decide, + &options.iter().map(Vec::as_slice).collect::>(), + ) + }) + .collect(); + // The permit and buffers remain owned until all submitted work has drained, + // including when a later row fails or its HTTP caller has disconnected. + let sync = self.model.synchronize(); + match result { + Ok(output) => { + sync?; + Ok(output) + } + Err(error) => { + let _ = sync; + Err(error) + } + } + } +} diff --git a/src/models/kev/native/src/head.rs b/src/models/kev/native/src/head.rs new file mode 100644 index 00000000..dc3eea2e --- /dev/null +++ b/src/models/kev/native/src/head.rs @@ -0,0 +1,102 @@ +//! Bilinear pointer readout from jaredpalmer/kev at 2074cc44256200bb37540103fa4572901faa10f3. +//! Learned projection stays separate from calibration and response interpretation. +use anyhow::{Context, Result, ensure}; +use safetensors::{Dtype, SafeTensors}; +use std::path::Path; + +pub struct PointerHead { + hidden: usize, + pointer: usize, + q: Vec, + k: Vec, + q_bias: Vec, + k_bias: Vec, +} + +fn tensor(tensors: &SafeTensors<'_>, name: &str, shape: &[usize]) -> Result> { + let view = tensors + .tensor(name) + .with_context(|| format!("missing {name}"))?; + ensure!( + view.dtype() == Dtype::F32 && view.shape() == shape, + "invalid {name} shape or dtype" + ); + let values: Vec = view + .data() + .as_chunks::<4>() + .0 + .iter() + .map(|&b| f32::from_le_bytes(b)) + .collect(); + ensure!(values.iter().all(|v| v.is_finite()), "non-finite {name}"); + Ok(values) +} + +fn project(weights: &[f32], bias: &[f32], input: &[f32]) -> Vec { + weights + .chunks_exact(input.len()) + .zip(bias) + .map(|(row, &bias)| row.iter().zip(input).map(|(&w, &x)| w * x).sum::() + bias) + .collect() +} + +impl PointerHead { + /// Load exactly q/k matrices and biases, preserving their FP32 representation. + /// Dimensions come from the head; Kev-4B exports use 2560 hidden / 256 pointer. + pub fn load(path: &Path) -> Result { + let bytes = std::fs::read(path)?; + let tensors = SafeTensors::deserialize(&bytes)?; + ensure!( + tensors.names().len() == 4, + "expected exactly four pointer-head tensors" + ); + let q = tensors.tensor("q.weight")?; + ensure!( + q.shape().len() == 2 && q.shape().iter().all(|&d| d > 0), + "invalid q.weight shape" + ); + let (pointer, hidden) = (q.shape()[0], q.shape()[1]); + Ok(Self { + hidden, + pointer, + q: tensor(&tensors, "q.weight", &[pointer, hidden])?, + k: tensor(&tensors, "k.weight", &[pointer, hidden])?, + q_bias: tensor(&tensors, "q.bias", &[pointer])?, + k_bias: tensor(&tensors, "k.bias", &[pointer])?, + }) + } + + /// h_decide is the final-normalized state at <|fim_suffix|>; each h_opt is + /// at an option's closing <|box_end|> in the SAME complete question row. + /// Returns one raw FP32 logit per option, in caller order. + pub fn logits(&self, h_decide: &[f32], h_opts: &[&[f32]]) -> Result> { + ensure!( + (1..=255).contains(&h_opts.len()), + "expected 1..=255 options" + ); + ensure!( + std::iter::once(h_decide) + .chain(h_opts.iter().copied()) + .all(|row| row.len() == self.hidden && row.iter().all(|v| v.is_finite())), + "invalid hidden-state shape or values" + ); + let query = project(&self.q, &self.q_bias, h_decide); + let scale = (self.pointer as f64).sqrt().recip() as f32; + let logits: Vec = h_opts + .iter() + .map(|row| { + project(&self.k, &self.k_bias, row) + .iter() + .zip(&query) + .map(|(&k, &q)| k * q) + .sum::() + * scale + }) + .collect(); + ensure!( + logits.iter().all(|v| v.is_finite()), + "non-finite pointer logits" + ); + Ok(logits) + } +} diff --git a/src/models/kev/native/src/http.rs b/src/models/kev/native/src/http.rs new file mode 100644 index 00000000..845d054d --- /dev/null +++ b/src/models/kev/native/src/http.rs @@ -0,0 +1,106 @@ +//! HTTP transport only; parsing, processing and answer semantics stay in Kev modules. +use crate::{contract, engine::Engine}; +use axum::{ + Json, Router, + body::Bytes, + extract::{DefaultBodyLimit, State, rejection::BytesRejection}, + http::{HeaderMap, StatusCode, header}, + response::{IntoResponse, Response}, + routing::{get, post}, +}; +use omni_runtime::{admission::BoundedAdmission, control::ExecutionContext}; +use serde_json::json; +use std::sync::Arc; + +#[derive(Clone)] +struct Shared { + engine: Arc, + admission: BoundedAdmission, +} + +pub fn error(status: StatusCode, detail: impl ToString) -> Response { + (status, Json(json!({"detail":detail.to_string()}))).into_response() +} + +async fn decide( + State(shared): State, + headers: HeaderMap, + body: Result, +) -> Response { + if !headers + .get(header::CONTENT_TYPE) + .and_then(|v| v.to_str().ok()) + .unwrap_or("") + .split(';') + .next() + .unwrap_or("") + .trim() + .eq_ignore_ascii_case("application/json") + { + return error( + StatusCode::UNSUPPORTED_MEDIA_TYPE, + "Content-Type must be application/json", + ); + } + let raw = match body { + Ok(b) => b, + Err(e) => return error(e.status(), e.body_text()), + }; + let Some(permit) = shared.admission.try_admit(raw.len()) else { + return ( + StatusCode::SERVICE_UNAVAILABLE, + [(header::RETRY_AFTER, "1")], + Json(json!({"detail":"worker is busy"})), + ) + .into_response(); + }; + let context = ExecutionContext::default().retain(permit); + let _cancel = context.cancel_on_drop(); + let processor = shared.engine.processor.clone(); + let preparing = context.clone(); + let prepared = tokio::task::spawn_blocking(move || { + let _hold = preparing; + let request = + omni_qwen3_5_native::json::parse(&raw).map_err(|e| (StatusCode::BAD_REQUEST, e))?; + processor + .prepare(&request) + .map_err(|e| (StatusCode::UNPROCESSABLE_ENTITY, e.to_string())) + }) + .await; + let prepared = match prepared { + Ok(Ok(p)) => p, + Ok(Err((status, detail))) => return error(status, detail), + Err(e) => { + eprintln!("Kev preparation failed: {e:#}"); + return error(StatusCode::INTERNAL_SERVER_ERROR, "preparation failed"); + } + }; + match shared.engine.infer(prepared, context).await { + Ok(body) => Json(body).into_response(), + Err(e) => { + eprintln!("Kev inference failed: {e:#}"); + error(StatusCode::INTERNAL_SERVER_ERROR, "model inference failed") + } + } +} + +/// Call only after Engine::warmup succeeds; the binary binds its listener afterward. +pub fn router(engine: Arc) -> Router { + let shared = Shared { + engine, + admission: BoundedAdmission::new(16, Some(16 * contract::MAX_BODY)), + }; + Router::new() + .route( + "/health", + get(|| async { + Json(json!({ + "status":"ready", "model":contract::MODEL, + "execution":"eager", "prefix_reuse":false + })) + }), + ) + .route("/v1/systemone", post(decide)) + .layer(DefaultBodyLimit::max(contract::MAX_BODY)) + .with_state(shared) +} diff --git a/src/models/kev/native/src/lib.rs b/src/models/kev/native/src/lib.rs new file mode 100644 index 00000000..108457a4 --- /dev/null +++ b/src/models/kev/native/src/lib.rs @@ -0,0 +1,14 @@ +//! Kev-4B: independent processing and calibrated answers around shared Qwen CUDA execution. +pub mod checkpoint; +pub mod contract; +pub mod engine; +pub mod executor; +pub mod head; +pub mod http; +mod planning; +pub mod postprocess; +pub mod processing; + +#[cfg(test)] +#[path = "../../../../../tests/kev/native.rs"] +mod native_tests; diff --git a/src/models/kev/native/src/main.rs b/src/models/kev/native/src/main.rs new file mode 100644 index 00000000..624afe24 --- /dev/null +++ b/src/models/kev/native/src/main.rs @@ -0,0 +1,24 @@ +//! KEV_MODEL= omni-kev-native +use anyhow::{Context, Result}; +use omni_kev_native::engine::Engine; +use omni_qwen3_5_native::cuda; +use std::sync::Arc; + +#[tokio::main] +async fn main() -> Result<()> { + let dir = + std::env::var_os("KEV_MODEL").context("set KEV_MODEL to the pinned export directory")?; + let library = std::env::var_os("KEV_CUDA_LIB") + .map(Into::into) + .map_or_else(cuda::default_library, Ok)?; + let engine = Arc::new(Engine::load(dir.as_ref(), &library).await?); + engine.warmup().await.context("Kev real warmup failed")?; + let host = std::env::var("KEV_HOST").unwrap_or_else(|_| "127.0.0.1".into()); + let port: u16 = std::env::var("KEV_PORT") + .map_or(Ok(8000), |v| v.parse()) + .context("KEV_PORT")?; + let listener = tokio::net::TcpListener::bind((host.as_str(), port)).await?; + println!("Kev ready on {host}:{port}"); + axum::serve(listener, omni_kev_native::http::router(engine)).await?; + Ok(()) +} diff --git a/src/models/kev/native/src/planning.rs b/src/models/kev/native/src/planning.rs new file mode 100644 index 00000000..a6705e70 --- /dev/null +++ b/src/models/kev/native/src/planning.rs @@ -0,0 +1,141 @@ +//! Question geometry adapted from kev/cuda_graphs.py at the pinned revision. +//! See ../LICENSE.kev. Preserve padding and GEMM shapes without retaining prefixes. +use crate::{contract, processing::Row}; +use anyhow::{Context, Result, ensure}; + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(crate) struct Plan { + pub split: usize, + pub state: usize, + pub question: usize, + pub prefix: usize, + pub count: usize, + pub index: usize, +} + +fn bucket(n: usize, steps: usize, floor: usize) -> usize { + let step = floor.max(n.next_power_of_two() / steps); + n.div_ceil(step) * step +} + +fn groups(lengths: &[usize]) -> Vec> { + let mut order: Vec<_> = (0..lengths.len()).collect(); + order.sort_by_key(|&i| lengths[i]); + let mut best = vec![usize::MAX; order.len() + 1]; + let mut cut = vec![0; order.len() + 1]; + best[0] = 0; + for j in 1..=order.len() { + for i in j.saturating_sub(32)..j { + let cost = + best[i] + 256.max(bucket(j - i, 4, 1) * bucket(lengths[order[j - 1]], 8, 16)); + if cost < best[j] { + best[j] = cost; + cut[j] = i; + } + } + } + let mut result = vec![]; + let mut j = order.len(); + while j > 0 { + result.push(order[cut[j]..j].to_vec()); + j = cut[j]; + } + result.reverse(); + result +} + +pub(crate) fn geometry(split: usize, lengths: &[usize]) -> Vec { + let state = if bucket(split, 8, 16) <= 1024 { + bucket(split, 8, 16) + } else { + split + }; + let longest = *lengths.iter().max().unwrap(); + let graphed = bucket(longest, 8, 16) <= 1024 && split <= 4096; + let prefix = if graphed { + split.max(16).next_power_of_two() + } else { + split + }; + let mut plans = vec![ + Plan { + split, + state, + prefix, + ..Plan::default() + }; + lengths.len() + ]; + let groups = if graphed { + groups(lengths) + } else { + let count = (16384 / (split + longest)).max(1); + (0..lengths.len()) + .collect::>() + .chunks(count) + .map(<[usize]>::to_vec) + .collect() + }; + for group in groups { + let longest = group.iter().map(|&i| lengths[i]).max().unwrap(); + let question = if graphed { + bucket(longest, 8, 16) + } else { + longest + }; + let limit = if graphed { + let cap = 32.min(32768 / (prefix + question)); + (1..=cap).filter(|&n| bucket(n, 4, 1) <= cap).max().unwrap() + } else { + group.len() + }; + for part in group.chunks(limit) { + let count = if graphed { + bucket(part.len(), 4, 1) + } else { + part.len() + }; + for (index, &i) in part.iter().enumerate() { + plans[i].question = question; + plans[i].count = count; + plans[i].index = index; + } + } + } + plans +} + +pub(crate) fn plan(rows: &[Row]) -> Result> { + ensure!( + (1..=contract::MAX_QUESTIONS).contains(&rows.len()), + "invalid question count" + ); + let split = rows[0] + .ids + .iter() + .position(|&id| id == 248061) + .context("missing question boundary")?; + ensure!( + (1..=contract::MAX_STATE).contains(&split), + "invalid state length" + ); + let state = &rows[0].ids[..split]; + for row in rows { + ensure!( + row.ids.len() > split && row.ids[..split] == *state && row.ids[split] == 248061, + "questions must share the same state and boundary" + ); + ensure!( + row.ids[split + 1..].iter().all(|&id| id != 248061), + "extra question boundary" + ); + } + Ok(geometry( + split, + &rows.iter().map(|r| r.ids.len() - split).collect::>(), + )) +} + +#[cfg(test)] +#[path = "../../../../../tests/kev/planning.rs"] +mod tests; diff --git a/src/models/kev/native/src/postprocess.rs b/src/models/kev/native/src/postprocess.rs new file mode 100644 index 00000000..b5fae4e4 --- /dev/null +++ b/src/models/kev/native/src/postprocess.rs @@ -0,0 +1,117 @@ +//! Kev checkpoint-temperature calibration and official response interpretation. +use anyhow::{Result, ensure}; + +pub fn probabilities(logits: &[f32], temperature: f64) -> Result> { + ensure!((1..=255).contains(&logits.len()), "expected 1..=255 logits"); + let temperature = temperature as f32; + ensure!( + temperature.is_finite() && temperature > 0.0, + "invalid temperature" + ); + let scaled: Vec = logits.iter().map(|v| v / temperature).collect(); + ensure!( + scaled.iter().all(|v| v.is_finite()), + "non-finite calibrated logits" + ); + let max = scaled.iter().copied().fold(f32::NEG_INFINITY, f32::max); + let mut values: Vec = scaled.iter().map(|v| (v - max).exp()).collect(); + let sum: f32 = values.iter().sum(); + values.iter_mut().for_each(|v| *v /= sum); + Ok(values) +} + +use crate::contract::{Kind, Question}; +use serde_json::{Map, Value, json}; + +/// Python round(float(x), 4), including ties to even, before wire serialization. +fn rounded(value: f64) -> f64 { + format!("{value:.4}") + .parse() + .expect("finite rounded scalar") +} + +pub fn answers(questions: &[Question], probabilities: &[Vec]) -> Result { + ensure!( + questions.len() == probabilities.len(), + "question probability count mismatch" + ); + let mut answers = Map::new(); + for (q, p) in questions.iter().zip(probabilities) { + ensure!( + p.len() == q.keys.len() + && !p.is_empty() + && p.iter().all(|x| x.is_finite() && *x >= 0.0 && *x <= 1.0), + "invalid probabilities" + ); + let total: f64 = p.iter().sum(); + ensure!((total - 1.0).abs() < 1e-4, "probabilities must sum to one"); + let mode = (1..p.len()).fold(0, |best, i| if p[i] > p[best] { i } else { best }); + let dist: Map = q + .keys + .iter() + .cloned() + .zip(p.iter().map(|&v| json!(rounded(v)))) + .collect(); + // Construct in the official insertion order: billing tokenization observes it. + let mut answer = Map::new(); + match q.kind { + Kind::Noul => { + answer.insert("type".into(), json!("noul")); + answer.insert("noul".into(), json!(rounded(p[1]))); + } + Kind::Choice => { + let n = p.len() as f64; + let confidence = if p.len() == 1 { + 1.0 + } else { + (p[mode] / total - 1.0 / n) / (1.0 - 1.0 / n) + }; + answer.insert("type".into(), json!("choice")); + answer.insert("choice".into(), json!(q.keys[mode])); + answer.insert("confidence".into(), json!(rounded(confidence))); + answer.insert("probabilities".into(), Value::Object(dist)); + } + Kind::Score => { + let n = p.len() as f64; + let score: f64 = p.iter().enumerate().map(|(i, &p)| i as f64 * p).sum(); + let confidence = if p.len() == 1 { + 1.0 + } else { + let d = (0..p.len()) + .map(|i| (i as f64 - (n - 1.0) / 2.0).abs()) + .sum::() + / n; + let distance = p + .iter() + .enumerate() + .map(|(i, p)| i.abs_diff(mode) as f64 * p / total) + .sum::(); + (1.0 - distance / d).max(0.0) + }; + answer.insert("type".into(), json!("score")); + answer.insert("score".into(), json!(rounded(score))); + answer.insert("legend".into(), q.legend.clone().unwrap_or(Value::Null)); + answer.insert("probabilities".into(), Value::Object(dist)); + answer.insert("confidence".into(), json!(rounded(confidence))); + } + } + answers.insert(q.id.clone(), Value::Object(answer)); + } + Ok(Value::Object(answers)) +} + +/// json.dumps(answers), ensure_ascii=True, as used by Kev's output_tokens. +pub fn billing_json(value: &Value) -> String { + use std::fmt::Write; + let mut out = String::new(); + for c in omni_qwen3_5_native::json::dumps(value).chars() { + if c.is_ascii() && c != '\u{7f}' { + out.push(c); + } else { + for unit in c.encode_utf16(&mut [0; 2]) { + write!(out, "\\u{unit:04x}").unwrap(); + } + } + } + out +} diff --git a/src/models/kev/native/src/processing.rs b/src/models/kev/native/src/processing.rs new file mode 100644 index 00000000..2b834663 --- /dev/null +++ b/src/models/kev/native/src/processing.rs @@ -0,0 +1,178 @@ +//! Independent preparation: Kev fragments, complete causal question rows and readouts. +use crate::contract::{self, Question, SPECIAL}; +use crate::postprocess; +use anyhow::{Result, ensure}; +use regex::Regex; +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value}; +use std::path::Path; +use std::sync::{Arc, LazyLock}; +use tokenizers::Tokenizer; + +static CONTROL: LazyLock = LazyLock::new(|| Regex::new(r"<\|([A-Za-z0-9_]+)\|>").unwrap()); + +pub fn sanitize(text: &str) -> String { + CONTROL.replace_all(text, "<¦${1}¦>").into_owned() +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Row { + pub ids: Vec, + pub positions: Vec, + /// Candidate closing markers in order, followed by the decision marker. + pub readouts: Vec, +} + +pub struct Prepared { + pub rows: Vec, + pub context: ResponseContext, +} + +pub struct ResponseContext { + pub model: String, + pub questions: Vec, + pub input_tokens: usize, + tokenizer: Arc, +} + +#[derive(Clone)] +pub struct Processor { + tokenizer: Arc, +} + +impl Processor { + pub fn load(dir: &Path) -> Result { + Self::new(Tokenizer::from_file(dir.join("tokenizer.json")).map_err(anyhow::Error::msg)?) + } + + pub fn new(mut tokenizer: Tokenizer) -> Result { + for (text, id) in SPECIAL { + ensure!( + tokenizer.token_to_id(text) == Some(id), + "base tokenizer delimiter mismatch: {text}" + ); + } + // The pinned Transformers 5.17 Qwen2Tokenizer rebuilds this component: + // its regex treats Unicode marks differently from the base JSON file. + // Match the loaded official tokenizer, including emoji variation selectors. + let pretokenizer: tokenizers::pre_tokenizers::PreTokenizerWrapper = serde_json::from_value( + serde_json::json!({ + "type": "Sequence", "pretokenizers": [ + {"type": "Split", "pattern": {"Regex": r"(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+"}, + "behavior": "Isolated", "invert": false}, + {"type": "ByteLevel", "add_prefix_space": false, + "trim_offsets": true, "use_regex": false} + ] + }), + )?; + tokenizer.with_pre_tokenizer(Some(pretokenizer)); + // Fragment encoding must never pad or truncate silently. + tokenizer.with_padding(None); + tokenizer + .with_truncation(None) + .map_err(anyhow::Error::msg)?; + Ok(Self { + tokenizer: Arc::new(tokenizer), + }) + } + + fn user_tokens(&self, text: &str) -> Result> { + Ok(self + .tokenizer + .encode(sanitize(text), false) + .map_err(anyhow::Error::msg)? + .get_ids() + .to_vec()) + } + + pub fn prepare(&self, request: &Map) -> Result { + let request = contract::compile(request)?; + let mut state = vec![SPECIAL[0].1]; + state.extend(self.user_tokens(&request.state)?); + ensure!( + state.len() <= contract::MAX_STATE, + "state exceeds {} tokens; no truncation", + contract::MAX_STATE + ); + let mut rows = Vec::with_capacity(request.questions.len()); + let mut input_tokens = state.len(); + let mut compute_tokens = 0; + for q in &request.questions { + let mut ids = state.clone(); + ids.push(SPECIAL[1].1); + ids.extend(self.user_tokens(&q.instructions)?); + let mut readouts = Vec::with_capacity(q.options.len() + 1); + for option in &q.options { + ids.push(SPECIAL[2].1); + ids.extend(self.user_tokens(option)?); + ids.push(SPECIAL[3].1); + readouts.push(ids.len() - 1); + ensure!( + ids.len() < contract::MAX_ROW, + "question {} exceeds {} row tokens", + q.id, + contract::MAX_ROW + ); + } + ids.push(SPECIAL[4].1); + readouts.push(ids.len() - 1); + ensure!( + ids.len() <= contract::MAX_ROW, + "question {} exceeds {} row tokens", + q.id, + contract::MAX_ROW + ); + compute_tokens += ids.len(); + ensure!( + compute_tokens <= contract::MAX_REQUEST_TOKENS, + "request exceeds {} computed row tokens", + contract::MAX_REQUEST_TOKENS + ); + input_tokens += ids.len() - state.len(); + let positions = (0..ids.len() as i64).collect(); + rows.push(Row { + ids, + positions, + readouts, + }); + } + Ok(Prepared { + rows, + context: ResponseContext { + model: request.model, + questions: request.questions, + input_tokens, + tokenizer: self.tokenizer.clone(), + }, + }) + } +} + +impl ResponseContext { + pub fn finish(self, logits: Vec>) -> Result { + ensure!( + logits.len() == self.questions.len(), + "question output count mismatch" + ); + let probs = logits + .iter() + .map(|v| { + postprocess::probabilities(v, contract::TEMPERATURE) + .map(|p| p.into_iter().map(f64::from).collect::>()) + }) + .collect::>>()?; + self.finish_probabilities(&probs) + } + + pub fn finish_probabilities(&self, probabilities: &[Vec]) -> Result { + let answers = postprocess::answers(&self.questions, probabilities)?; + let serialized = postprocess::billing_json(&answers); + let output_tokens = self + .tokenizer + .encode(serialized, false) + .map_err(anyhow::Error::msg)? + .len(); + Ok(serde_json::json!({"model":self.model,"answers":answers, + "usage":{"input_tokens":self.input_tokens,"output_tokens":output_tokens}})) + } +} diff --git a/src/models/qwen3_5/native/THIRD_PARTY_NOTICES.md b/src/models/qwen3_5/native/THIRD_PARTY_NOTICES.md index 70b115f1..0fc51ae7 100644 --- a/src/models/qwen3_5/native/THIRD_PARTY_NOTICES.md +++ b/src/models/qwen3_5/native/THIRD_PARTY_NOTICES.md @@ -28,3 +28,20 @@ The native vision geometry, rotary, block and merger execution in 2025 The Qwen team, Alibaba Group and the HuggingFace Inc. team, under the [Apache License, Version 2.0](licenses/APACHE-2.0). The implementation is adapted to Rust/CUDA optional separate FP32 visual LoRA execution and the shared 4B/27B layouts. + +## Kev default-service attention + +`src/backends/cuda/qwen3_5/service_readout.cu` instantiates the memory-efficient +attention templates from PyTorch 2.8.0 +`ATen/native/transformers/cuda/mem_eff_attention/kernel_forward.h`, Copyright +(c) Meta Platforms, Inc. and affiliates, under the retained +[PyTorch 2.8 license](licenses/PYTORCH-2.8-LICENSE). Its CUTLASS dependency is +pinned to `ad7b2f5e84fcfa124cb02b91d5bd26d238c0459e`, Copyright (c) 2017–2025 +NVIDIA CORPORATION & AFFILIATES, under the retained +[CUTLASS license](licenses/CUTLASS-LICENSE). + +The templates are compiled into the optional CUDA library; no Python or libtorch +runtime is linked. `recipe/kev/build_cuda.sh` places both license texts and +`KEV_CUDA_NOTICES.txt` beside the resulting library. Include these files when +redistributing that binary. The ordinary shared Qwen build does not instantiate +this optional attention kernel. diff --git a/src/models/qwen3_5/native/licenses/CUTLASS-LICENSE b/src/models/qwen3_5/native/licenses/CUTLASS-LICENSE new file mode 100644 index 00000000..47016fa7 --- /dev/null +++ b/src/models/qwen3_5/native/licenses/CUTLASS-LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: BSD-3-Clause + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this +list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/src/models/qwen3_5/native/licenses/PYTORCH-2.8-LICENSE b/src/models/qwen3_5/native/licenses/PYTORCH-2.8-LICENSE new file mode 100644 index 00000000..966a609b --- /dev/null +++ b/src/models/qwen3_5/native/licenses/PYTORCH-2.8-LICENSE @@ -0,0 +1,84 @@ +From PyTorch: + +Copyright (c) 2016- Facebook, Inc (Adam Paszke) +Copyright (c) 2014- Facebook, Inc (Soumith Chintala) +Copyright (c) 2011-2014 Idiap Research Institute (Ronan Collobert) +Copyright (c) 2012-2014 Deepmind Technologies (Koray Kavukcuoglu) +Copyright (c) 2011-2012 NEC Laboratories America (Koray Kavukcuoglu) +Copyright (c) 2011-2013 NYU (Clement Farabet) +Copyright (c) 2006-2010 NEC Laboratories America (Ronan Collobert, Leon Bottou, Iain Melvin, Jason Weston) +Copyright (c) 2006 Idiap Research Institute (Samy Bengio) +Copyright (c) 2001-2004 Idiap Research Institute (Ronan Collobert, Samy Bengio, Johnny Mariethoz) + +From Caffe2: + +Copyright (c) 2016-present, Facebook Inc. All rights reserved. + +All contributions by Facebook: +Copyright (c) 2016 Facebook Inc. + +All contributions by Google: +Copyright (c) 2015 Google Inc. +All rights reserved. + +All contributions by Yangqing Jia: +Copyright (c) 2015 Yangqing Jia +All rights reserved. + +All contributions by Kakao Brain: +Copyright 2019-2020 Kakao Brain + +All contributions by Cruise LLC: +Copyright (c) 2022 Cruise LLC. +All rights reserved. + +All contributions by Tri Dao: +Copyright (c) 2024 Tri Dao. +All rights reserved. + +All contributions by Arm: +Copyright (c) 2021, 2023-2024 Arm Limited and/or its affiliates + +All contributions from Caffe: +Copyright(c) 2013, 2014, 2015, the respective contributors +All rights reserved. + +All other contributions: +Copyright(c) 2015, 2016 the respective contributors +All rights reserved. + +Caffe2 uses a copyright model similar to Caffe: each contributor holds +copyright over their contributions to Caffe2. The project versioning records +all such contribution and copyright details. If a contributor wants to further +mark their specific copyright on a particular contribution, they should +indicate their copyright solely in the commit message of the change when it is +committed. + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +3. Neither the names of Facebook, Deepmind Technologies, NYU, NEC Laboratories America + and IDIAP Research Institute nor the names of its contributors may be + used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. diff --git a/src/models/qwen3_5/native/src/cuda.rs b/src/models/qwen3_5/native/src/cuda.rs index 612d49da..de9b1070 100644 --- a/src/models/qwen3_5/native/src/cuda.rs +++ b/src/models/qwen3_5/native/src/cuda.rs @@ -202,11 +202,15 @@ macro_rules! api { _lib: libloading::Library, vision_v2: Option, reference: Option, + torch28_reference: Option, $(pub $name: unsafe extern "C" fn($($ty),*) $(-> $ret)?,)* } impl Api { /// Language arithmetic does not require cuDNN patch convolution. + pub fn torch28_reference(&self) -> Result<&Reference> { + self.torch28_reference.as_ref().context("Kev requires the optional Torch 2.8 CUDA profile; run recipe/kev/build_cuda.sh") + } pub fn language_reference(&self) -> Result<&Reference> { self.reference.as_ref().context("missing framework language CUDA symbols; rebuild the library") } @@ -232,7 +236,8 @@ macro_rules! api { )* let vision_v2=VisionV2::resolve(&lib); let reference=Reference::resolve(&lib); - Ok(Self { _lib: lib, vision_v2, reference, $($name,)* }) + let torch28_reference=Reference::resolve_torch28(&lib); + Ok(Self { _lib: lib, vision_v2, reference, torch28_reference, $($name,)* }) } } }; @@ -343,14 +348,76 @@ type Fp32GdnRun = unsafe extern "C" fn( f32, Stream, ) -> c_int; +type Fp32GdnStateRun = unsafe extern "C" fn( + *mut c_void, + *const c_void, + *const c_void, + *const c_void, + *const f32, + *const c_void, + *mut c_void, + *mut f32, + c_int, + c_int, + c_int, + f32, + *const c_void, + *mut c_void, + Stream, +) -> c_int; /// Model-owned FP32 chunk solver. Destruction drains its last producer stream. pub struct Fp32Gdn { handle: *mut c_void, run: Fp32GdnRun, + with_state: Option, destroy: unsafe extern "C" fn(*mut c_void), size: unsafe extern "C" fn(c_int, c_int) -> usize, } impl Fp32Gdn { + /// # Safety + /// Same allocations and stream ownership as `run`; non-null state pointers + /// each hold heads * 128 * 128 FP32 entries, retained until completion. + #[allow(clippy::too_many_arguments)] + pub unsafe fn run_state( + &self, + q: *const c_void, + k: *const c_void, + v: *const c_void, + g: *const f32, + beta: *const c_void, + out: *mut c_void, + work: *mut f32, + t: c_int, + h: c_int, + hk: c_int, + scale: f32, + initial: *const c_void, + final_state: *mut c_void, + stream: Stream, + ) -> c_int { + let Some(run) = self.with_state else { + return 801; + }; + unsafe { + run( + self.handle, + q, + k, + v, + g, + beta, + out, + work, + t, + h, + hk, + scale, + initial, + final_state, + stream, + ) + } + } pub fn workspace_floats(&self, tokens: usize, heads: usize) -> Result { ensure!(tokens <= 65536 && heads <= 256, "FP32 GDN shape limit"); // SAFETY: this getter does not dereference memory. @@ -505,12 +572,25 @@ impl Api { } pub fn fp32_gdn(&self) -> Result { + self.make_fp32_gdn(false) + } + + pub fn torch28_gdn(&self) -> Result { + self.make_fp32_gdn(true) + } + + fn make_fp32_gdn(&self, torch28: bool) -> Result { + let name: &[u8] = if torch28 { + b"cs1_gdn_fp32_create_torch28\0" + } else { + b"cs1_gdn_fp32_create\0" + }; // SAFETY: these optional signatures match ops.h; the library lives in static API. let (create, destroy, run, size) = unsafe { ( *self ._lib - .get:: *mut c_void>(b"cs1_gdn_fp32_create\0")?, + .get:: *mut c_void>(name)?, *self ._lib .get::(b"cs1_gdn_fp32_destroy\0")?, @@ -528,6 +608,12 @@ impl Api { Ok(Fp32Gdn { handle, run, + with_state: unsafe { + self._lib + .get::(b"cs1_gdn_prefill_fp32_x\0") + .ok() + .map(|f| *f) + }, destroy, size, }) @@ -877,3 +963,52 @@ impl Drop for Graph { #[cfg(test)] #[path = "../../../../../tests/qwen3_5/device_cuda.rs"] mod device_tests; + +#[derive(Clone, Copy)] +pub struct ServiceReadout { + pub gated_norm: unsafe extern "C" fn( + *const c_void, + *const c_void, + c_int, + *const c_void, + *mut c_void, + c_int, + c_int, + c_int, + f32, + Stream, + ) -> c_int, + pub bytes: unsafe extern "C" fn(c_int, c_int, c_int) -> usize, + pub zero: unsafe extern "C" fn(*mut c_void, usize, Stream) -> c_int, + pub attention: unsafe extern "C" fn( + *const c_void, + *const c_void, + *const c_void, + c_int, + *const c_void, + *mut c_void, + c_int, + c_int, + c_int, + c_int, + c_int, + c_int, + c_int, + *mut c_void, + Stream, + ) -> c_int, +} +impl Api { + pub fn service_readout(&self) -> Result { + // SAFETY: additive signatures match ops.h; Api owns the library. + unsafe { + let _: Fp32GdnStateRun = *self._lib.get(b"cs1_gdn_prefill_fp32_x\0")?; + Ok(ServiceReadout { + gated_norm: *self._lib.get(b"cs1_torch28_gated_rms_norm\0")?, + bytes: *self._lib.get(b"cs1_service_attention_bytes\0")?, + zero: *self._lib.get(b"cs1_service_zero\0")?, + attention: *self._lib.get(b"cs1_service_attention\0")?, + }) + } + } +} diff --git a/src/models/qwen3_5/native/src/cuda/reference.rs b/src/models/qwen3_5/native/src/cuda/reference.rs index bd93b388..56148932 100644 --- a/src/models/qwen3_5/native/src/cuda/reference.rs +++ b/src/models/qwen3_5/native/src/cuda/reference.rs @@ -109,6 +109,17 @@ pub struct Reference { pub patch_destroy: unsafe extern "C" fn(*mut c_void) -> (), } impl Reference { + pub(super) fn resolve_torch28(lib: &libloading::Library) -> Option { + let mut r = Self::resolve(lib)?; + // SAFETY: optional signatures match ops.h; the Api retains the library. + unsafe { + r.rms_norm = *lib.get(b"cs1_torch28_rms_norm\0").ok()?; + r.add_rms_norm = *lib.get(b"cs1_torch28_add_rms_norm\0").ok()?; + r.attn_prep = *lib.get(b"cs1_torch28_attn_prep\0").ok()?; + } + Some(r) + } + pub(super) fn resolve(lib: &libloading::Library) -> Option { // SAFETY: signatures match the additive cs1_reference_* declarations in ops.h. unsafe { diff --git a/src/models/qwen3_5/native/src/inputs.rs b/src/models/qwen3_5/native/src/inputs.rs index a6bc918e..858efce1 100644 --- a/src/models/qwen3_5/native/src/inputs.rs +++ b/src/models/qwen3_5/native/src/inputs.rs @@ -180,9 +180,27 @@ pub(crate) fn rotary_tables( half: usize, theta: f64, sections: [usize; 3], +) -> (Vec, Vec) { + rotary_tables_with_precision(positions, half, theta, sections, false) +} + +/// Kev casts the entire loaded backbone (including inv_freq buffers) to BF16. +pub(crate) fn rotary_tables_with_precision( + positions: [&[i64]; 3], + half: usize, + theta: f64, + sections: [usize; 3], + bf16_inv: bool, ) -> (Vec, Vec) { let inv: Vec = (0..half) - .map(|i| 1f32 / (theta as f32).powf((2 * i) as f32 / (2 * half) as f32)) + .map(|i| { + let f = 1f32 / (theta as f32).powf((2 * i) as f32 / (2 * half) as f32); + if bf16_inv { + bf16::from_f32(f).to_f32() + } else { + f + } + }) .collect(); let mut cos = Vec::with_capacity(positions[0].len() * half * 2); let mut sin = Vec::with_capacity(cos.capacity()); diff --git a/src/models/qwen3_5/native/src/model.rs b/src/models/qwen3_5/native/src/model.rs index 05f8c6cb..305c9e48 100644 --- a/src/models/qwen3_5/native/src/model.rs +++ b/src/models/qwen3_5/native/src/model.rs @@ -13,6 +13,8 @@ //! once and every branch from the state after it (see `SharedPrompts`). mod matched; +mod service; +use service::ServiceWindow; use std::collections::{HashMap, VecDeque}; use std::ffi::c_void; @@ -23,7 +25,9 @@ use anyhow::{Context, Result, bail, ensure}; use serde_json::Value as Json; use crate::cuda::{self, DeviceBuffer, Stream, check}; -use crate::inputs::{DeviceMultimodalInput, MultimodalInput, rotary_tables}; +use crate::inputs::{ + DeviceMultimodalInput, MultimodalInput, rotary_tables, rotary_tables_with_precision, +}; use crate::lora::{self, Group}; const ALIGN: usize = 256; @@ -852,6 +856,7 @@ pub struct Model { matched_ref: usize, matched_handles: HashMap, reference: Option<&'static cuda::Reference>, + service_ops: Option, fp32_gdn: Option, batched_gdn: Option, packed_gemm: Option, @@ -1029,6 +1034,7 @@ impl Model { matched_ref: 0, matched_handles: HashMap::new(), reference: None, + service_ops: None, fp32_gdn: None, batched_gdn: None, packed_gemm: None, @@ -1603,6 +1609,7 @@ impl Model { None, self.gemm, projection, + None, )?; self.synchronize()?; state.initialized = true; @@ -1647,6 +1654,7 @@ impl Model { Some(&state), self.gemm, projection, + None, )?; hidden[index] = self.last_hidden(s, ids.len())?; } @@ -1865,6 +1873,7 @@ impl Model { prefix, gemm, None, + None, )?; if let Some(cap) = capture { cap.initialized = true; @@ -2001,11 +2010,12 @@ impl Model { } fn upload_positions(&self, s: &Scratch, positions: [&[i64]; 3]) -> Result<()> { - let (cos, sin) = rotary_tables( + let (cos, sin) = rotary_tables_with_precision( positions, self.cfg.rotary_half, self.cfg.rope_theta, self.cfg.mrope_section, + self.service_ops.is_some(), ); // SAFETY: tables contain at most s.cap rows of rotary_half BF16 values. unsafe { @@ -2072,6 +2082,10 @@ impl Model { len <= self.cfg.max_positions, "cached prefix exceeds the configured maximum length" ); + self.alloc_prefix_storage(len) + } + + fn alloc_prefix_storage(&self, len: usize) -> Result { let (kvrow, nfull, ngdn, state_bytes, tail_bytes) = self.prefix_layout(); let mut attn_kv = Vec::with_capacity(nfull); for _ in 0..nfull { @@ -2148,7 +2162,7 @@ impl Model { self.upload_positions(s, input.position_ids)?; self.embed_tokens(s, input.token_ids)?; self.overwrite_image_rows(s, input, 0)?; - self.run_window(s, 0, t, true, Some(state), None, self.gemm, None)?; + self.run_window(s, 0, t, true, Some(state), None, self.gemm, None, None)?; self.synchronize()?; state.initialized = true; Ok(()) @@ -2199,6 +2213,7 @@ impl Model { Some(prefix), self.gemm, None, + None, )?; self.last_hidden(s, tend) } @@ -2229,7 +2244,7 @@ impl Model { let gemm = self.fixed_gemm()?; let s = self.scratch.as_ref().unwrap(); self.embed_tokens(s, ids)?; - self.run_window(s, 0, t, false, None, None, gemm, None)?; + self.run_window(s, 0, t, false, None, None, gemm, None, None)?; self.last_hidden(s, t) } @@ -2256,7 +2271,17 @@ impl Model { let gemm = self.fixed_gemm()?; let s = self.scratch.as_ref().unwrap(); self.embed_tokens(s, ids)?; - self.run_window(s, 0, ids.len(), false, Some(&mut *state), None, gemm, None)?; + self.run_window( + s, + 0, + ids.len(), + false, + Some(&mut *state), + None, + gemm, + None, + None, + )?; self.synchronize() })(); if result.is_err() { @@ -2293,7 +2318,17 @@ impl Model { let gemm = self.fixed_gemm()?; let s = self.scratch.as_ref().unwrap(); self.embed_tokens_at(s, rest, prefix.len)?; - self.run_window(s, prefix.len, tend, false, None, Some(prefix), gemm, None)?; + self.run_window( + s, + prefix.len, + tend, + false, + None, + Some(prefix), + gemm, + None, + None, + )?; self.last_hidden(s, tend) })(); if result.is_err() { @@ -2371,7 +2406,7 @@ impl Model { let request = match request { Some(state) => { self.embed_tokens(s, &runs.prefix)?; - self.run_window(s, 0, p, false, Some(&mut *state), None, gemm, None)?; + self.run_window(s, 0, p, false, Some(&mut *state), None, gemm, None, None)?; Some(&*state) } None => None, @@ -2386,7 +2421,17 @@ impl Model { let state = group.as_deref_mut().unwrap(); state.len = start; self.embed_tokens_at(s, &run.prefix, p)?; - self.run_window(s, p, start, false, Some(&mut *state), request, gemm, None)?; + self.run_window( + s, + p, + start, + false, + Some(&mut *state), + request, + gemm, + None, + None, + )?; Some(&*state) }; let rows = run @@ -2395,7 +2440,7 @@ impl Model { .map(|b| { let end = start + b.len(); self.embed_tokens_at(s, b, start)?; - self.run_window(s, start, end, false, None, from, gemm, None)?; + self.run_window(s, start, end, false, None, from, gemm, None, None)?; self.last_hidden(s, end) }) .collect::>>()?; @@ -2762,6 +2807,10 @@ impl Model { bidirectional: bool, suffix: Option<&SuffixBatch<'_>>, ) -> Result<()> { + ensure!( + self.service_ops.is_none(), + "use segmented readout for the service profile" + ); let bidir = if bidirectional { ensure!( self.reference.is_none() && self.lora.is_none() && suffix.is_none(), @@ -3204,9 +3253,14 @@ impl Model { prefix: Option<&PrefixState>, gemm: *mut c_void, reuse: Option<(cuda::ReuseGemm, usize)>, + service: Option<&ServiceWindow>, ) -> Result<()> { ensure!( - self.fp32_gdn.is_none(), + self.service_ops.is_none() || service.is_some(), + "use segmented readout for the service profile" + ); + ensure!( + self.fp32_gdn.is_none() || service.is_some(), "FP32 GDN does not support prefix continuation" ); ensure!( @@ -3219,6 +3273,46 @@ impl Model { let st = self.stream; let rows = tend - qb; let project = |x, weight: &Tensor, y, group| -> Result<()> { + if let Some(plan) = service.and_then(|p| p.projection.as_ref()) { + let (n, k) = (weight.shape[0], weight.shape[1]); + let x_offset = plan.index * rows * k * BF16; + let y_offset = plan.index * rows * n * BF16; + // SAFETY: each temporary has projection_rows * max_projection_width + // elements. Preserve the reference GEMM shape and this row's offset; + // other independent matrix rows are zero and never read back. + return unsafe { + check( + (service.unwrap().ops.zero)(plan.input, plan.rows * k * BF16, st), + "zero projection padding", + )?; + cuda::copy_dd( + plan.input.wrapping_byte_add(x_offset), + s.at(x), + rows * k * BF16, + st, + )?; + check( + (cuda::api().cs1_gemm)( + gemm, + plan.input, + weight.ptr, + plan.output, + plan.rows as i32, + n as i32, + k as i32, + n as i32, + st, + ), + "service batch-shape projection", + )?; + cuda::copy_dd( + s.at(y), + plan.output.wrapping_byte_add(y_offset), + rows * n * BF16, + st, + ) + }; + } if let Some((api, original_rows)) = reuse { // SAFETY: compatibility was checked before prefix capture. Scratch // owns rows at the offsets passed by the unchanged layer layout. @@ -3304,6 +3398,22 @@ impl Model { s.gdn_in + qb * ldbb, &[rows], )?; + if let Some(plan) = service { + unsafe { + check( + (plan.ops.zero)(p(s.gdn_in + qb * ldbb), plan.leading * ldbb, st), + "mask leading GDN rows", + )?; + check( + (plan.ops.zero)( + p(s.gdn_in + (qb + plan.real_end) * ldbb), + (rows - plan.real_end) * ldbb, + st, + ), + "mask trailing GDN rows", + )?; + } + } let ld = ldb as i32; let z = s.gdn_in + w.conv * BF16; let b = z + vd * BF16; @@ -3314,7 +3424,7 @@ impl Model { .as_deref() .map_or(std::ptr::null_mut(), |cap| cap.gdn_state[la_i].at(0)); unsafe { - if qb > 0 { + if qb > 0 && service.is_none() { cuda::copy_dd( p(s.gdn_in + (qb - 3) * ldbb), prefix.unwrap().conv_tail[la_i].at(0), @@ -3337,18 +3447,38 @@ impl Model { // those three outputs so continuation rows see all taps. let conv_start = qb.saturating_sub(3); check( - (convolution)( - p(s.gdn_in + conv_start * ldbb), - ld, - la.conv.ptr, - p(s.lq + conv_start * kb), - p(s.lk + conv_start * kb), - p(s.lv + conv_start * vb), - (tend - conv_start) as i32, - kd as i32, - vd as i32, - st, - ), + if service.is_some() + && let Some(pre) = prefix + { + let ops = + cuda::api().shared_suffix().context("service convolution")?; + (ops.conv)( + p(s.gdn_in + qb * ldbb), + pre.conv_tail[la_i].at(0), + ld, + la.conv.ptr, + p(s.lq + qb * kb), + p(s.lk + qb * kb), + p(s.lv + qb * vb), + ti, + kd as i32, + vd as i32, + st, + ) + } else { + (convolution)( + p(s.gdn_in + conv_start * ldbb), + ld, + la.conv.ptr, + p(s.lq + conv_start * kb), + p(s.lk + conv_start * kb), + p(s.lv + conv_start * vb), + (tend - conv_start) as i32, + kd as i32, + vd as i32, + st, + ) + }, "gdn conv", )?; check( @@ -3367,26 +3497,62 @@ impl Model { "gdn gates", )?; check( - (cuda::api().cs1_gdn_prefill_x)( - p(s.lq + qb * kb), - p(s.lk + qb * kb), - p(s.lv + qb * vb), - p(s.g + qb * hv * F32).cast(), - p(s.beta + qb * hv * BF16), - p(s.lo + qb * vb), - p(s.workspace).cast(), - ti, - hv as i32, - cfg.lin_k_heads as i32, - (cfg.lin_k_dim as f32).powf(-0.5), - s_in, - s_out, - st, - ), + if let Some(gdn) = &self.fp32_gdn { + if service.is_some() { + gdn.run_state( + p(s.lq + qb * kb), + p(s.lk + qb * kb), + p(s.lv + qb * vb), + p(s.g + qb * hv * F32).cast(), + p(s.beta + qb * hv * BF16), + p(s.lo + qb * vb), + p(s.workspace).cast(), + ti, + hv as i32, + cfg.lin_k_heads as i32, + (cfg.lin_k_dim as f32).powf(-0.5), + s_in, + s_out, + st, + ) + } else { + gdn.run( + p(s.lq + qb * kb), + p(s.lk + qb * kb), + p(s.lv + qb * vb), + p(s.g + qb * hv * F32).cast(), + p(s.beta + qb * hv * BF16), + p(s.lo + qb * vb), + p(s.workspace).cast(), + ti, + hv as i32, + cfg.lin_k_heads as i32, + (cfg.lin_k_dim as f32).powf(-0.5), + st, + ) + } + } else { + (cuda::api().cs1_gdn_prefill_x)( + p(s.lq + qb * kb), + p(s.lk + qb * kb), + p(s.lv + qb * vb), + p(s.g + qb * hv * F32).cast(), + p(s.beta + qb * hv * BF16), + p(s.lo + qb * vb), + p(s.workspace).cast(), + ti, + hv as i32, + cfg.lin_k_heads as i32, + (cfg.lin_k_dim as f32).powf(-0.5), + s_in, + s_out, + st, + ) + }, "gdn prefill", )?; check( - (cuda::api().cs1_gated_rms_norm)( + (service.map_or(cuda::api().cs1_gated_rms_norm, |p| p.ops.gated_norm))( p(s.lo + qb * vb), p(z + qb * ldbb), ld, @@ -3468,21 +3634,41 @@ impl Model { cuda::copy2d(cap.attn_kv[fa_i].1.at(0), kvb, p(v), ab, kvb, tend, st)?; } check( - (cuda::api().cs1_attention_gated_prefix)( - p(s.aq), - p(s.ak), - p(v), - w.attn_in as i32, - p(s.agate), - p(s.ao), - t_flash, - hq, - hk, - hd, - scale, - q_base, - st, - ), + if let Some(plan) = service { + (plan.ops.attention)( + p(s.aq + qb * ob), + p(s.ak), + p(v), + w.attn_in as i32, + p(s.agate + qb * ob), + p(s.ao + qb * ob), + ti, + tend as i32, + hq, + hk, + qb as i32, + plan.key_left as i32, + plan.real_end as i32, + plan.workspace, + st, + ) + } else { + (cuda::api().cs1_attention_gated_prefix)( + p(s.aq), + p(s.ak), + p(v), + w.attn_in as i32, + p(s.agate), + p(s.ao), + t_flash, + hq, + hk, + hd, + scale, + q_base, + st, + ) + }, "gated attention", )?; } @@ -3498,6 +3684,7 @@ impl Model { fa_i += 1; } } + unsafe { check( (self @@ -3515,6 +3702,7 @@ impl Model { "post-attention norm", )?; } + project( s.x + qb * hb, &layer.gate_up, @@ -3556,6 +3744,7 @@ impl Model { s.delta + qb * hb, &[rows], )?; + let next = self .layers .get(i + 1) @@ -3578,6 +3767,7 @@ impl Model { )?; } } + Ok(()) } } diff --git a/src/models/qwen3_5/native/src/model/service.rs b/src/models/qwen3_5/native/src/model/service.rs new file mode 100644 index 00000000..4e9677ae --- /dev/null +++ b/src/models/qwen3_5/native/src/model/service.rs @@ -0,0 +1,279 @@ +//! Eager segmented text readout with the pinned Torch 2.8 arithmetic. +//! Each call owns temporary state; no prefix survives across calls. +use super::*; + +pub(super) struct ServiceWindow { + pub ops: cuda::ServiceReadout, + pub leading: usize, + pub real_end: usize, + pub key_left: usize, + pub workspace: *mut c_void, + pub projection: Option, +} + +pub(super) struct ProjectionShape { + pub rows: usize, + pub index: usize, + pub input: *mut c_void, + pub output: *mut c_void, +} + +impl Model { + /// Select Kev's pinned default-service arithmetic before any execution. + /// The optional CUDA symbols leave other Qwen consumers on their old profile. + pub fn enable_service_readout_numerics(&mut self) -> Result<()> { + ensure!( + self.cfg.hidden == 2560 + && self.cfg.head_dim == 256 + && self.cfg.lin_k_dim == 128 + && self.cfg.lin_v_dim == 128 + && self.cfg.lin_k_heads == 16 + && self.cfg.lin_v_heads == 32, + "service readout requires the pinned 4B layout" + ); + ensure!( + self.scratch.is_none() + && self.lora.is_none() + && self.reference.is_none() + && self.fp32_gdn.is_none() + && !self.graph_enabled + && self.batched_gdn.is_none() + && self.packed_gemm.is_none(), + "select service readout on a merged eager model before its first forward" + ); + cuda::set_device(0)?; + let ops = cuda::api() + .service_readout() + .context("Kev requires recipe/kev/build_cuda.sh")?; + ensure!( + cuda::api().shared_suffix().is_some(), + "missing prefix convolution CUDA symbol" + ); + let gdn = cuda::api().torch28_gdn()?; + let reference = cuda::api().torch28_reference()?; + self.synchronize()?; + // SAFETY: no queued work/capture uses the old model-owned handle. + let gemm = unsafe { (reference.gemm_create)(GEMM_WORKSPACE) }; + ensure!(!gemm.is_null(), "service cuBLASLt setup failed"); + unsafe { (cuda::api().cs1_gemm_destroy)(self.gemm) }; + self.gemm = gemm; + self.reference = Some(reference); + self.fp32_gdn = Some(gdn); + self.service_ops = Some(ops); + Ok(()) + } + + /// Compute state then question with explicit padding, returning selected + /// question positions in caller order. All temporary allocations are drained + /// on success and failure before their ownership ends. + #[allow(clippy::too_many_arguments)] + pub fn forward_segmented_readout( + &mut self, + input: &MultimodalInput<'_>, + split: usize, + state_left_pad: usize, + branch_right_pad: usize, + prefix_left_pad: usize, + pad_id: u32, + batch_rows: usize, + batch_index: usize, + readouts: &[usize], + ) -> Result { + let ops = self + .service_ops + .context("select service readout numerics first")?; + input.validate( + self.cfg.hidden, + self.embed.shape[0], + self.cfg.image_token_id.unwrap_or(u32::MAX), + self.cfg.max_positions, + )?; + ensure!( + input + .position_ids + .iter() + .all(|p| *p == input.position_ids[0]), + "segmented readout requires text positions" + ); + ensure!( + (pad_id as usize) < self.embed.shape[0], + "padding token outside vocabulary" + ); + ensure!( + split > 0 && split < input.token_ids.len(), + "invalid state split" + ); + ensure!( + input.image_token_indices.is_empty() && input.image_embeddings.is_empty(), + "text readout only" + ); + ensure!( + readouts + .iter() + .all(|&r| r >= split && r < input.token_ids.len()), + "readout outside question" + ); + let state_len = split + .checked_add(state_left_pad) + .context("state length overflow")?; + let question_len = input.token_ids.len() - split; + let rows = question_len + .checked_add(branch_right_pad) + .context("question length overflow")?; + let prefix_len = split + .checked_add(prefix_left_pad) + .context("prefix length overflow")?; + let total = prefix_len + .checked_add(rows) + .context("total length overflow")?; + ensure!( + state_len >= 3 + && total <= self.cfg.max_positions + && state_len <= self.cfg.max_positions, + "segment capacity" + ); + ensure!(state_len <= 65536 && total <= 65536, "service shape limit"); + self.prepare_scratch(total.max(state_len))?; + let mut state = self.alloc_prefix_storage(state_len.max(prefix_len))?; + state.len = state_len; + // SAFETY: pure size queries for the checked nonempty extents. + let size = unsafe { + (ops.bytes)(state_len as i32, state_len as i32, self.cfg.heads as i32).max((ops.bytes)( + rows as i32, + total as i32, + self.cfg.heads as i32, + )) + }; + ensure!(size > 0, "invalid service workspace size"); + let workspace = DeviceBuffer::new(size)?; + ensure!( + (1..=32).contains(&batch_rows) && batch_index < batch_rows, + "invalid projection batch" + ); + let projection_rows = rows + .checked_mul(batch_rows) + .context("projection shape overflow")?; + ensure!(projection_rows <= 65536, "projection shape limit"); + let width = self + .layers + .iter() + .flat_map(|l| [&l.gate_up, &l.down]) + .flat_map(|w| w.shape.iter().copied()) + .max() + .unwrap() + .max(Widths::of(&self.cfg).gdn_in) + .max(Widths::of(&self.cfg).attn_in); + let projection = if batch_rows > 1 { + Some(( + DeviceBuffer::new(projection_rows * width * BF16)?, + DeviceBuffer::new(projection_rows * width * BF16)?, + )) + } else { + None + }; + let result = (|| { + let s = self.scratch.as_ref().unwrap(); + let mut ids = vec![pad_id; state_left_pad]; + ids.extend_from_slice(&input.token_ids[..split]); + let mut pos = vec![0; state_left_pad]; + pos.extend_from_slice(&input.position_ids[0][..split]); + self.upload_positions(s, [&pos, &pos, &pos])?; + self.embed_tokens_at(s, &ids, 0)?; + let plan = ServiceWindow { + ops, + leading: state_left_pad, + real_end: state_len, + key_left: state_left_pad, + workspace: workspace.at(0), + projection: None, + }; + self.run_window( + s, + 0, + state_len, + true, + Some(&mut state), + None, + self.gemm, + None, + Some(&plan), + )?; + // Keep the real prefix KV, right-aligned in the continuation's key extent. + let kb = self.cfg.kv_heads * self.cfg.head_dim * BF16; + for (k, v) in &state.attn_kv { + for buffer in [k, v] { + // SAFETY: state buffers hold cap * kv width; scratch holds split rows. + unsafe { + cuda::copy_dd( + s.at(s.ak), + buffer.at(state_left_pad * kb), + split * kb, + self.stream, + )?; + check( + (ops.zero)(buffer.at(0), state.cap * kb, self.stream), + "clear padding keys", + )?; + cuda::copy_dd( + buffer.at(prefix_left_pad * kb), + s.at(s.ak), + split * kb, + self.stream, + )?; + } + } + } + state.len = prefix_len; + state.initialized = true; + let mut ids = input.token_ids[split..].to_vec(); + ids.resize(rows, pad_id); + let mut pos = input.position_ids[0][split..].to_vec(); + pos.resize(rows, 0); + self.upload_positions(s, [&pos, &pos, &pos])?; + self.embed_tokens_at(s, &ids, prefix_len)?; + let plan = ServiceWindow { + ops, + leading: 0, + real_end: question_len, + key_left: prefix_left_pad, + workspace: workspace.at(0), + projection: projection.as_ref().map(|(x, y)| ProjectionShape { + rows: projection_rows, + index: batch_index, + input: x.at(0), + output: y.at(0), + }), + }; + self.run_window( + s, + prefix_len, + total, + true, + None, + Some(&state), + self.gemm, + None, + Some(&plan), + )?; + let hidden = readouts + .iter() + .map(|&r| self.hidden_row(s, r - split + prefix_len)) + .collect::>>()?; + Ok(Readout { + hidden, + logprobs: vec![], + }) + })(); + let drained = self.synchronize(); + match result { + Ok(r) => { + drained?; + Ok(r) + } + Err(e) => { + let _ = drained; + Err(e) + } + } + } +} diff --git a/tests/kev/boundary_discovery.py b/tests/kev/boundary_discovery.py new file mode 100644 index 00000000..c745bbfb --- /dev/null +++ b/tests/kev/boundary_discovery.py @@ -0,0 +1,110 @@ +"""Freeze deduplicated development Noul inputs before official-only discovery.""" +import argparse +import collections +import json +from pathlib import Path + +from service_acceptance import GATES, REFERENCE, ROOT, digest, module, save + + +def source_identity(case, question_id): + if "origin" in case: + return tuple(case["origin"][k] for k in ("record_id", "question_id")) + # Existing acceptance names are suite:source-record-id; synthetic names have no colon. + if ":" in case["name"]: + return case["name"].split(":", 1)[1], question_id + return None + + +def previous_keys(corpora): + identities, tokens = set(), set() + for corpus in corpora: + if corpus["gates"] != GATES: + raise ValueError("previous corpus gates changed") + for case in corpus["cases"]: + for meta, row in zip(case["meta"], case["rows"], strict=True): + if meta["type"] == "noul": + key = source_identity(case, meta["id"]) + if key: + identities.add(key) + tokens.add(tuple(row["ids"])) + return identities, tokens + + +def prepare(args): + if args.out.exists(): + raise ValueError("preserve previous discovery corpus") + args.out.mkdir(parents=True) + pins = json.loads((ROOT / "tests/kev/data/boundary-sources.json").read_text()) + if pins["reference"] != REFERENCE: + raise ValueError("reference pin mismatch") + for name, sha in pins["sources"].items(): + if digest(args.reference / name) != sha: + raise ValueError(f"changed development source: {name}") + fixture = module("kev_fixture", ROOT / "tests/kev/generate_processing_fixture.py") + api, model = fixture.official(args.reference) + tok = model.load_tokenizer(args.base) + seen, token_rows = previous_keys([json.loads(p.read_text()) for p in args.previous]) + captured, exclusions = [], [] + for source in pins["sources"]: + records = [json.loads(line) for line in (args.reference / source).read_text().splitlines() if line] + for record in records: + meta = record["_meta"] + if meta.get("variant") != "clean": + continue + for qid, question in record["questions"].items(): + if question["type"] != "noul": + continue + origin = {"file": source, "record_id": meta["id"], "question_id": qid, + "source": meta["source"], "group_id": meta.get("group_id", meta["id"])} + identity = meta["id"], qid + if identity in seen: + exclusions.append({**origin, "reason": "existing_source_question"}) + continue + seen.add(identity) + request = {"model": "kev-latest", "state": record["state"], "questions": { + qid: {k: v for k, v in question.items() if k in ("type", "instructions", "criteria")}}} + case = {"name": f"noul-v4:{source}:{meta['id']}:{qid}", "population": "discovery", + "source": meta["source"], "group": meta.get("group_id", meta["id"]), + "labels": {}, "source_label": question.get("label"), "origin": origin, "request": request} + # Strict encoding does not truncate. Length exclusions happen before any model run. + try: + item = fixture.capture(case, api, model, tok) + except model.ContextOverflow as error: + exclusions.append({**origin, "reason": "encoder_context_limit", "detail": str(error)}) + continue + ids = tuple(item["rows"][0]["ids"]) + split = ids.index(248061) + if split > 1024 or len(ids)-split > 1024: + exclusions.append({**origin, "reason": "discovery_length_budget", "state": split, "branch": len(ids)-split}) + continue + if ids in token_rows: + exclusions.append({**origin, "reason": "identical_token_row"}) + continue + token_rows.add(ids) + # capture() includes artificial CPU probabilities for postprocess tests only; + # remove those from the discovery corpus so membership cannot use them. + for key in ("probabilities", "answers", "output_tokens"): + item.pop(key) + captured.append(item) + if not captured or len({c["name"] for c in captured}) != len(captured): + raise ValueError("empty or duplicate discovery corpus") + corpus = {"reference": REFERENCE, "sources": pins["sources"], "gates": GATES, + "counts": {"discovery": len(captured)}, "cases": captured} + save(args.out / "corpus.json", corpus) + save(args.out / "exclusions.json", exclusions) + save(args.out / "freeze.json", { + "generator_sha256": digest(__file__), "source_manifest_sha256": digest(ROOT / "tests/kev/data/boundary-sources.json"), + "corpus_sha256": digest(args.out / "corpus.json"), "requests": len(captured), + "previous": {str(p): digest(p) for p in args.previous}, + "source_counts": collections.Counter(c["source"] for c in captured), + "exclusion_counts": collections.Counter(e["reason"] for e in exclusions)}) + print(json.dumps(json.loads((args.out / "freeze.json").read_text()), indent=2)) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + for flag in ("reference", "base", "out"): + parser.add_argument("--"+flag, type=Path, required=True) + parser.add_argument("--previous", type=Path, action="append", required=True) + prepare(parser.parse_args()) diff --git a/tests/kev/checkpoint.rs b/tests/kev/checkpoint.rs new file mode 100644 index 00000000..a964cf40 --- /dev/null +++ b/tests/kev/checkpoint.rs @@ -0,0 +1,74 @@ +use omni_kev_native::checkpoint::{PINS, validate_manifest}; +use serde_json::{Value, json}; + +fn manifest() -> Value { + let mut value: Value = serde_json::from_str(PINS).unwrap(); + let mut files = serde_json::Map::new(); + for name in ["config.json", "tokenizer.json", "tokenizer_config.json"] { + files.insert(name.into(), value["inputs"]["base"][name].clone()); + } + for name in [ + "head.safetensors", + "model.safetensors.index.json", + "model.safetensors-00001-of-00002.safetensors", + "model.safetensors-00002-of-00002.safetensors", + ] { + files.insert(name.into(), json!("0".repeat(64))); + } + files.insert( + "head.safetensors".into(), + json!("65698f2568fc1b6ebb60a1c2a24c7bb6622dcc23e1c6d9efe8d3274ad918a9a7"), + ); + value["files"] = Value::Object(files); + value +} +#[test] +fn pinned_metadata_and_complete_inventory_are_required() { + let good = manifest(); + validate_manifest(&good).unwrap(); + for key in [ + "revision", + "base_revision", + "reference_revision", + "temperature", + "special_tokens", + "head_dtype", + "hidden_size", + "merged_targets", + "inputs", + "merge_numerics", + ] { + let mut bad = good.clone(); + bad[key] = Value::Null; + assert!(validate_manifest(&bad).is_err(), "{key}"); + } + let mut bad = good.clone(); + bad["files"] + .as_object_mut() + .unwrap() + .remove("head.safetensors"); + assert!(validate_manifest(&bad).is_err()); + let mut bad = good; + bad["files"]["../other"] = json!("0".repeat(64)); + assert!(validate_manifest(&bad).is_err()); +} + +#[test] +fn corrupted_export_fails_before_device_loading() { + let root = std::env::temp_dir().join(format!("kev-bad-export-{}", std::process::id())); + std::fs::create_dir(&root).unwrap(); + std::fs::write( + root.join("kev_export.json"), + serde_json::to_vec(&manifest()).unwrap(), + ) + .unwrap(); + std::fs::write(root.join("config.json"), b"{}").unwrap(); + let error = omni_kev_native::checkpoint::verify(&root) + .unwrap_err() + .to_string(); + std::fs::remove_dir_all(&root).unwrap(); + assert!( + error.contains("export hash mismatch: config.json"), + "{error}" + ); +} diff --git a/tests/kev/compare_service.py b/tests/kev/compare_service.py new file mode 100644 index 00000000..c8d07996 --- /dev/null +++ b/tests/kev/compare_service.py @@ -0,0 +1,217 @@ +"""Compare the frozen default BF16 service campaign; never infer new gates.""" +import argparse +import collections +import json +import math +from pathlib import Path + +from service_acceptance import GATES, ROOT, digest, module, read_lines, save + + +def distribution(p, size): + return (len(p) == size and all(isinstance(x, (int, float)) and math.isfinite(x) and 0 <= x <= 1 for x in p) + and abs(sum(p)-1) <= 1e-4) + + +def question(meta, p, n, official, native): + """Independent probability, scalar and decision gates, including wire rounding.""" + kind, size = meta['type'], len(meta['keys']) + if not distribution(p, size) or not distribution(n, size): + return {'failures': ['distribution'], 'type': kind} + error = max(abs(a-b) for a, b in zip(p, n)) + result = {'type': kind, 'levels': size, 'probability_max_abs': error, 'failures': []} + if error > GATES['probability_max_abs']: + result['failures'].append('probability') + if kind == 'score': + ref_score, native_score = (sum(i*x for i, x in enumerate(row)) for row in (p, n)) + raw_error = abs(ref_score-native_score) + wire_error = abs(official['score']-native['score']) + result.update(official_score=ref_score, native_score=native_score, score_abs=raw_error, + score_wire_abs=wire_error, score_normalized_abs=raw_error/(size-1) if size > 1 else 0) + if max(raw_error, wire_error) > GATES['score_max_abs'] or (size == 1 and max(abs(official['score']), abs(native['score'])) != 0): + result['failures'].append('score') + else: + if kind == 'choice': + ordered = sorted(p, reverse=True) + distance = ordered[0]-ordered[1] if size > 1 else 1.0 + limit = GATES['choice_margin'] + a, b = (max(range(size), key=row.__getitem__) for row in (p, n)) + wire_flip = official['choice'] != native['choice'] + else: + distance, limit = abs(p[1]-0.5), GATES['noul_distance'] + a, b = p[1] >= 0.5, n[1] >= 0.5 + wire_flip = (official['noul'] >= 0.5) != (native['noul'] >= 0.5) + result.update(distance=distance, near_boundary=distance <= limit, raw_flip=a != b, wire_flip=wire_flip) + if distance > limit and (a != b or wire_flip): + result['failures'].append('decision') + return result + + +def labeled_metrics(kind, p, gold): + if not 0 <= gold < len(p): + raise ValueError('invalid frozen gold label') + result = {'accuracy': float(max(range(len(p)), key=p.__getitem__) == gold), + 'brier': sum((x-float(i == gold))**2 for i, x in enumerate(p)), + 'nll': -math.log(max(p[gold], 1e-9))} + if kind == 'score': + error = abs(sum(i*x for i, x in enumerate(p))-gold) + result.update(score_mae=error, normalized_score_mae=error/max(1, len(p)-1)) + return result + + +def aggregate(rows): + import numpy as np + result = {'questions': len(rows)} + for metric in ('accuracy', 'brier', 'nll', 'score_mae', 'normalized_score_mae'): + chosen = [r for r in rows if metric in r['official']] + if not chosen: + continue + a = sum(r['official'][metric] for r in chosen)/len(chosen) + b = sum(r['native'][metric] for r in chosen)/len(chosen) + # Pair and resample whole source records, retaining all questions in a record. + groups = collections.defaultdict(list) + for r in chosen: + groups[r['group']].append(r['native'][metric]-r['official'][metric]) + sums = np.array([sum(g) for g in groups.values()]) + counts = np.array([len(g) for g in groups.values()]) + rng = np.random.default_rng(27) + draws = rng.integers(0, len(groups), size=(2000, len(groups))) + boot = sums[draws].sum(axis=1)/counts[draws].sum(axis=1) + result[metric] = {'official': a, 'native': b, 'native_minus_official': b-a, + 'paired_record_bootstrap_95pct': np.quantile(boot, [0.025, 0.975]).tolist(), + 'questions': len(chosen), 'groups': len(groups)} + return result + + +def compare(corpus, official, native): + expected = [c['name'] for c in corpus['cases']] + if len(set(expected)) != len(expected) or any([c['name'] for c in rows] != expected for rows in (official, native)): + raise ValueError('missing, duplicate, or reordered cases') + if corpus['gates'] != GATES: + raise ValueError('gates changed after freeze') + results, metrics, errors, boundaries = [], [], [], [] + for case, ref, nat in zip(corpus['cases'], official, native): + metas = case['meta'] + ids = [m['id'] for m in metas] + for record in (ref, nat): + if len(record['probabilities']) != len(metas) or list(record['body']['answers']) != ids: + raise ValueError(f'question count/order: {case["name"]}') + if record['body']['usage']['input_tokens'] != case['input_tokens']: + errors.append({'case': case['name'], 'failure': 'input_tokens'}) + for meta, p, n in zip(metas, ref['probabilities'], nat['probabilities']): + qid, kind = meta['id'], meta['type'] + row = {'case': case['name'], 'question': qid, 'source': case['source'], 'population': case['population'], + **question(meta, p, n, ref['body']['answers'][qid], nat['body']['answers'][qid])} + if row['failures']: + errors.append(row) + if row.get('near_boundary'): + boundaries.append({**row, 'official_probabilities': p, 'native_probabilities': n, + 'official_answer': ref['body']['answers'][qid], 'native_answer': nat['body']['answers'][qid]}) + results.append(row) + if case['population'] == 'task' and 'distribution' not in row['failures']: + metrics.append({'case': case['name'], 'question': qid, 'type': kind, 'source': case['source'], + 'group': case['name'].split(':')[0]+':'+case['group'], + 'gold': case['labels'][qid], + 'official': labeled_metrics(kind, p, case['labels'][qid]), + 'native': labeled_metrics(kind, n, case['labels'][qid])}) + quality = {'all': aggregate(metrics), + 'by_type': {k: aggregate([r for r in metrics if r['type'] == k]) for k in ('choice', 'noul', 'score')}, + 'by_source': {k: aggregate([r for r in metrics if r['source'] == k]) for k in sorted({r['source'] for r in metrics})}} + for group, values in [('all', quality['all']), *quality['by_type'].items()]: + if not values['questions']: + errors.append({'failure': 'missing_task_type', 'group': group}) + continue + limit = GATES['accuracy_drop' if group == 'all' else 'per_type_accuracy_drop'] + if values['accuracy']['native_minus_official'] < -limit: + errors.append({'failure': 'task_accuracy', 'group': group, 'limit': limit}) + if quality['all'].get('brier', {}).get('native_minus_official', 0) > GATES['brier_increase']: + errors.append({'failure': 'task_brier'}) + if quality['all'].get('normalized_score_mae', {}).get('native_minus_official', 0) > GATES['normalized_score_mae_increase']: + errors.append({'failure': 'task_score_mae'}) + counts = collections.Counter(r['type'] for r in boundaries) + for kind in ('choice', 'noul'): + if counts[kind] < GATES['minimum_boundary_per_type']: + errors.append({'failure': 'boundary_coverage', 'type': kind, 'observed': counts[kind]}) + repeats = [] + for records, label in ((official, 'official'), (native, 'native')): + mixed = next(r for r in records if r['name'] == 'mixed') + for r in records: + if r['name'].startswith('mixed-repeat-'): + error = max(abs(a-b) for p, n in zip(mixed['probabilities'], r['probabilities']) for a, b in zip(p, n)) + same = mixed['body']['answers'] == r['body']['answers'] + repeats.append({'kind': label, 'case': r['name'], 'probability_max_abs': error, 'answers_equal': same}) + if label == 'native' and (error != 0 or not same): + errors.append({'failure': 'native_repeat', 'case': r['name']}) + return {'gates': GATES, 'requests': len(expected), 'questions': len(results), 'results': results, + 'boundaries': boundaries, 'boundary_counts': counts, 'task_quality': quality, 'labeled_rows': metrics, + 'repeats': repeats, 'failures': errors, 'passed': not errors} + + +def supplements(corpora, officials, natives): + """Keep each official service run separate, including repeated CLI inputs.""" + if not len(corpora) == len(officials) == len(natives): + raise ValueError('all three supplement files are required for each run') + return list(zip(corpora, officials, natives, strict=True)) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + for flag in ('corpus', 'official', 'native', 'reference', 'base', 'out'): + parser.add_argument('--'+flag, type=Path, required=True) + for flag in ('supplement-corpus', 'supplement-official', 'supplement-native'): + parser.add_argument('--'+flag, type=Path, action='append', default=[]) + args = parser.parse_args() + if args.out.exists(): + raise ValueError('preserve previous report') + corpus = json.loads(args.corpus.read_text()) + official = json.loads(args.official.read_text())['cases'] + native = read_lines(args.native) + runs = [official] + extra = supplements(args.supplement_corpus, args.supplement_official, args.supplement_native) + for corpus_path, official_path, native_path in extra: + supplement = json.loads(corpus_path.read_text()) + if supplement['gates'] != GATES: + raise ValueError('supplement gates differ') + additional = json.loads(official_path.read_text())['cases'] + corpus['cases'] += supplement['cases'] + runs.append(additional) + official = official + additional + native += read_lines(native_path) + report = compare(corpus, official, native) + fixture = module('kev_fixture', ROOT/'tests/kev/generate_processing_fixture.py') + api, model = fixture.official(args.reference) + tok = model.load_tokenizer(args.base) + for kind, records in (('official', official), ('native', native)): + for case, record in zip(corpus['cases'], records): + body = record['body'] + if body['usage']['output_tokens'] != api.output_tokens(tok, body['answers']): + report['failures'].append({'failure': 'output_tokens', 'kind': kind, 'case': case['name']}) + expected = api.to_answers(record['probabilities'], case['meta']) + if body['answers'] != expected: + report['failures'].append({'failure': 'postprocess', 'kind': kind, 'case': case['name'], 'expected': expected, 'actual': body['answers']}) + report['official_execution'] = [{**{k: run[-1][k] for k in ('runtime', 'graphs', 'graph_replays')}, + 'requests': len(run), 'cache_hits': sum(r['stats']['prefix_cache_hit'] for r in run)} for run in runs] + if (any(not run[-1]['graph_replays'] or not run[-1]['graphs']['captured'] for run in runs) + or not sum(r['stats']['prefix_cache_hit'] for r in official)): + report['failures'].append({'failure': 'official_execution_coverage'}) + for record in official: + runtime = record['runtime'] + if (runtime['dtype'] != 'bfloat16' or runtime['head_dtype'] != 'torch.float32' + or runtime['attention'] != 'sdpa' or runtime['fused_available'] + or runtime['remaining_lora_modules'] != 0 or runtime['temperature'] != 2.406050072164233): + report['failures'].append({'failure': 'official_runtime', 'case': record['name'], 'runtime': runtime}) + report['provenance'] = {k: {'path': str(getattr(args, k)), 'sha256': digest(getattr(args, k))} + for k in ('corpus', 'official', 'native')} + report['provenance']['supplements'] = [ + {k: {'path': str(path), 'sha256': digest(path)} + for k, path in zip(('corpus', 'official', 'native'), paths, strict=True)} + for paths in extra] + report['passed'] = not report['failures'] + save(args.out, report) + print(json.dumps({'passed': report['passed'], 'requests': report['requests'], 'questions': report['questions'], + 'boundary_counts': report['boundary_counts'], 'failure_counts': collections.Counter(r.get('failure', f) for r in report['failures'] for f in r.get('failures', ['other']))}, indent=2)) + raise SystemExit(0 if report['passed'] else 1) + + +if __name__ == '__main__': + main() diff --git a/tests/kev/data/boundary-sources.json b/tests/kev/data/boundary-sources.json new file mode 100644 index 00000000..2962a493 --- /dev/null +++ b/tests/kev/data/boundary-sources.json @@ -0,0 +1,34 @@ +{ + "reference": "2074cc44256200bb37540103fa4572901faa10f3", + "sources": { + "evals/decision-v1/development.jsonl": "82adc559bdda908e54ef681bdc3e2ae28dfbe95e2f7873eac965c7c67797d10b", + "evals/decision-v2/development.jsonl": "10e5a72102b844a167ade94c60cf4556cc63950b2ded17fb87bf96028cb557f1", + "evals/devtools-v1/development.jsonl": "3df3aa83890c43db65ac73b5bb6e4fbb1466362fc1a25dbd383d4d2edd0a1fe6", + "evals/documents-v1/development.jsonl": "bfca7b41b0d8553b03f9781f48c942dfea368ce19b8508794316e555de8280f5", + "evals/external/semif-v1/development.jsonl": "b9e0c7939a25a04a7ef7bc12334b5fa45b2027d04904d32021295b5539a97303", + "evals/hard-v1/development.jsonl": "3cdbf12a6b7c3b70c73e8c39f2677127ea61ef32f0ea75b55a43556d884b2d98", + "evals/public-pool-v4/development.jsonl": "1542b2def96f9c86510ea2976bbc68f878acef0c36aff4cd6ed19868c2d2349f", + "evals/public-pool-v5/development.jsonl": "1542b2def96f9c86510ea2976bbc68f878acef0c36aff4cd6ed19868c2d2349f", + "evals/public-pool-v6/development.jsonl": "0d3b60dc355b4101372449bed321d420eb69618ad400b184684e46be4c33ee77", + "evals/round3/decision-r3/development.jsonl": "8d5765d7aec4d08c61854f4664ca79ec2ba44ed092e9b967eaf61ed86c496d9c", + "evals/round3/transfer-r3/development.jsonl": "ff374c49c6c9f15f8a56fb274b4a4857d20497eb8dd1ac07ce01560e682a5f2e", + "evals/round4/longstate-v1/development.jsonl": "efb8c47995d5146a58f1b562f55884b0a2166fef611e3dab8085f0031a051c4b", + "evals/round5/longstate-v2/development.jsonl": "4c03830a64479b9470fffee0b2e7da979dffa1a2c531126bd53047841c41046b", + "evals/round5/transfer-r5/development.jsonl": "ff374c49c6c9f15f8a56fb274b4a4857d20497eb8dd1ac07ce01560e682a5f2e", + "evals/smoke-v1/development.jsonl": "e0b18f4b8d95aaa2479a63bfc6b0af5f44bc2fd11bb87e5ca42317c5a5d8a306", + "evals/transfer-v1/development.jsonl": "c8a325cd2e04b4b4831e53a4e3165dbeed21d1db8c4b4520c942f38b936c0636", + "evals/transfer-v2/development.jsonl": "02732ee2e662b7ec0c276f7f89ae29f44e7ce262ffac8e9c116c5bc82e5608b8", + "evals/v3/decision-v3/development.jsonl": "8d5765d7aec4d08c61854f4664ca79ec2ba44ed092e9b967eaf61ed86c496d9c", + "evals/v3/smoke-v3/development.jsonl": "b8a8df68bf8b673eb88888e8c2885b50db4fd98dbc91f1a7329e618d59005c01", + "evals/v3/transfer-v3/development.jsonl": "ff374c49c6c9f15f8a56fb274b4a4857d20497eb8dd1ac07ce01560e682a5f2e", + "evals/v4/decision-v4/development.jsonl": "8d5765d7aec4d08c61854f4664ca79ec2ba44ed092e9b967eaf61ed86c496d9c", + "evals/v4/transfer-v4/development.jsonl": "ff374c49c6c9f15f8a56fb274b4a4857d20497eb8dd1ac07ce01560e682a5f2e", + "evals/v5/decision-v5/development.jsonl": "8d5765d7aec4d08c61854f4664ca79ec2ba44ed092e9b967eaf61ed86c496d9c", + "evals/v5/transfer-v5/development.jsonl": "ff374c49c6c9f15f8a56fb274b4a4857d20497eb8dd1ac07ce01560e682a5f2e", + "evals/v6/decision-v6/development.jsonl": "8d5765d7aec4d08c61854f4664ca79ec2ba44ed092e9b967eaf61ed86c496d9c", + "evals/v6/transfer-v6/development.jsonl": "ff374c49c6c9f15f8a56fb274b4a4857d20497eb8dd1ac07ce01560e682a5f2e", + "evals/v7/decision-v7/development.jsonl": "8d5765d7aec4d08c61854f4664ca79ec2ba44ed092e9b967eaf61ed86c496d9c", + "evals/v8/decision-v8/development.jsonl": "8d5765d7aec4d08c61854f4664ca79ec2ba44ed092e9b967eaf61ed86c496d9c", + "evals/v9/transfer-v9/development.jsonl": "53872ba045564315cf3c3dd8fb656e76441c6c514b1a7ff985ff0db3955710cf" + } +} diff --git a/tests/kev/data/head.json b/tests/kev/data/head.json new file mode 100644 index 00000000..68c8cff8 --- /dev/null +++ b/tests/kev/data/head.json @@ -0,0 +1,638 @@ +{ + "reference_revision": "2074cc44256200bb37540103fa4572901faa10f3", + "reference_model_sha256": "2ff3b2e67c048e3bc5a0d1a32aafe50804ed315a5564fce06bb4b95697c5271c", + "head_pt_sha256": "dd633435998ecc751ac538717a3742e32149500fabf7d7276287dbf0693f347c", + "torch": "2.8.0+cpu", + "device": "cpu", + "temperature": 2.406050072164233, + "hidden_rule": "((i * 37 + row * 19) % 101 - 50) / 64; row 0 = decide", + "cases": [ + { + "options": 1, + "logits": [ + 0.16330909729003906 + ], + "probabilities": [ + 1.0 + ] + }, + { + "options": 2, + "logits": [ + 0.16330908238887787, + -0.04296588525176048 + ], + "probabilities": [ + 0.5214198231697083, + 0.47858014702796936 + ] + }, + { + "options": 7, + "logits": [ + 0.16330906748771667, + -0.04296588897705078, + -0.1620718538761139, + 0.012366419658064842, + 0.047717608511447906, + 0.15346293151378632, + -0.011503779329359531 + ], + "probabilities": [ + 0.1512979418039322, + 0.1388673633337021, + 0.13216042518615723, + 0.14209792017936707, + 0.1442011296749115, + 0.1506800651550293, + 0.14069513976573944 + ] + }, + { + "options": 31, + "logits": [ + 0.16330909729003906, + -0.04296589270234108, + -0.1620718389749527, + 0.012366367504000664, + 0.04771760478615761, + 0.1534629762172699, + -0.011503748595714569, + -0.15352031588554382, + -0.12729023396968842, + -0.018939895555377007, + 0.07874026149511337, + 0.1336236596107483, + -0.07686153799295425, + -0.1321595162153244, + -0.038695480674505234, + 0.08788273483514786, + 0.18759208917617798, + -0.036322422325611115, + -0.18305128812789917, + 0.025450605899095535, + 0.07632403075695038, + 0.15722697973251343, + 0.00885787419974804, + -0.14430277049541473, + -0.09872071444988251, + 0.013050932437181473, + 0.10318844020366669, + 0.11492683738470078, + -0.07907889038324356, + -0.12160822004079819, + -0.05631116405129433 + ], + "probabilities": [ + 0.03454456478357315, + 0.031706396490335464, + 0.030175060033798218, + 0.032444000244140625, + 0.03292420879006386, + 0.0344034880399704, + 0.03212371841073036, + 0.03028249740600586, + 0.03061443567276001, + 0.03202458843588829, + 0.03335146605968475, + 0.03412097692489624, + 0.03126285597681999, + 0.030552541837096214, + 0.03176271915435791, + 0.03347843885421753, + 0.03489496931433678, + 0.03179406002163887, + 0.02991309203207493, + 0.03262091055512428, + 0.033317990601062775, + 0.03445735201239586, + 0.03239672631025314, + 0.03039873205125332, + 0.030980117619037628, + 0.032453231513500214, + 0.03369208425283432, + 0.03385685756802559, + 0.031234057620167732, + 0.030686818063259125, + 0.03153102099895477 + ] + }, + { + "options": 255, + "logits": [ + 0.16330909729003906, + -0.04296589270234108, + -0.1620718389749527, + 0.012366367504000664, + 0.04771760478615761, + 0.1534629762172699, + -0.011503748595714569, + -0.15352031588554382, + -0.12729023396968842, + -0.018939895555377007, + 0.07874026149511337, + 0.1336236596107483, + -0.07686153799295425, + -0.1321595162153244, + -0.038695480674505234, + 0.08788273483514786, + 0.18759208917617798, + -0.036322422325611115, + -0.18305128812789917, + 0.025450605899095535, + 0.07632403075695038, + 0.15722697973251343, + 0.00885787419974804, + -0.14430277049541473, + -0.09872071444988251, + 0.013050932437181473, + 0.10318844020366669, + 0.11492683738470078, + -0.07907890528440475, + -0.12160823494195938, + -0.05631116405129433, + 0.07068727165460587, + 0.167240172624588, + -0.055517807602882385, + -0.15244919061660767, + -0.0018523912876844406, + 0.05228900536894798, + 0.18918664753437042, + 0.003996687941253185, + -0.14683139324188232, + -0.09803099185228348, + 0.07003703713417053, + 0.15613742172718048, + 0.09519751369953156, + -0.06439848989248276, + -0.12586374580860138, + -0.05452625826001167, + 0.10998698323965073, + 0.1803295910358429, + -0.1095396876335144, + -0.17051587998867035, + -0.058045294135808945, + 0.06769002228975296, + 0.17876161634922028, + 0.02370007336139679, + -0.14835882186889648, + -0.04036284238100052, + 0.024112947285175323, + 0.14640827476978302, + 0.09839790314435959, + -0.08899127691984177, + -0.1127111166715622, + -0.05864831432700157, + 0.09408044815063477, + 0.15783268213272095, + -0.08195436745882034, + -0.14064112305641174, + -0.06312527507543564, + 0.0818595364689827, + 0.1974586397409439, + -0.008748650550842285, + -0.15902179479599, + -0.009172530844807625, + 0.020293492823839188, + 0.1351587176322937, + 0.04563114047050476, + -0.07360135018825531, + -0.11956068873405457, + -0.01191377080976963, + 0.07325820624828339, + 0.14599785208702087, + -0.08439307659864426, + -0.11306572705507278, + -0.07357587665319443, + 0.10550788789987564, + 0.19497254490852356, + -0.04602048918604851, + -0.17397882044315338, + -0.01272417139261961, + 0.05303587391972542, + 0.12375271320343018, + 0.0177279245108366, + -0.11302593350410461, + -0.10215660184621811, + 0.006327805109322071, + 0.05703119561076164, + 0.13509738445281982, + -0.07641758024692535, + -0.13544845581054688, + -0.04186152666807175, + 0.09860523045063019, + 0.16330909729003906, + -0.04296589270234108, + -0.1620718389749527, + 0.012366367504000664, + 0.04771760478615761, + 0.1534629762172699, + -0.011503748595714569, + -0.15352031588554382, + -0.12729023396968842, + -0.018939895555377007, + 0.07874026149511337, + 0.1336236596107483, + -0.07686153799295425, + -0.1321595162153244, + -0.038695480674505234, + 0.08788273483514786, + 0.18759208917617798, + -0.036322422325611115, + -0.18305128812789917, + 0.025450605899095535, + 0.07632403075695038, + 0.15722697973251343, + 0.00885787419974804, + -0.14430277049541473, + -0.09872071444988251, + 0.013050932437181473, + 0.10318844020366669, + 0.11492683738470078, + -0.07907890528440475, + -0.12160823494195938, + -0.05631116405129433, + 0.07068727165460587, + 0.167240172624588, + -0.055517807602882385, + -0.15244919061660767, + -0.0018523912876844406, + 0.05228900536894798, + 0.18918664753437042, + 0.003996687941253185, + -0.14683139324188232, + -0.09803099185228348, + 0.07003703713417053, + 0.15613742172718048, + 0.09519751369953156, + -0.06439848989248276, + -0.12586374580860138, + -0.05452625826001167, + 0.10998698323965073, + 0.1803295910358429, + -0.1095396876335144, + -0.17051587998867035, + -0.058045294135808945, + 0.06769002228975296, + 0.17876161634922028, + 0.02370007336139679, + -0.14835882186889648, + -0.04036284238100052, + 0.024112947285175323, + 0.14640827476978302, + 0.09839790314435959, + -0.08899127691984177, + -0.1127111166715622, + -0.05864831432700157, + 0.09408044815063477, + 0.15783268213272095, + -0.08195436745882034, + -0.14064112305641174, + -0.06312527507543564, + 0.0818595364689827, + 0.1974586397409439, + -0.008748650550842285, + -0.15902179479599, + -0.009172530844807625, + 0.020293492823839188, + 0.1351587176322937, + 0.04563114047050476, + -0.07360135018825531, + -0.11956068873405457, + -0.01191377080976963, + 0.07325820624828339, + 0.14599785208702087, + -0.08439307659864426, + -0.11306572705507278, + -0.07357587665319443, + 0.10550788789987564, + 0.19497254490852356, + -0.04602048918604851, + -0.17397882044315338, + -0.01272417139261961, + 0.05303587391972542, + 0.12375271320343018, + 0.0177279245108366, + -0.11302593350410461, + -0.10215660184621811, + 0.006327805109322071, + 0.05703119561076164, + 0.13509738445281982, + -0.07641758024692535, + -0.13544845581054688, + -0.04186152666807175, + 0.09860523045063019, + 0.16330909729003906, + -0.04296589270234108, + -0.1620718389749527, + 0.012366367504000664, + 0.04771760478615761, + 0.1534629762172699, + -0.011503748595714569, + -0.15352031588554382, + -0.12729023396968842, + -0.018939895555377007, + 0.07874026149511337, + 0.1336236596107483, + -0.07686153799295425, + -0.1321595162153244, + -0.038695480674505234, + 0.08788273483514786, + 0.18759208917617798, + -0.036322422325611115, + -0.18305128812789917, + 0.025450605899095535, + 0.07632403075695038, + 0.15722697973251343, + 0.00885787419974804, + -0.14430277049541473, + -0.09872071444988251, + 0.013050932437181473, + 0.10318844020366669, + 0.11492683738470078, + -0.07907890528440475, + -0.12160823494195938, + -0.05631116405129433, + 0.07068727165460587, + 0.167240172624588, + -0.055517807602882385, + -0.15244919061660767, + -0.0018523912876844406, + 0.05228900536894798, + 0.18918664753437042, + 0.003996687941253185, + -0.14683139324188232, + -0.09803099185228348, + 0.07003703713417053, + 0.15613742172718048, + 0.09519751369953156, + -0.06439848989248276, + -0.12586374580860138, + -0.05452625826001167, + 0.10998698323965073, + 0.1803295910358429, + -0.1095396876335144, + -0.17051589488983154, + -0.058045294135808945, + 0.06769000738859177 + ], + "probabilities": [ + 0.004190933890640736, + 0.0038466083351522684, + 0.0036608269438147545, + 0.003936094231903553, + 0.003994353115558624, + 0.00417381851002574, + 0.0038972378242760897, + 0.0036738612689077854, + 0.0037141318898648024, + 0.00388521165587008, + 0.004046187736093998, + 0.004139544442296028, + 0.0037927983794361353, + 0.003706623101606965, + 0.003853441448882222, + 0.004061591811478138, + 0.004233445040881634, + 0.003857244038954377, + 0.0036290453281253576, + 0.003957557491958141, + 0.004042126704007387, + 0.004180353134870529, + 0.003930358681827784, + 0.0036879628896713257, + 0.003758496604859829, + 0.003937214147299528, + 0.004087510984390974, + 0.004107501823455095, + 0.003789304755628109, + 0.0037229135632514954, + 0.003825332038104534, + 0.004032668191939592, + 0.004197786562144756, + 0.003826593514531851, + 0.003675497369840741, + 0.003912901971489191, + 0.004001948982477188, + 0.0042362515814602375, + 0.003922426141798496, + 0.0036840890534222126, + 0.003759574145078659, + 0.004031578078866005, + 0.0041784606873989105, + 0.00407395837828517, + 0.003812495619058609, + 0.0037163347005844116, + 0.003828170709311962, + 0.004099077079445124, + 0.004220685921609402, + 0.0037416343111544847, + 0.0036480019334703684, + 0.0038225757889449596, + 0.004027647897601128, + 0.0042179361917078495, + 0.003954679239541292, + 0.0036817509680986404, + 0.0038507720455527306, + 0.0039553577080369, + 0.00416159862652421, + 0.004079381003975868, + 0.003773725824430585, + 0.0037367057520896196, + 0.0038216179236769676, + 0.004072067327797413, + 0.004181405529379845, + 0.0037847787607461214, + 0.0036935796961188316, + 0.003814513562247157, + 0.004051436670124531, + 0.004250840749591589, + 0.00390170281752944, + 0.003665470751002431, + 0.0039010157343000174, + 0.003949083853513002, + 0.004142186604440212, + 0.003990890458226204, + 0.003797941142693162, + 0.003726082853972912, + 0.003896573791280389, + 0.004036979284137487, + 0.004160888958722353, + 0.0037809445057064295, + 0.0037361548747867346, + 0.003797981422394514, + 0.004091453738510609, + 0.004246450960636139, + 0.0038417279720306396, + 0.0036427550949156284, + 0.0038952615577727556, + 0.004003191366791725, + 0.004122596699744463, + 0.003944875206798315, + 0.003736216574907303, + 0.0037531331181526184, + 0.003926228266209364, + 0.004009844269603491, + 0.004142080899327993, + 0.003793498268350959, + 0.0037015597335994244, + 0.0038483741227537394, + 0.004079732578247786, + 0.004190933890640736, + 0.0038466083351522684, + 0.0036608269438147545, + 0.003936094231903553, + 0.003994353115558624, + 0.00417381851002574, + 0.0038972378242760897, + 0.0036738612689077854, + 0.0037141318898648024, + 0.00388521165587008, + 0.004046187736093998, + 0.004139544442296028, + 0.0037927983794361353, + 0.003706623101606965, + 0.003853441448882222, + 0.004061591811478138, + 0.004233445040881634, + 0.003857244038954377, + 0.0036290453281253576, + 0.003957557491958141, + 0.004042126704007387, + 0.004180353134870529, + 0.003930358681827784, + 0.0036879628896713257, + 0.003758496604859829, + 0.003937214147299528, + 0.004087510984390974, + 0.004107501823455095, + 0.003789304755628109, + 0.0037229135632514954, + 0.003825332038104534, + 0.004032668191939592, + 0.004197786562144756, + 0.003826593514531851, + 0.003675497369840741, + 0.003912901971489191, + 0.004001948982477188, + 0.0042362515814602375, + 0.003922426141798496, + 0.0036840890534222126, + 0.003759574145078659, + 0.004031578078866005, + 0.0041784606873989105, + 0.00407395837828517, + 0.003812495619058609, + 0.0037163347005844116, + 0.003828170709311962, + 0.004099077079445124, + 0.004220685921609402, + 0.0037416343111544847, + 0.0036480019334703684, + 0.0038225757889449596, + 0.004027647897601128, + 0.0042179361917078495, + 0.003954679239541292, + 0.0036817509680986404, + 0.0038507720455527306, + 0.0039553577080369, + 0.00416159862652421, + 0.004079381003975868, + 0.003773725824430585, + 0.0037367057520896196, + 0.0038216179236769676, + 0.004072067327797413, + 0.004181405529379845, + 0.0037847787607461214, + 0.0036935796961188316, + 0.003814513562247157, + 0.004051436670124531, + 0.004250840749591589, + 0.00390170281752944, + 0.003665470751002431, + 0.0039010157343000174, + 0.003949083853513002, + 0.004142186604440212, + 0.003990890458226204, + 0.003797941142693162, + 0.003726082853972912, + 0.003896573791280389, + 0.004036979284137487, + 0.004160888958722353, + 0.0037809445057064295, + 0.0037361548747867346, + 0.003797981422394514, + 0.004091453738510609, + 0.004246450960636139, + 0.0038417279720306396, + 0.0036427550949156284, + 0.0038952615577727556, + 0.004003191366791725, + 0.004122596699744463, + 0.003944875206798315, + 0.003736216574907303, + 0.0037531331181526184, + 0.003926228266209364, + 0.004009844269603491, + 0.004142080899327993, + 0.003793498268350959, + 0.0037015597335994244, + 0.0038483741227537394, + 0.004079732578247786, + 0.004190933890640736, + 0.0038466083351522684, + 0.0036608269438147545, + 0.003936094231903553, + 0.003994353115558624, + 0.00417381851002574, + 0.0038972378242760897, + 0.0036738612689077854, + 0.0037141318898648024, + 0.00388521165587008, + 0.004046187736093998, + 0.004139544442296028, + 0.0037927983794361353, + 0.003706623101606965, + 0.003853441448882222, + 0.004061591811478138, + 0.004233445040881634, + 0.003857244038954377, + 0.0036290453281253576, + 0.003957557491958141, + 0.004042126704007387, + 0.004180353134870529, + 0.003930358681827784, + 0.0036879628896713257, + 0.003758496604859829, + 0.003937214147299528, + 0.004087510984390974, + 0.004107501823455095, + 0.003789304755628109, + 0.0037229135632514954, + 0.003825332038104534, + 0.004032668191939592, + 0.004197786562144756, + 0.003826593514531851, + 0.003675497369840741, + 0.003912901971489191, + 0.004001948982477188, + 0.0042362515814602375, + 0.003922426141798496, + 0.0036840890534222126, + 0.003759574145078659, + 0.004031578078866005, + 0.0041784606873989105, + 0.00407395837828517, + 0.003812495619058609, + 0.0037163347005844116, + 0.003828170709311962, + 0.004099077079445124, + 0.004220685921609402, + 0.0037416343111544847, + 0.0036480019334703684, + 0.0038225757889449596, + 0.00402764743193984 + ] + } + ] +} diff --git a/tests/kev/data/planning.json b/tests/kev/data/planning.json new file mode 100644 index 00000000..e1262d6a --- /dev/null +++ b/tests/kev/data/planning.json @@ -0,0 +1 @@ +{"reference":"2074cc44256200bb37540103fa4572901faa10f3","cases":[{"split":1,"lengths":[1],"plans":[{"split":1,"state":16,"prefix":16,"question":16,"count":1,"index":0}]},{"split":29,"lengths":[27,12],"plans":[{"split":29,"state":32,"prefix":32,"question":32,"count":2,"index":1},{"split":29,"state":32,"prefix":32,"question":32,"count":2,"index":0}]},{"split":67,"lengths":[66,70],"plans":[{"split":67,"state":80,"prefix":128,"question":80,"count":2,"index":0},{"split":67,"state":80,"prefix":128,"question":80,"count":2,"index":1}]},{"split":52,"lengths":[69],"plans":[{"split":52,"state":64,"prefix":64,"question":80,"count":1,"index":0}]},{"split":7,"lengths":[1679],"plans":[{"split":7,"state":16,"prefix":7,"question":1679,"count":1,"index":0}]},{"split":29,"lengths":[12,12,12,12,12],"plans":[{"split":29,"state":32,"prefix":32,"question":16,"count":6,"index":0},{"split":29,"state":32,"prefix":32,"question":16,"count":6,"index":1},{"split":29,"state":32,"prefix":32,"question":16,"count":6,"index":2},{"split":29,"state":32,"prefix":32,"question":16,"count":6,"index":3},{"split":29,"state":32,"prefix":32,"question":16,"count":6,"index":4}]},{"split":29,"lengths":[12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12],"plans":[{"split":29,"state":32,"prefix":32,"question":16,"count":32,"index":0},{"split":29,"state":32,"prefix":32,"question":16,"count":32,"index":1},{"split":29,"state":32,"prefix":32,"question":16,"count":32,"index":2},{"split":29,"state":32,"prefix":32,"question":16,"count":32,"index":3},{"split":29,"state":32,"prefix":32,"question":16,"count":32,"index":4},{"split":29,"state":32,"prefix":32,"question":16,"count":32,"index":5},{"split":29,"state":32,"prefix":32,"question":16,"count":32,"index":6},{"split":29,"state":32,"prefix":32,"question":16,"count":32,"index":7},{"split":29,"state":32,"prefix":32,"question":16,"count":32,"index":8},{"split":29,"state":32,"prefix":32,"question":16,"count":32,"index":9},{"split":29,"state":32,"prefix":32,"question":16,"count":32,"index":10},{"split":29,"state":32,"prefix":32,"question":16,"count":32,"index":11},{"split":29,"state":32,"prefix":32,"question":16,"count":32,"index":12},{"split":29,"state":32,"prefix":32,"question":16,"count":32,"index":13},{"split":29,"state":32,"prefix":32,"question":16,"count":32,"index":14},{"split":29,"state":32,"prefix":32,"question":16,"count":32,"index":15},{"split":29,"state":32,"prefix":32,"question":16,"count":32,"index":16},{"split":29,"state":32,"prefix":32,"question":16,"count":32,"index":17},{"split":29,"state":32,"prefix":32,"question":16,"count":32,"index":18},{"split":29,"state":32,"prefix":32,"question":16,"count":32,"index":19},{"split":29,"state":32,"prefix":32,"question":16,"count":32,"index":20},{"split":29,"state":32,"prefix":32,"question":16,"count":32,"index":21},{"split":29,"state":32,"prefix":32,"question":16,"count":32,"index":22},{"split":29,"state":32,"prefix":32,"question":16,"count":32,"index":23},{"split":29,"state":32,"prefix":32,"question":16,"count":32,"index":24},{"split":29,"state":32,"prefix":32,"question":16,"count":32,"index":25},{"split":29,"state":32,"prefix":32,"question":16,"count":32,"index":26},{"split":29,"state":32,"prefix":32,"question":16,"count":32,"index":27},{"split":29,"state":32,"prefix":32,"question":16,"count":32,"index":28},{"split":29,"state":32,"prefix":32,"question":16,"count":32,"index":29},{"split":29,"state":32,"prefix":32,"question":16,"count":32,"index":30},{"split":29,"state":32,"prefix":32,"question":16,"count":32,"index":31},{"split":29,"state":32,"prefix":32,"question":16,"count":32,"index":0},{"split":29,"state":32,"prefix":32,"question":16,"count":32,"index":1},{"split":29,"state":32,"prefix":32,"question":16,"count":32,"index":2},{"split":29,"state":32,"prefix":32,"question":16,"count":32,"index":3},{"split":29,"state":32,"prefix":32,"question":16,"count":32,"index":4},{"split":29,"state":32,"prefix":32,"question":16,"count":32,"index":5},{"split":29,"state":32,"prefix":32,"question":16,"count":32,"index":6},{"split":29,"state":32,"prefix":32,"question":16,"count":32,"index":7},{"split":29,"state":32,"prefix":32,"question":16,"count":32,"index":8},{"split":29,"state":32,"prefix":32,"question":16,"count":32,"index":9},{"split":29,"state":32,"prefix":32,"question":16,"count":32,"index":10},{"split":29,"state":32,"prefix":32,"question":16,"count":32,"index":11},{"split":29,"state":32,"prefix":32,"question":16,"count":32,"index":12},{"split":29,"state":32,"prefix":32,"question":16,"count":32,"index":13},{"split":29,"state":32,"prefix":32,"question":16,"count":32,"index":14},{"split":29,"state":32,"prefix":32,"question":16,"count":32,"index":15},{"split":29,"state":32,"prefix":32,"question":16,"count":32,"index":16},{"split":29,"state":32,"prefix":32,"question":16,"count":32,"index":17},{"split":29,"state":32,"prefix":32,"question":16,"count":32,"index":18},{"split":29,"state":32,"prefix":32,"question":16,"count":32,"index":19},{"split":29,"state":32,"prefix":32,"question":16,"count":32,"index":20},{"split":29,"state":32,"prefix":32,"question":16,"count":32,"index":21},{"split":29,"state":32,"prefix":32,"question":16,"count":32,"index":22},{"split":29,"state":32,"prefix":32,"question":16,"count":32,"index":23},{"split":29,"state":32,"prefix":32,"question":16,"count":32,"index":24},{"split":29,"state":32,"prefix":32,"question":16,"count":32,"index":25},{"split":29,"state":32,"prefix":32,"question":16,"count":32,"index":26},{"split":29,"state":32,"prefix":32,"question":16,"count":32,"index":27},{"split":29,"state":32,"prefix":32,"question":16,"count":32,"index":28},{"split":29,"state":32,"prefix":32,"question":16,"count":32,"index":29},{"split":29,"state":32,"prefix":32,"question":16,"count":32,"index":30},{"split":29,"state":32,"prefix":32,"question":16,"count":32,"index":31}]},{"split":4096,"lengths":[18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18],"plans":[{"split":4096,"state":4096,"prefix":4096,"question":32,"count":6,"index":0},{"split":4096,"state":4096,"prefix":4096,"question":32,"count":6,"index":1},{"split":4096,"state":4096,"prefix":4096,"question":32,"count":6,"index":2},{"split":4096,"state":4096,"prefix":4096,"question":32,"count":6,"index":3},{"split":4096,"state":4096,"prefix":4096,"question":32,"count":6,"index":4},{"split":4096,"state":4096,"prefix":4096,"question":32,"count":6,"index":5},{"split":4096,"state":4096,"prefix":4096,"question":32,"count":6,"index":0},{"split":4096,"state":4096,"prefix":4096,"question":32,"count":6,"index":1},{"split":4096,"state":4096,"prefix":4096,"question":32,"count":6,"index":2},{"split":4096,"state":4096,"prefix":4096,"question":32,"count":6,"index":3},{"split":4096,"state":4096,"prefix":4096,"question":32,"count":6,"index":4},{"split":4096,"state":4096,"prefix":4096,"question":32,"count":6,"index":5},{"split":4096,"state":4096,"prefix":4096,"question":32,"count":6,"index":0},{"split":4096,"state":4096,"prefix":4096,"question":32,"count":6,"index":1},{"split":4096,"state":4096,"prefix":4096,"question":32,"count":6,"index":2},{"split":4096,"state":4096,"prefix":4096,"question":32,"count":6,"index":3},{"split":4096,"state":4096,"prefix":4096,"question":32,"count":6,"index":4},{"split":4096,"state":4096,"prefix":4096,"question":32,"count":6,"index":5},{"split":4096,"state":4096,"prefix":4096,"question":32,"count":6,"index":0},{"split":4096,"state":4096,"prefix":4096,"question":32,"count":6,"index":1},{"split":4096,"state":4096,"prefix":4096,"question":32,"count":6,"index":2},{"split":4096,"state":4096,"prefix":4096,"question":32,"count":6,"index":3},{"split":4096,"state":4096,"prefix":4096,"question":32,"count":6,"index":4},{"split":4096,"state":4096,"prefix":4096,"question":32,"count":6,"index":5},{"split":4096,"state":4096,"prefix":4096,"question":32,"count":6,"index":0},{"split":4096,"state":4096,"prefix":4096,"question":32,"count":6,"index":1},{"split":4096,"state":4096,"prefix":4096,"question":32,"count":6,"index":2},{"split":4096,"state":4096,"prefix":4096,"question":32,"count":6,"index":3},{"split":4096,"state":4096,"prefix":4096,"question":32,"count":6,"index":4},{"split":4096,"state":4096,"prefix":4096,"question":32,"count":6,"index":5},{"split":4096,"state":4096,"prefix":4096,"question":32,"count":2,"index":0},{"split":4096,"state":4096,"prefix":4096,"question":32,"count":2,"index":1}]},{"split":1025,"lengths":[999,32,400,33,8],"plans":[{"split":1025,"state":1025,"prefix":2048,"question":1024,"count":1,"index":0},{"split":1025,"state":1025,"prefix":2048,"question":48,"count":3,"index":1},{"split":1025,"state":1025,"prefix":2048,"question":448,"count":1,"index":0},{"split":1025,"state":1025,"prefix":2048,"question":48,"count":3,"index":2},{"split":1025,"state":1025,"prefix":2048,"question":48,"count":3,"index":0}]},{"split":29,"lengths":[16,200,900,16],"plans":[{"split":29,"state":32,"prefix":32,"question":16,"count":2,"index":0},{"split":29,"state":32,"prefix":32,"question":224,"count":1,"index":0},{"split":29,"state":32,"prefix":32,"question":1024,"count":1,"index":0},{"split":29,"state":32,"prefix":32,"question":16,"count":2,"index":1}]},{"split":8000,"lengths":[5000,32,500,200],"plans":[{"split":8000,"state":8000,"prefix":8000,"question":5000,"count":1,"index":0},{"split":8000,"state":8000,"prefix":8000,"question":32,"count":1,"index":0},{"split":8000,"state":8000,"prefix":8000,"question":500,"count":1,"index":0},{"split":8000,"state":8000,"prefix":8000,"question":200,"count":1,"index":0}]},{"split":7,"lengths":[1679,14,1680],"plans":[{"split":7,"state":16,"prefix":7,"question":1680,"count":3,"index":0},{"split":7,"state":16,"prefix":7,"question":1680,"count":3,"index":1},{"split":7,"state":16,"prefix":7,"question":1680,"count":3,"index":2}]}]} diff --git a/tests/kev/data/processing.json b/tests/kev/data/processing.json new file mode 100644 index 00000000..fd13165d --- /dev/null +++ b/tests/kev/data/processing.json @@ -0,0 +1 @@ +{"reference":"2074cc44256200bb37540103fa4572901faa10f3","source_hashes":{"model.py":"2ff3b2e67c048e3bc5a0d1a32aafe50804ed315a5564fce06bb4b95697c5271c","api.py":"a6f53af5354f8ea17915abc78360fdf291a5108ce39a53a2d9135d48c603ae58","checkpoint.py":"02062399977ee0da316dd52e03a79fa5e10134aa49c0b5ed801598c52042dc32"},"cases":[{"name":"choice-1","request":{"state":"The number is 0.","questions":{"pick":{"type":"choice","instructions":"Select the option matching the state.","criteria":{"k0":"number 0"}}}},"record":{"state":"The number is 0.","questions":[{"instr":"Select the option matching the state.","options":["k0: number 0"],"label":0}]},"meta":[{"id":"pick","type":"choice","keys":["k0"]}],"rows":[{"ids":[248060,760,1324,369,220,15,13,248061,3270,279,2904,12219,279,1528,13,248049,74,15,25,1324,220,15,248050,248062],"positions":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23],"readouts":[22,23]}],"input_tokens":24,"probabilities":[[1.0]],"answers":{"pick":{"type":"choice","choice":"k0","confidence":1.0,"probabilities":{"k0":1.0}}},"output_tokens":37},{"name":"choice-2","request":{"state":"The number is 1.","questions":{"pick":{"type":"choice","instructions":"Select the option matching the state.","criteria":{"k0":"number 0","k1":"number 1"}}}},"record":{"state":"The number is 1.","questions":[{"instr":"Select the option matching the state.","options":["k0: number 0","k1: number 1"],"label":0}]},"meta":[{"id":"pick","type":"choice","keys":["k0","k1"]}],"rows":[{"ids":[248060,760,1324,369,220,16,13,248061,3270,279,2904,12219,279,1528,13,248049,74,15,25,1324,220,15,248050,248049,74,16,25,1324,220,16,248050,248062],"positions":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31],"readouts":[22,30,31]}],"input_tokens":32,"probabilities":[[0.3333333333333333,0.6666666666666666]],"answers":{"pick":{"type":"choice","choice":"k1","confidence":0.3333,"probabilities":{"k0":0.3333,"k1":0.6667}}},"output_tokens":55},{"name":"choice-7","request":{"state":"The number is 6.","questions":{"pick":{"type":"choice","instructions":"Select the option matching the state.","criteria":{"k0":"number 0","k1":"number 1","k2":"number 2","k3":"number 3","k4":"number 4","k5":"number 5","k6":"number 6"}}}},"record":{"state":"The number is 6.","questions":[{"instr":"Select the option matching the state.","options":["k0: number 0","k1: number 1","k2: number 2","k3: number 3","k4: number 4","k5: number 5","k6: number 6"],"label":0}]},"meta":[{"id":"pick","type":"choice","keys":["k0","k1","k2","k3","k4","k5","k6"]}],"rows":[{"ids":[248060,760,1324,369,220,21,13,248061,3270,279,2904,12219,279,1528,13,248049,74,15,25,1324,220,15,248050,248049,74,16,25,1324,220,16,248050,248049,74,17,25,1324,220,17,248050,248049,74,18,25,1324,220,18,248050,248049,74,19,25,1324,220,19,248050,248049,74,20,25,1324,220,20,248050,248049,74,21,25,1324,220,21,248050,248062],"positions":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71],"readouts":[22,30,38,46,54,62,70,71]}],"input_tokens":72,"probabilities":[[0.03571428571428571,0.07142857142857142,0.10714285714285714,0.14285714285714285,0.17857142857142858,0.21428571428571427,0.25]],"answers":{"pick":{"type":"choice","choice":"k6","confidence":0.125,"probabilities":{"k0":0.0357,"k1":0.0714,"k2":0.1071,"k3":0.1429,"k4":0.1786,"k5":0.2143,"k6":0.25}}},"output_tokens":112},{"name":"choice-31","request":{"state":"The number is 30.","questions":{"pick":{"type":"choice","instructions":"Select the option matching the state.","criteria":{"k0":"number 0","k1":"number 1","k2":"number 2","k3":"number 3","k4":"number 4","k5":"number 5","k6":"number 6","k7":"number 7","k8":"number 8","k9":"number 9","k10":"number 10","k11":"number 11","k12":"number 12","k13":"number 13","k14":"number 14","k15":"number 15","k16":"number 16","k17":"number 17","k18":"number 18","k19":"number 19","k20":"number 20","k21":"number 21","k22":"number 22","k23":"number 23","k24":"number 24","k25":"number 25","k26":"number 26","k27":"number 27","k28":"number 28","k29":"number 29","k30":"number 30"}}}},"record":{"state":"The number is 30.","questions":[{"instr":"Select the option matching the state.","options":["k0: number 0","k1: number 1","k2: number 2","k3: number 3","k4: number 4","k5: number 5","k6: number 6","k7: number 7","k8: number 8","k9: number 9","k10: number 10","k11: number 11","k12: number 12","k13: number 13","k14: number 14","k15: number 15","k16: number 16","k17: number 17","k18: number 18","k19: number 19","k20: number 20","k21: number 21","k22: number 22","k23: number 23","k24: number 24","k25: number 25","k26: number 26","k27: number 27","k28: number 28","k29: number 29","k30: number 30"],"label":0}]},"meta":[{"id":"pick","type":"choice","keys":["k0","k1","k2","k3","k4","k5","k6","k7","k8","k9","k10","k11","k12","k13","k14","k15","k16","k17","k18","k19","k20","k21","k22","k23","k24","k25","k26","k27","k28","k29","k30"]}],"rows":[{"ids":[248060,760,1324,369,220,18,15,13,248061,3270,279,2904,12219,279,1528,13,248049,74,15,25,1324,220,15,248050,248049,74,16,25,1324,220,16,248050,248049,74,17,25,1324,220,17,248050,248049,74,18,25,1324,220,18,248050,248049,74,19,25,1324,220,19,248050,248049,74,20,25,1324,220,20,248050,248049,74,21,25,1324,220,21,248050,248049,74,22,25,1324,220,22,248050,248049,74,23,25,1324,220,23,248050,248049,74,24,25,1324,220,24,248050,248049,74,16,15,25,1324,220,16,15,248050,248049,74,16,16,25,1324,220,16,16,248050,248049,74,16,17,25,1324,220,16,17,248050,248049,74,16,18,25,1324,220,16,18,248050,248049,74,16,19,25,1324,220,16,19,248050,248049,74,16,20,25,1324,220,16,20,248050,248049,74,16,21,25,1324,220,16,21,248050,248049,74,16,22,25,1324,220,16,22,248050,248049,74,16,23,25,1324,220,16,23,248050,248049,74,16,24,25,1324,220,16,24,248050,248049,74,17,15,25,1324,220,17,15,248050,248049,74,17,16,25,1324,220,17,16,248050,248049,74,17,17,25,1324,220,17,17,248050,248049,74,17,18,25,1324,220,17,18,248050,248049,74,17,19,25,1324,220,17,19,248050,248049,74,17,20,25,1324,220,17,20,248050,248049,74,17,21,25,1324,220,17,21,248050,248049,74,17,22,25,1324,220,17,22,248050,248049,74,17,23,25,1324,220,17,23,248050,248049,74,17,24,25,1324,220,17,24,248050,248049,74,18,15,25,1324,220,18,15,248050,248062],"positions":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255,256,257,258,259,260,261,262,263,264,265,266,267,268,269,270,271,272,273,274,275,276,277,278,279,280,281,282,283,284,285,286,287,288,289,290,291,292,293,294,295,296,297,298,299,300,301,302,303,304,305,306],"readouts":[23,31,39,47,55,63,71,79,87,95,105,115,125,135,145,155,165,175,185,195,205,215,225,235,245,255,265,275,285,295,305,306]}],"input_tokens":307,"probabilities":[[0.0020161290322580645,0.004032258064516129,0.006048387096774193,0.008064516129032258,0.010080645161290322,0.012096774193548387,0.014112903225806451,0.016129032258064516,0.018145161290322582,0.020161290322580645,0.02217741935483871,0.024193548387096774,0.02620967741935484,0.028225806451612902,0.03024193548387097,0.03225806451612903,0.034274193548387094,0.036290322580645164,0.038306451612903226,0.04032258064516129,0.04233870967741935,0.04435483870967742,0.046370967741935484,0.04838709677419355,0.05040322580645161,0.05241935483870968,0.05443548387096774,0.056451612903225805,0.05846774193548387,0.06048387096774194,0.0625]],"answers":{"pick":{"type":"choice","choice":"k30","confidence":0.0312,"probabilities":{"k0":0.002,"k1":0.004,"k2":0.006,"k3":0.0081,"k4":0.0101,"k5":0.0121,"k6":0.0141,"k7":0.0161,"k8":0.0181,"k9":0.0202,"k10":0.0222,"k11":0.0242,"k12":0.0262,"k13":0.0282,"k14":0.0302,"k15":0.0323,"k16":0.0343,"k17":0.0363,"k18":0.0383,"k19":0.0403,"k20":0.0423,"k21":0.0444,"k22":0.0464,"k23":0.0484,"k24":0.0504,"k25":0.0524,"k26":0.0544,"k27":0.0565,"k28":0.0585,"k29":0.0605,"k30":0.0625}}},"output_tokens":422},{"name":"choice-255","request":{"state":"The number is 254.","questions":{"pick":{"type":"choice","instructions":"Select the option matching the state.","criteria":{"k0":"number 0","k1":"number 1","k2":"number 2","k3":"number 3","k4":"number 4","k5":"number 5","k6":"number 6","k7":"number 7","k8":"number 8","k9":"number 9","k10":"number 10","k11":"number 11","k12":"number 12","k13":"number 13","k14":"number 14","k15":"number 15","k16":"number 16","k17":"number 17","k18":"number 18","k19":"number 19","k20":"number 20","k21":"number 21","k22":"number 22","k23":"number 23","k24":"number 24","k25":"number 25","k26":"number 26","k27":"number 27","k28":"number 28","k29":"number 29","k30":"number 30","k31":"number 31","k32":"number 32","k33":"number 33","k34":"number 34","k35":"number 35","k36":"number 36","k37":"number 37","k38":"number 38","k39":"number 39","k40":"number 40","k41":"number 41","k42":"number 42","k43":"number 43","k44":"number 44","k45":"number 45","k46":"number 46","k47":"number 47","k48":"number 48","k49":"number 49","k50":"number 50","k51":"number 51","k52":"number 52","k53":"number 53","k54":"number 54","k55":"number 55","k56":"number 56","k57":"number 57","k58":"number 58","k59":"number 59","k60":"number 60","k61":"number 61","k62":"number 62","k63":"number 63","k64":"number 64","k65":"number 65","k66":"number 66","k67":"number 67","k68":"number 68","k69":"number 69","k70":"number 70","k71":"number 71","k72":"number 72","k73":"number 73","k74":"number 74","k75":"number 75","k76":"number 76","k77":"number 77","k78":"number 78","k79":"number 79","k80":"number 80","k81":"number 81","k82":"number 82","k83":"number 83","k84":"number 84","k85":"number 85","k86":"number 86","k87":"number 87","k88":"number 88","k89":"number 89","k90":"number 90","k91":"number 91","k92":"number 92","k93":"number 93","k94":"number 94","k95":"number 95","k96":"number 96","k97":"number 97","k98":"number 98","k99":"number 99","k100":"number 100","k101":"number 101","k102":"number 102","k103":"number 103","k104":"number 104","k105":"number 105","k106":"number 106","k107":"number 107","k108":"number 108","k109":"number 109","k110":"number 110","k111":"number 111","k112":"number 112","k113":"number 113","k114":"number 114","k115":"number 115","k116":"number 116","k117":"number 117","k118":"number 118","k119":"number 119","k120":"number 120","k121":"number 121","k122":"number 122","k123":"number 123","k124":"number 124","k125":"number 125","k126":"number 126","k127":"number 127","k128":"number 128","k129":"number 129","k130":"number 130","k131":"number 131","k132":"number 132","k133":"number 133","k134":"number 134","k135":"number 135","k136":"number 136","k137":"number 137","k138":"number 138","k139":"number 139","k140":"number 140","k141":"number 141","k142":"number 142","k143":"number 143","k144":"number 144","k145":"number 145","k146":"number 146","k147":"number 147","k148":"number 148","k149":"number 149","k150":"number 150","k151":"number 151","k152":"number 152","k153":"number 153","k154":"number 154","k155":"number 155","k156":"number 156","k157":"number 157","k158":"number 158","k159":"number 159","k160":"number 160","k161":"number 161","k162":"number 162","k163":"number 163","k164":"number 164","k165":"number 165","k166":"number 166","k167":"number 167","k168":"number 168","k169":"number 169","k170":"number 170","k171":"number 171","k172":"number 172","k173":"number 173","k174":"number 174","k175":"number 175","k176":"number 176","k177":"number 177","k178":"number 178","k179":"number 179","k180":"number 180","k181":"number 181","k182":"number 182","k183":"number 183","k184":"number 184","k185":"number 185","k186":"number 186","k187":"number 187","k188":"number 188","k189":"number 189","k190":"number 190","k191":"number 191","k192":"number 192","k193":"number 193","k194":"number 194","k195":"number 195","k196":"number 196","k197":"number 197","k198":"number 198","k199":"number 199","k200":"number 200","k201":"number 201","k202":"number 202","k203":"number 203","k204":"number 204","k205":"number 205","k206":"number 206","k207":"number 207","k208":"number 208","k209":"number 209","k210":"number 210","k211":"number 211","k212":"number 212","k213":"number 213","k214":"number 214","k215":"number 215","k216":"number 216","k217":"number 217","k218":"number 218","k219":"number 219","k220":"number 220","k221":"number 221","k222":"number 222","k223":"number 223","k224":"number 224","k225":"number 225","k226":"number 226","k227":"number 227","k228":"number 228","k229":"number 229","k230":"number 230","k231":"number 231","k232":"number 232","k233":"number 233","k234":"number 234","k235":"number 235","k236":"number 236","k237":"number 237","k238":"number 238","k239":"number 239","k240":"number 240","k241":"number 241","k242":"number 242","k243":"number 243","k244":"number 244","k245":"number 245","k246":"number 246","k247":"number 247","k248":"number 248","k249":"number 249","k250":"number 250","k251":"number 251","k252":"number 252","k253":"number 253","k254":"number 254"}}}},"record":{"state":"The number is 254.","questions":[{"instr":"Select the option matching the state.","options":["k0: number 0","k1: number 1","k2: number 2","k3: number 3","k4: number 4","k5: number 5","k6: number 6","k7: number 7","k8: number 8","k9: number 9","k10: number 10","k11: number 11","k12: number 12","k13: number 13","k14: number 14","k15: number 15","k16: number 16","k17: number 17","k18: number 18","k19: number 19","k20: number 20","k21: number 21","k22: number 22","k23: number 23","k24: number 24","k25: number 25","k26: number 26","k27: number 27","k28: number 28","k29: number 29","k30: number 30","k31: number 31","k32: number 32","k33: number 33","k34: number 34","k35: number 35","k36: number 36","k37: number 37","k38: number 38","k39: number 39","k40: number 40","k41: number 41","k42: number 42","k43: number 43","k44: number 44","k45: number 45","k46: number 46","k47: number 47","k48: number 48","k49: number 49","k50: number 50","k51: number 51","k52: number 52","k53: number 53","k54: number 54","k55: number 55","k56: number 56","k57: number 57","k58: number 58","k59: number 59","k60: number 60","k61: number 61","k62: number 62","k63: number 63","k64: number 64","k65: number 65","k66: number 66","k67: number 67","k68: number 68","k69: number 69","k70: number 70","k71: number 71","k72: number 72","k73: number 73","k74: number 74","k75: number 75","k76: number 76","k77: number 77","k78: number 78","k79: number 79","k80: number 80","k81: number 81","k82: number 82","k83: number 83","k84: number 84","k85: number 85","k86: number 86","k87: number 87","k88: number 88","k89: number 89","k90: number 90","k91: number 91","k92: number 92","k93: number 93","k94: number 94","k95: number 95","k96: number 96","k97: number 97","k98: number 98","k99: number 99","k100: number 100","k101: number 101","k102: number 102","k103: number 103","k104: number 104","k105: number 105","k106: number 106","k107: number 107","k108: number 108","k109: number 109","k110: number 110","k111: number 111","k112: number 112","k113: number 113","k114: number 114","k115: number 115","k116: number 116","k117: number 117","k118: number 118","k119: number 119","k120: number 120","k121: number 121","k122: number 122","k123: number 123","k124: number 124","k125: number 125","k126: number 126","k127: number 127","k128: number 128","k129: number 129","k130: number 130","k131: number 131","k132: number 132","k133: number 133","k134: number 134","k135: number 135","k136: number 136","k137: number 137","k138: number 138","k139: number 139","k140: number 140","k141: number 141","k142: number 142","k143: number 143","k144: number 144","k145: number 145","k146: number 146","k147: number 147","k148: number 148","k149: number 149","k150: number 150","k151: number 151","k152: number 152","k153: number 153","k154: number 154","k155: number 155","k156: number 156","k157: number 157","k158: number 158","k159: number 159","k160: number 160","k161: number 161","k162: number 162","k163: number 163","k164: number 164","k165: number 165","k166: number 166","k167: number 167","k168: number 168","k169: number 169","k170: number 170","k171: number 171","k172: number 172","k173: number 173","k174: number 174","k175: number 175","k176: number 176","k177: number 177","k178: number 178","k179: number 179","k180: number 180","k181: number 181","k182: number 182","k183: number 183","k184: number 184","k185: number 185","k186: number 186","k187: number 187","k188: number 188","k189: number 189","k190: number 190","k191: number 191","k192: number 192","k193: number 193","k194: number 194","k195: number 195","k196: number 196","k197: number 197","k198: number 198","k199: number 199","k200: number 200","k201: number 201","k202: number 202","k203: number 203","k204: number 204","k205: number 205","k206: number 206","k207: number 207","k208: number 208","k209: number 209","k210: number 210","k211: number 211","k212: number 212","k213: number 213","k214: number 214","k215: number 215","k216: number 216","k217: number 217","k218: number 218","k219: number 219","k220: number 220","k221: number 221","k222: number 222","k223: number 223","k224: number 224","k225: number 225","k226: number 226","k227: number 227","k228: number 228","k229: number 229","k230: number 230","k231: number 231","k232: number 232","k233: number 233","k234: number 234","k235: number 235","k236: number 236","k237: number 237","k238: number 238","k239: number 239","k240: number 240","k241: number 241","k242: number 242","k243: number 243","k244: number 244","k245: number 245","k246: number 246","k247: number 247","k248: number 248","k249: number 249","k250: number 250","k251: number 251","k252: number 252","k253: number 253","k254: number 254"],"label":0}]},"meta":[{"id":"pick","type":"choice","keys":["k0","k1","k2","k3","k4","k5","k6","k7","k8","k9","k10","k11","k12","k13","k14","k15","k16","k17","k18","k19","k20","k21","k22","k23","k24","k25","k26","k27","k28","k29","k30","k31","k32","k33","k34","k35","k36","k37","k38","k39","k40","k41","k42","k43","k44","k45","k46","k47","k48","k49","k50","k51","k52","k53","k54","k55","k56","k57","k58","k59","k60","k61","k62","k63","k64","k65","k66","k67","k68","k69","k70","k71","k72","k73","k74","k75","k76","k77","k78","k79","k80","k81","k82","k83","k84","k85","k86","k87","k88","k89","k90","k91","k92","k93","k94","k95","k96","k97","k98","k99","k100","k101","k102","k103","k104","k105","k106","k107","k108","k109","k110","k111","k112","k113","k114","k115","k116","k117","k118","k119","k120","k121","k122","k123","k124","k125","k126","k127","k128","k129","k130","k131","k132","k133","k134","k135","k136","k137","k138","k139","k140","k141","k142","k143","k144","k145","k146","k147","k148","k149","k150","k151","k152","k153","k154","k155","k156","k157","k158","k159","k160","k161","k162","k163","k164","k165","k166","k167","k168","k169","k170","k171","k172","k173","k174","k175","k176","k177","k178","k179","k180","k181","k182","k183","k184","k185","k186","k187","k188","k189","k190","k191","k192","k193","k194","k195","k196","k197","k198","k199","k200","k201","k202","k203","k204","k205","k206","k207","k208","k209","k210","k211","k212","k213","k214","k215","k216","k217","k218","k219","k220","k221","k222","k223","k224","k225","k226","k227","k228","k229","k230","k231","k232","k233","k234","k235","k236","k237","k238","k239","k240","k241","k242","k243","k244","k245","k246","k247","k248","k249","k250","k251","k252","k253","k254"]}],"rows":[{"ids":[248060,760,1324,369,220,17,20,19,13,248061,3270,279,2904,12219,279,1528,13,248049,74,15,25,1324,220,15,248050,248049,74,16,25,1324,220,16,248050,248049,74,17,25,1324,220,17,248050,248049,74,18,25,1324,220,18,248050,248049,74,19,25,1324,220,19,248050,248049,74,20,25,1324,220,20,248050,248049,74,21,25,1324,220,21,248050,248049,74,22,25,1324,220,22,248050,248049,74,23,25,1324,220,23,248050,248049,74,24,25,1324,220,24,248050,248049,74,16,15,25,1324,220,16,15,248050,248049,74,16,16,25,1324,220,16,16,248050,248049,74,16,17,25,1324,220,16,17,248050,248049,74,16,18,25,1324,220,16,18,248050,248049,74,16,19,25,1324,220,16,19,248050,248049,74,16,20,25,1324,220,16,20,248050,248049,74,16,21,25,1324,220,16,21,248050,248049,74,16,22,25,1324,220,16,22,248050,248049,74,16,23,25,1324,220,16,23,248050,248049,74,16,24,25,1324,220,16,24,248050,248049,74,17,15,25,1324,220,17,15,248050,248049,74,17,16,25,1324,220,17,16,248050,248049,74,17,17,25,1324,220,17,17,248050,248049,74,17,18,25,1324,220,17,18,248050,248049,74,17,19,25,1324,220,17,19,248050,248049,74,17,20,25,1324,220,17,20,248050,248049,74,17,21,25,1324,220,17,21,248050,248049,74,17,22,25,1324,220,17,22,248050,248049,74,17,23,25,1324,220,17,23,248050,248049,74,17,24,25,1324,220,17,24,248050,248049,74,18,15,25,1324,220,18,15,248050,248049,74,18,16,25,1324,220,18,16,248050,248049,74,18,17,25,1324,220,18,17,248050,248049,74,18,18,25,1324,220,18,18,248050,248049,74,18,19,25,1324,220,18,19,248050,248049,74,18,20,25,1324,220,18,20,248050,248049,74,18,21,25,1324,220,18,21,248050,248049,74,18,22,25,1324,220,18,22,248050,248049,74,18,23,25,1324,220,18,23,248050,248049,74,18,24,25,1324,220,18,24,248050,248049,74,19,15,25,1324,220,19,15,248050,248049,74,19,16,25,1324,220,19,16,248050,248049,74,19,17,25,1324,220,19,17,248050,248049,74,19,18,25,1324,220,19,18,248050,248049,74,19,19,25,1324,220,19,19,248050,248049,74,19,20,25,1324,220,19,20,248050,248049,74,19,21,25,1324,220,19,21,248050,248049,74,19,22,25,1324,220,19,22,248050,248049,74,19,23,25,1324,220,19,23,248050,248049,74,19,24,25,1324,220,19,24,248050,248049,74,20,15,25,1324,220,20,15,248050,248049,74,20,16,25,1324,220,20,16,248050,248049,74,20,17,25,1324,220,20,17,248050,248049,74,20,18,25,1324,220,20,18,248050,248049,74,20,19,25,1324,220,20,19,248050,248049,74,20,20,25,1324,220,20,20,248050,248049,74,20,21,25,1324,220,20,21,248050,248049,74,20,22,25,1324,220,20,22,248050,248049,74,20,23,25,1324,220,20,23,248050,248049,74,20,24,25,1324,220,20,24,248050,248049,74,21,15,25,1324,220,21,15,248050,248049,74,21,16,25,1324,220,21,16,248050,248049,74,21,17,25,1324,220,21,17,248050,248049,74,21,18,25,1324,220,21,18,248050,248049,74,21,19,25,1324,220,21,19,248050,248049,74,21,20,25,1324,220,21,20,248050,248049,74,21,21,25,1324,220,21,21,248050,248049,74,21,22,25,1324,220,21,22,248050,248049,74,21,23,25,1324,220,21,23,248050,248049,74,21,24,25,1324,220,21,24,248050,248049,74,22,15,25,1324,220,22,15,248050,248049,74,22,16,25,1324,220,22,16,248050,248049,74,22,17,25,1324,220,22,17,248050,248049,74,22,18,25,1324,220,22,18,248050,248049,74,22,19,25,1324,220,22,19,248050,248049,74,22,20,25,1324,220,22,20,248050,248049,74,22,21,25,1324,220,22,21,248050,248049,74,22,22,25,1324,220,22,22,248050,248049,74,22,23,25,1324,220,22,23,248050,248049,74,22,24,25,1324,220,22,24,248050,248049,74,23,15,25,1324,220,23,15,248050,248049,74,23,16,25,1324,220,23,16,248050,248049,74,23,17,25,1324,220,23,17,248050,248049,74,23,18,25,1324,220,23,18,248050,248049,74,23,19,25,1324,220,23,19,248050,248049,74,23,20,25,1324,220,23,20,248050,248049,74,23,21,25,1324,220,23,21,248050,248049,74,23,22,25,1324,220,23,22,248050,248049,74,23,23,25,1324,220,23,23,248050,248049,74,23,24,25,1324,220,23,24,248050,248049,74,24,15,25,1324,220,24,15,248050,248049,74,24,16,25,1324,220,24,16,248050,248049,74,24,17,25,1324,220,24,17,248050,248049,74,24,18,25,1324,220,24,18,248050,248049,74,24,19,25,1324,220,24,19,248050,248049,74,24,20,25,1324,220,24,20,248050,248049,74,24,21,25,1324,220,24,21,248050,248049,74,24,22,25,1324,220,24,22,248050,248049,74,24,23,25,1324,220,24,23,248050,248049,74,24,24,25,1324,220,24,24,248050,248049,74,16,15,15,25,1324,220,16,15,15,248050,248049,74,16,15,16,25,1324,220,16,15,16,248050,248049,74,16,15,17,25,1324,220,16,15,17,248050,248049,74,16,15,18,25,1324,220,16,15,18,248050,248049,74,16,15,19,25,1324,220,16,15,19,248050,248049,74,16,15,20,25,1324,220,16,15,20,248050,248049,74,16,15,21,25,1324,220,16,15,21,248050,248049,74,16,15,22,25,1324,220,16,15,22,248050,248049,74,16,15,23,25,1324,220,16,15,23,248050,248049,74,16,15,24,25,1324,220,16,15,24,248050,248049,74,16,16,15,25,1324,220,16,16,15,248050,248049,74,16,16,16,25,1324,220,16,16,16,248050,248049,74,16,16,17,25,1324,220,16,16,17,248050,248049,74,16,16,18,25,1324,220,16,16,18,248050,248049,74,16,16,19,25,1324,220,16,16,19,248050,248049,74,16,16,20,25,1324,220,16,16,20,248050,248049,74,16,16,21,25,1324,220,16,16,21,248050,248049,74,16,16,22,25,1324,220,16,16,22,248050,248049,74,16,16,23,25,1324,220,16,16,23,248050,248049,74,16,16,24,25,1324,220,16,16,24,248050,248049,74,16,17,15,25,1324,220,16,17,15,248050,248049,74,16,17,16,25,1324,220,16,17,16,248050,248049,74,16,17,17,25,1324,220,16,17,17,248050,248049,74,16,17,18,25,1324,220,16,17,18,248050,248049,74,16,17,19,25,1324,220,16,17,19,248050,248049,74,16,17,20,25,1324,220,16,17,20,248050,248049,74,16,17,21,25,1324,220,16,17,21,248050,248049,74,16,17,22,25,1324,220,16,17,22,248050,248049,74,16,17,23,25,1324,220,16,17,23,248050,248049,74,16,17,24,25,1324,220,16,17,24,248050,248049,74,16,18,15,25,1324,220,16,18,15,248050,248049,74,16,18,16,25,1324,220,16,18,16,248050,248049,74,16,18,17,25,1324,220,16,18,17,248050,248049,74,16,18,18,25,1324,220,16,18,18,248050,248049,74,16,18,19,25,1324,220,16,18,19,248050,248049,74,16,18,20,25,1324,220,16,18,20,248050,248049,74,16,18,21,25,1324,220,16,18,21,248050,248049,74,16,18,22,25,1324,220,16,18,22,248050,248049,74,16,18,23,25,1324,220,16,18,23,248050,248049,74,16,18,24,25,1324,220,16,18,24,248050,248049,74,16,19,15,25,1324,220,16,19,15,248050,248049,74,16,19,16,25,1324,220,16,19,16,248050,248049,74,16,19,17,25,1324,220,16,19,17,248050,248049,74,16,19,18,25,1324,220,16,19,18,248050,248049,74,16,19,19,25,1324,220,16,19,19,248050,248049,74,16,19,20,25,1324,220,16,19,20,248050,248049,74,16,19,21,25,1324,220,16,19,21,248050,248049,74,16,19,22,25,1324,220,16,19,22,248050,248049,74,16,19,23,25,1324,220,16,19,23,248050,248049,74,16,19,24,25,1324,220,16,19,24,248050,248049,74,16,20,15,25,1324,220,16,20,15,248050,248049,74,16,20,16,25,1324,220,16,20,16,248050,248049,74,16,20,17,25,1324,220,16,20,17,248050,248049,74,16,20,18,25,1324,220,16,20,18,248050,248049,74,16,20,19,25,1324,220,16,20,19,248050,248049,74,16,20,20,25,1324,220,16,20,20,248050,248049,74,16,20,21,25,1324,220,16,20,21,248050,248049,74,16,20,22,25,1324,220,16,20,22,248050,248049,74,16,20,23,25,1324,220,16,20,23,248050,248049,74,16,20,24,25,1324,220,16,20,24,248050,248049,74,16,21,15,25,1324,220,16,21,15,248050,248049,74,16,21,16,25,1324,220,16,21,16,248050,248049,74,16,21,17,25,1324,220,16,21,17,248050,248049,74,16,21,18,25,1324,220,16,21,18,248050,248049,74,16,21,19,25,1324,220,16,21,19,248050,248049,74,16,21,20,25,1324,220,16,21,20,248050,248049,74,16,21,21,25,1324,220,16,21,21,248050,248049,74,16,21,22,25,1324,220,16,21,22,248050,248049,74,16,21,23,25,1324,220,16,21,23,248050,248049,74,16,21,24,25,1324,220,16,21,24,248050,248049,74,16,22,15,25,1324,220,16,22,15,248050,248049,74,16,22,16,25,1324,220,16,22,16,248050,248049,74,16,22,17,25,1324,220,16,22,17,248050,248049,74,16,22,18,25,1324,220,16,22,18,248050,248049,74,16,22,19,25,1324,220,16,22,19,248050,248049,74,16,22,20,25,1324,220,16,22,20,248050,248049,74,16,22,21,25,1324,220,16,22,21,248050,248049,74,16,22,22,25,1324,220,16,22,22,248050,248049,74,16,22,23,25,1324,220,16,22,23,248050,248049,74,16,22,24,25,1324,220,16,22,24,248050,248049,74,16,23,15,25,1324,220,16,23,15,248050,248049,74,16,23,16,25,1324,220,16,23,16,248050,248049,74,16,23,17,25,1324,220,16,23,17,248050,248049,74,16,23,18,25,1324,220,16,23,18,248050,248049,74,16,23,19,25,1324,220,16,23,19,248050,248049,74,16,23,20,25,1324,220,16,23,20,248050,248049,74,16,23,21,25,1324,220,16,23,21,248050,248049,74,16,23,22,25,1324,220,16,23,22,248050,248049,74,16,23,23,25,1324,220,16,23,23,248050,248049,74,16,23,24,25,1324,220,16,23,24,248050,248049,74,16,24,15,25,1324,220,16,24,15,248050,248049,74,16,24,16,25,1324,220,16,24,16,248050,248049,74,16,24,17,25,1324,220,16,24,17,248050,248049,74,16,24,18,25,1324,220,16,24,18,248050,248049,74,16,24,19,25,1324,220,16,24,19,248050,248049,74,16,24,20,25,1324,220,16,24,20,248050,248049,74,16,24,21,25,1324,220,16,24,21,248050,248049,74,16,24,22,25,1324,220,16,24,22,248050,248049,74,16,24,23,25,1324,220,16,24,23,248050,248049,74,16,24,24,25,1324,220,16,24,24,248050,248049,74,17,15,15,25,1324,220,17,15,15,248050,248049,74,17,15,16,25,1324,220,17,15,16,248050,248049,74,17,15,17,25,1324,220,17,15,17,248050,248049,74,17,15,18,25,1324,220,17,15,18,248050,248049,74,17,15,19,25,1324,220,17,15,19,248050,248049,74,17,15,20,25,1324,220,17,15,20,248050,248049,74,17,15,21,25,1324,220,17,15,21,248050,248049,74,17,15,22,25,1324,220,17,15,22,248050,248049,74,17,15,23,25,1324,220,17,15,23,248050,248049,74,17,15,24,25,1324,220,17,15,24,248050,248049,74,17,16,15,25,1324,220,17,16,15,248050,248049,74,17,16,16,25,1324,220,17,16,16,248050,248049,74,17,16,17,25,1324,220,17,16,17,248050,248049,74,17,16,18,25,1324,220,17,16,18,248050,248049,74,17,16,19,25,1324,220,17,16,19,248050,248049,74,17,16,20,25,1324,220,17,16,20,248050,248049,74,17,16,21,25,1324,220,17,16,21,248050,248049,74,17,16,22,25,1324,220,17,16,22,248050,248049,74,17,16,23,25,1324,220,17,16,23,248050,248049,74,17,16,24,25,1324,220,17,16,24,248050,248049,74,17,17,15,25,1324,220,17,17,15,248050,248049,74,17,17,16,25,1324,220,17,17,16,248050,248049,74,17,17,17,25,1324,220,17,17,17,248050,248049,74,17,17,18,25,1324,220,17,17,18,248050,248049,74,17,17,19,25,1324,220,17,17,19,248050,248049,74,17,17,20,25,1324,220,17,17,20,248050,248049,74,17,17,21,25,1324,220,17,17,21,248050,248049,74,17,17,22,25,1324,220,17,17,22,248050,248049,74,17,17,23,25,1324,220,17,17,23,248050,248049,74,17,17,24,25,1324,220,17,17,24,248050,248049,74,17,18,15,25,1324,220,17,18,15,248050,248049,74,17,18,16,25,1324,220,17,18,16,248050,248049,74,17,18,17,25,1324,220,17,18,17,248050,248049,74,17,18,18,25,1324,220,17,18,18,248050,248049,74,17,18,19,25,1324,220,17,18,19,248050,248049,74,17,18,20,25,1324,220,17,18,20,248050,248049,74,17,18,21,25,1324,220,17,18,21,248050,248049,74,17,18,22,25,1324,220,17,18,22,248050,248049,74,17,18,23,25,1324,220,17,18,23,248050,248049,74,17,18,24,25,1324,220,17,18,24,248050,248049,74,17,19,15,25,1324,220,17,19,15,248050,248049,74,17,19,16,25,1324,220,17,19,16,248050,248049,74,17,19,17,25,1324,220,17,19,17,248050,248049,74,17,19,18,25,1324,220,17,19,18,248050,248049,74,17,19,19,25,1324,220,17,19,19,248050,248049,74,17,19,20,25,1324,220,17,19,20,248050,248049,74,17,19,21,25,1324,220,17,19,21,248050,248049,74,17,19,22,25,1324,220,17,19,22,248050,248049,74,17,19,23,25,1324,220,17,19,23,248050,248049,74,17,19,24,25,1324,220,17,19,24,248050,248049,74,17,20,15,25,1324,220,17,20,15,248050,248049,74,17,20,16,25,1324,220,17,20,16,248050,248049,74,17,20,17,25,1324,220,17,20,17,248050,248049,74,17,20,18,25,1324,220,17,20,18,248050,248049,74,17,20,19,25,1324,220,17,20,19,248050,248062],"positions":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255,256,257,258,259,260,261,262,263,264,265,266,267,268,269,270,271,272,273,274,275,276,277,278,279,280,281,282,283,284,285,286,287,288,289,290,291,292,293,294,295,296,297,298,299,300,301,302,303,304,305,306,307,308,309,310,311,312,313,314,315,316,317,318,319,320,321,322,323,324,325,326,327,328,329,330,331,332,333,334,335,336,337,338,339,340,341,342,343,344,345,346,347,348,349,350,351,352,353,354,355,356,357,358,359,360,361,362,363,364,365,366,367,368,369,370,371,372,373,374,375,376,377,378,379,380,381,382,383,384,385,386,387,388,389,390,391,392,393,394,395,396,397,398,399,400,401,402,403,404,405,406,407,408,409,410,411,412,413,414,415,416,417,418,419,420,421,422,423,424,425,426,427,428,429,430,431,432,433,434,435,436,437,438,439,440,441,442,443,444,445,446,447,448,449,450,451,452,453,454,455,456,457,458,459,460,461,462,463,464,465,466,467,468,469,470,471,472,473,474,475,476,477,478,479,480,481,482,483,484,485,486,487,488,489,490,491,492,493,494,495,496,497,498,499,500,501,502,503,504,505,506,507,508,509,510,511,512,513,514,515,516,517,518,519,520,521,522,523,524,525,526,527,528,529,530,531,532,533,534,535,536,537,538,539,540,541,542,543,544,545,546,547,548,549,550,551,552,553,554,555,556,557,558,559,560,561,562,563,564,565,566,567,568,569,570,571,572,573,574,575,576,577,578,579,580,581,582,583,584,585,586,587,588,589,590,591,592,593,594,595,596,597,598,599,600,601,602,603,604,605,606,607,608,609,610,611,612,613,614,615,616,617,618,619,620,621,622,623,624,625,626,627,628,629,630,631,632,633,634,635,636,637,638,639,640,641,642,643,644,645,646,647,648,649,650,651,652,653,654,655,656,657,658,659,660,661,662,663,664,665,666,667,668,669,670,671,672,673,674,675,676,677,678,679,680,681,682,683,684,685,686,687,688,689,690,691,692,693,694,695,696,697,698,699,700,701,702,703,704,705,706,707,708,709,710,711,712,713,714,715,716,717,718,719,720,721,722,723,724,725,726,727,728,729,730,731,732,733,734,735,736,737,738,739,740,741,742,743,744,745,746,747,748,749,750,751,752,753,754,755,756,757,758,759,760,761,762,763,764,765,766,767,768,769,770,771,772,773,774,775,776,777,778,779,780,781,782,783,784,785,786,787,788,789,790,791,792,793,794,795,796,797,798,799,800,801,802,803,804,805,806,807,808,809,810,811,812,813,814,815,816,817,818,819,820,821,822,823,824,825,826,827,828,829,830,831,832,833,834,835,836,837,838,839,840,841,842,843,844,845,846,847,848,849,850,851,852,853,854,855,856,857,858,859,860,861,862,863,864,865,866,867,868,869,870,871,872,873,874,875,876,877,878,879,880,881,882,883,884,885,886,887,888,889,890,891,892,893,894,895,896,897,898,899,900,901,902,903,904,905,906,907,908,909,910,911,912,913,914,915,916,917,918,919,920,921,922,923,924,925,926,927,928,929,930,931,932,933,934,935,936,937,938,939,940,941,942,943,944,945,946,947,948,949,950,951,952,953,954,955,956,957,958,959,960,961,962,963,964,965,966,967,968,969,970,971,972,973,974,975,976,977,978,979,980,981,982,983,984,985,986,987,988,989,990,991,992,993,994,995,996,997,998,999,1000,1001,1002,1003,1004,1005,1006,1007,1008,1009,1010,1011,1012,1013,1014,1015,1016,1017,1018,1019,1020,1021,1022,1023,1024,1025,1026,1027,1028,1029,1030,1031,1032,1033,1034,1035,1036,1037,1038,1039,1040,1041,1042,1043,1044,1045,1046,1047,1048,1049,1050,1051,1052,1053,1054,1055,1056,1057,1058,1059,1060,1061,1062,1063,1064,1065,1066,1067,1068,1069,1070,1071,1072,1073,1074,1075,1076,1077,1078,1079,1080,1081,1082,1083,1084,1085,1086,1087,1088,1089,1090,1091,1092,1093,1094,1095,1096,1097,1098,1099,1100,1101,1102,1103,1104,1105,1106,1107,1108,1109,1110,1111,1112,1113,1114,1115,1116,1117,1118,1119,1120,1121,1122,1123,1124,1125,1126,1127,1128,1129,1130,1131,1132,1133,1134,1135,1136,1137,1138,1139,1140,1141,1142,1143,1144,1145,1146,1147,1148,1149,1150,1151,1152,1153,1154,1155,1156,1157,1158,1159,1160,1161,1162,1163,1164,1165,1166,1167,1168,1169,1170,1171,1172,1173,1174,1175,1176,1177,1178,1179,1180,1181,1182,1183,1184,1185,1186,1187,1188,1189,1190,1191,1192,1193,1194,1195,1196,1197,1198,1199,1200,1201,1202,1203,1204,1205,1206,1207,1208,1209,1210,1211,1212,1213,1214,1215,1216,1217,1218,1219,1220,1221,1222,1223,1224,1225,1226,1227,1228,1229,1230,1231,1232,1233,1234,1235,1236,1237,1238,1239,1240,1241,1242,1243,1244,1245,1246,1247,1248,1249,1250,1251,1252,1253,1254,1255,1256,1257,1258,1259,1260,1261,1262,1263,1264,1265,1266,1267,1268,1269,1270,1271,1272,1273,1274,1275,1276,1277,1278,1279,1280,1281,1282,1283,1284,1285,1286,1287,1288,1289,1290,1291,1292,1293,1294,1295,1296,1297,1298,1299,1300,1301,1302,1303,1304,1305,1306,1307,1308,1309,1310,1311,1312,1313,1314,1315,1316,1317,1318,1319,1320,1321,1322,1323,1324,1325,1326,1327,1328,1329,1330,1331,1332,1333,1334,1335,1336,1337,1338,1339,1340,1341,1342,1343,1344,1345,1346,1347,1348,1349,1350,1351,1352,1353,1354,1355,1356,1357,1358,1359,1360,1361,1362,1363,1364,1365,1366,1367,1368,1369,1370,1371,1372,1373,1374,1375,1376,1377,1378,1379,1380,1381,1382,1383,1384,1385,1386,1387,1388,1389,1390,1391,1392,1393,1394,1395,1396,1397,1398,1399,1400,1401,1402,1403,1404,1405,1406,1407,1408,1409,1410,1411,1412,1413,1414,1415,1416,1417,1418,1419,1420,1421,1422,1423,1424,1425,1426,1427,1428,1429,1430,1431,1432,1433,1434,1435,1436,1437,1438,1439,1440,1441,1442,1443,1444,1445,1446,1447,1448,1449,1450,1451,1452,1453,1454,1455,1456,1457,1458,1459,1460,1461,1462,1463,1464,1465,1466,1467,1468,1469,1470,1471,1472,1473,1474,1475,1476,1477,1478,1479,1480,1481,1482,1483,1484,1485,1486,1487,1488,1489,1490,1491,1492,1493,1494,1495,1496,1497,1498,1499,1500,1501,1502,1503,1504,1505,1506,1507,1508,1509,1510,1511,1512,1513,1514,1515,1516,1517,1518,1519,1520,1521,1522,1523,1524,1525,1526,1527,1528,1529,1530,1531,1532,1533,1534,1535,1536,1537,1538,1539,1540,1541,1542,1543,1544,1545,1546,1547,1548,1549,1550,1551,1552,1553,1554,1555,1556,1557,1558,1559,1560,1561,1562,1563,1564,1565,1566,1567,1568,1569,1570,1571,1572,1573,1574,1575,1576,1577,1578,1579,1580,1581,1582,1583,1584,1585,1586,1587,1588,1589,1590,1591,1592,1593,1594,1595,1596,1597,1598,1599,1600,1601,1602,1603,1604,1605,1606,1607,1608,1609,1610,1611,1612,1613,1614,1615,1616,1617,1618,1619,1620,1621,1622,1623,1624,1625,1626,1627,1628,1629,1630,1631,1632,1633,1634,1635,1636,1637,1638,1639,1640,1641,1642,1643,1644,1645,1646,1647,1648,1649,1650,1651,1652,1653,1654,1655,1656,1657,1658,1659,1660,1661,1662,1663,1664,1665,1666,1667,1668,1669,1670,1671,1672,1673,1674,1675,1676,1677,1678,1679,1680,1681,1682,1683,1684,1685,1686,1687,1688,1689,1690,1691,1692,1693,1694,1695,1696,1697,1698,1699,1700,1701,1702,1703,1704,1705,1706,1707,1708,1709,1710,1711,1712,1713,1714,1715,1716,1717,1718,1719,1720,1721,1722,1723,1724,1725,1726,1727,1728,1729,1730,1731,1732,1733,1734,1735,1736,1737,1738,1739,1740,1741,1742,1743,1744,1745,1746,1747,1748,1749,1750,1751,1752,1753,1754,1755,1756,1757,1758,1759,1760,1761,1762,1763,1764,1765,1766,1767,1768,1769,1770,1771,1772,1773,1774,1775,1776,1777,1778,1779,1780,1781,1782,1783,1784,1785,1786,1787,1788,1789,1790,1791,1792,1793,1794,1795,1796,1797,1798,1799,1800,1801,1802,1803,1804,1805,1806,1807,1808,1809,1810,1811,1812,1813,1814,1815,1816,1817,1818,1819,1820,1821,1822,1823,1824,1825,1826,1827,1828,1829,1830,1831,1832,1833,1834,1835,1836,1837,1838,1839,1840,1841,1842,1843,1844,1845,1846,1847,1848,1849,1850,1851,1852,1853,1854,1855,1856,1857,1858,1859,1860,1861,1862,1863,1864,1865,1866,1867,1868,1869,1870,1871,1872,1873,1874,1875,1876,1877,1878,1879,1880,1881,1882,1883,1884,1885,1886,1887,1888,1889,1890,1891,1892,1893,1894,1895,1896,1897,1898,1899,1900,1901,1902,1903,1904,1905,1906,1907,1908,1909,1910,1911,1912,1913,1914,1915,1916,1917,1918,1919,1920,1921,1922,1923,1924,1925,1926,1927,1928,1929,1930,1931,1932,1933,1934,1935,1936,1937,1938,1939,1940,1941,1942,1943,1944,1945,1946,1947,1948,1949,1950,1951,1952,1953,1954,1955,1956,1957,1958,1959,1960,1961,1962,1963,1964,1965,1966,1967,1968,1969,1970,1971,1972,1973,1974,1975,1976,1977,1978,1979,1980,1981,1982,1983,1984,1985,1986,1987,1988,1989,1990,1991,1992,1993,1994,1995,1996,1997,1998,1999,2000,2001,2002,2003,2004,2005,2006,2007,2008,2009,2010,2011,2012,2013,2014,2015,2016,2017,2018,2019,2020,2021,2022,2023,2024,2025,2026,2027,2028,2029,2030,2031,2032,2033,2034,2035,2036,2037,2038,2039,2040,2041,2042,2043,2044,2045,2046,2047,2048,2049,2050,2051,2052,2053,2054,2055,2056,2057,2058,2059,2060,2061,2062,2063,2064,2065,2066,2067,2068,2069,2070,2071,2072,2073,2074,2075,2076,2077,2078,2079,2080,2081,2082,2083,2084,2085,2086,2087,2088,2089,2090,2091,2092,2093,2094,2095,2096,2097,2098,2099,2100,2101,2102,2103,2104,2105,2106,2107,2108,2109,2110,2111,2112,2113,2114,2115,2116,2117,2118,2119,2120,2121,2122,2123,2124,2125,2126,2127,2128,2129,2130,2131,2132,2133,2134,2135,2136,2137,2138,2139,2140,2141,2142,2143,2144,2145,2146,2147,2148,2149,2150,2151,2152,2153,2154,2155,2156,2157,2158,2159,2160,2161,2162,2163,2164,2165,2166,2167,2168,2169,2170,2171,2172,2173,2174,2175,2176,2177,2178,2179,2180,2181,2182,2183,2184,2185,2186,2187,2188,2189,2190,2191,2192,2193,2194,2195,2196,2197,2198,2199,2200,2201,2202,2203,2204,2205,2206,2207,2208,2209,2210,2211,2212,2213,2214,2215,2216,2217,2218,2219,2220,2221,2222,2223,2224,2225,2226,2227,2228,2229,2230,2231,2232,2233,2234,2235,2236,2237,2238,2239,2240,2241,2242,2243,2244,2245,2246,2247,2248,2249,2250,2251,2252,2253,2254,2255,2256,2257,2258,2259,2260,2261,2262,2263,2264,2265,2266,2267,2268,2269,2270,2271,2272,2273,2274,2275,2276,2277,2278,2279,2280,2281,2282,2283,2284,2285,2286,2287,2288,2289,2290,2291,2292,2293,2294,2295,2296,2297,2298,2299,2300,2301,2302,2303,2304,2305,2306,2307,2308,2309,2310,2311,2312,2313,2314,2315,2316,2317,2318,2319,2320,2321,2322,2323,2324,2325,2326,2327,2328,2329,2330,2331,2332,2333,2334,2335,2336,2337,2338,2339,2340,2341,2342,2343,2344,2345,2346,2347,2348,2349,2350,2351,2352,2353,2354,2355,2356,2357,2358,2359,2360,2361,2362,2363,2364,2365,2366,2367,2368,2369,2370,2371,2372,2373,2374,2375,2376,2377,2378,2379,2380,2381,2382,2383,2384,2385,2386,2387,2388,2389,2390,2391,2392,2393,2394,2395,2396,2397,2398,2399,2400,2401,2402,2403,2404,2405,2406,2407,2408,2409,2410,2411,2412,2413,2414,2415,2416,2417,2418,2419,2420,2421,2422,2423,2424,2425,2426,2427,2428,2429,2430,2431,2432,2433,2434,2435,2436,2437,2438,2439,2440,2441,2442,2443,2444,2445,2446,2447,2448,2449,2450,2451,2452,2453,2454,2455,2456,2457,2458,2459,2460,2461,2462,2463,2464,2465,2466,2467,2468,2469,2470,2471,2472,2473,2474,2475,2476,2477,2478,2479,2480,2481,2482,2483,2484,2485,2486,2487,2488,2489,2490,2491,2492,2493,2494,2495,2496,2497,2498,2499,2500,2501,2502,2503,2504,2505,2506,2507,2508,2509,2510,2511,2512,2513,2514,2515,2516,2517,2518,2519,2520,2521,2522,2523,2524,2525,2526,2527,2528,2529,2530,2531,2532,2533,2534,2535,2536,2537,2538,2539,2540,2541,2542,2543,2544,2545,2546,2547,2548,2549,2550,2551,2552,2553,2554,2555,2556,2557,2558,2559,2560,2561,2562,2563,2564,2565,2566,2567,2568,2569,2570,2571,2572,2573,2574,2575,2576,2577,2578,2579,2580,2581,2582,2583,2584,2585,2586,2587,2588,2589,2590,2591,2592,2593,2594,2595,2596,2597,2598,2599,2600,2601,2602,2603,2604,2605,2606,2607,2608,2609,2610,2611,2612,2613,2614,2615,2616,2617,2618,2619,2620,2621,2622,2623,2624,2625,2626,2627,2628,2629,2630,2631,2632,2633,2634,2635,2636,2637,2638,2639,2640,2641,2642,2643,2644,2645,2646,2647,2648,2649,2650,2651,2652,2653,2654,2655,2656,2657,2658,2659,2660,2661,2662,2663,2664,2665,2666,2667,2668,2669,2670,2671,2672,2673,2674,2675,2676,2677,2678,2679,2680,2681,2682,2683,2684,2685,2686,2687,2688,2689,2690,2691,2692,2693,2694,2695,2696,2697,2698,2699,2700,2701,2702,2703,2704,2705,2706,2707,2708,2709,2710,2711,2712,2713,2714,2715,2716,2717,2718,2719,2720,2721,2722,2723,2724,2725,2726,2727,2728,2729,2730,2731,2732,2733,2734,2735,2736,2737,2738,2739,2740,2741,2742,2743,2744,2745,2746,2747,2748,2749,2750,2751,2752,2753,2754,2755,2756,2757,2758,2759,2760,2761,2762,2763,2764,2765,2766,2767,2768,2769,2770,2771,2772,2773,2774,2775,2776,2777,2778,2779,2780,2781,2782,2783,2784,2785,2786,2787,2788,2789,2790,2791,2792,2793,2794,2795,2796,2797,2798,2799,2800,2801,2802,2803,2804,2805,2806,2807,2808,2809,2810,2811,2812,2813,2814,2815,2816,2817,2818,2819,2820,2821,2822,2823,2824,2825,2826,2827,2828,2829,2830,2831,2832,2833,2834,2835,2836,2837,2838,2839,2840,2841,2842,2843,2844,2845,2846,2847,2848,2849,2850,2851,2852,2853,2854,2855,2856,2857],"readouts":[24,32,40,48,56,64,72,80,88,96,106,116,126,136,146,156,166,176,186,196,206,216,226,236,246,256,266,276,286,296,306,316,326,336,346,356,366,376,386,396,406,416,426,436,446,456,466,476,486,496,506,516,526,536,546,556,566,576,586,596,606,616,626,636,646,656,666,676,686,696,706,716,726,736,746,756,766,776,786,796,806,816,826,836,846,856,866,876,886,896,906,916,926,936,946,956,966,976,986,996,1008,1020,1032,1044,1056,1068,1080,1092,1104,1116,1128,1140,1152,1164,1176,1188,1200,1212,1224,1236,1248,1260,1272,1284,1296,1308,1320,1332,1344,1356,1368,1380,1392,1404,1416,1428,1440,1452,1464,1476,1488,1500,1512,1524,1536,1548,1560,1572,1584,1596,1608,1620,1632,1644,1656,1668,1680,1692,1704,1716,1728,1740,1752,1764,1776,1788,1800,1812,1824,1836,1848,1860,1872,1884,1896,1908,1920,1932,1944,1956,1968,1980,1992,2004,2016,2028,2040,2052,2064,2076,2088,2100,2112,2124,2136,2148,2160,2172,2184,2196,2208,2220,2232,2244,2256,2268,2280,2292,2304,2316,2328,2340,2352,2364,2376,2388,2400,2412,2424,2436,2448,2460,2472,2484,2496,2508,2520,2532,2544,2556,2568,2580,2592,2604,2616,2628,2640,2652,2664,2676,2688,2700,2712,2724,2736,2748,2760,2772,2784,2796,2808,2820,2832,2844,2856,2857]}],"input_tokens":2858,"probabilities":[[3.0637254901960784e-05,6.127450980392157e-05,9.191176470588235e-05,0.00012254901960784314,0.00015318627450980392,0.0001838235294117647,0.0002144607843137255,0.00024509803921568627,0.00027573529411764705,0.00030637254901960784,0.0003370098039215686,0.0003676470588235294,0.0003982843137254902,0.000428921568627451,0.00045955882352941176,0.0004901960784313725,0.0005208333333333333,0.0005514705882352941,0.0005821078431372549,0.0006127450980392157,0.0006433823529411765,0.0006740196078431372,0.000704656862745098,0.0007352941176470588,0.0007659313725490196,0.0007965686274509804,0.0008272058823529412,0.000857843137254902,0.0008884803921568627,0.0009191176470588235,0.0009497549019607843,0.000980392156862745,0.001011029411764706,0.0010416666666666667,0.0010723039215686275,0.0011029411764705882,0.0011335784313725491,0.0011642156862745098,0.0011948529411764707,0.0012254901960784314,0.0012561274509803922,0.001286764705882353,0.0013174019607843138,0.0013480392156862745,0.0013786764705882354,0.001409313725490196,0.001439950980392157,0.0014705882352941176,0.0015012254901960785,0.0015318627450980392,0.0015625,0.0015931372549019608,0.0016237745098039217,0.0016544117647058823,0.0016850490196078432,0.001715686274509804,0.0017463235294117648,0.0017769607843137255,0.0018075980392156864,0.001838235294117647,0.001868872549019608,0.0018995098039215686,0.0019301470588235295,0.00196078431372549,0.001991421568627451,0.002022058823529412,0.0020526960784313726,0.0020833333333333333,0.002113970588235294,0.002144607843137255,0.0021752450980392158,0.0022058823529411764,0.002236519607843137,0.0022671568627450982,0.002297794117647059,0.0023284313725490196,0.0023590686274509803,0.0023897058823529414,0.002420343137254902,0.0024509803921568627,0.0024816176470588234,0.0025122549019607845,0.002542892156862745,0.002573529411764706,0.0026041666666666665,0.0026348039215686276,0.0026654411764705883,0.002696078431372549,0.0027267156862745097,0.0027573529411764708,0.0027879901960784314,0.002818627450980392,0.002849264705882353,0.002879901960784314,0.0029105392156862746,0.0029411764705882353,0.002971813725490196,0.003002450980392157,0.0030330882352941177,0.0030637254901960784,0.003094362745098039,0.003125,0.003155637254901961,0.0031862745098039215,0.003216911764705882,0.0032475490196078433,0.003278186274509804,0.0033088235294117647,0.0033394607843137253,0.0033700980392156864,0.003400735294117647,0.003431372549019608,0.0034620098039215685,0.0034926470588235296,0.0035232843137254903,0.003553921568627451,0.0035845588235294116,0.0036151960784313727,0.0036458333333333334,0.003676470588235294,0.0037071078431372547,0.003737745098039216,0.0037683823529411765,0.003799019607843137,0.003829656862745098,0.003860294117647059,0.0038909313725490197,0.00392156862745098,0.003952205882352941,0.003982843137254902,0.004013480392156862,0.004044117647058824,0.004074754901960785,0.004105392156862745,0.004136029411764706,0.004166666666666667,0.004197303921568627,0.004227941176470588,0.004258578431372549,0.00428921568627451,0.004319852941176471,0.0043504901960784315,0.004381127450980392,0.004411764705882353,0.0044424019607843136,0.004473039215686274,0.004503676470588235,0.0045343137254901964,0.004564950980392157,0.004595588235294118,0.0046262254901960785,0.004656862745098039,0.0046875,0.0047181372549019605,0.004748774509803921,0.004779411764705883,0.004810049019607843,0.004840686274509804,0.004871323529411765,0.004901960784313725,0.004932598039215686,0.004963235294117647,0.0049938725490196074,0.005024509803921569,0.00505514705882353,0.00508578431372549,0.005116421568627451,0.005147058823529412,0.005177696078431372,0.005208333333333333,0.005238970588235294,0.005269607843137255,0.005300245098039216,0.005330882352941177,0.005361519607843137,0.005392156862745098,0.005422794117647059,0.005453431372549019,0.00548406862745098,0.0055147058823529415,0.005545343137254902,0.005575980392156863,0.0056066176470588236,0.005637254901960784,0.005667892156862745,0.005698529411764706,0.005729166666666666,0.005759803921568628,0.0057904411764705885,0.005821078431372549,0.00585171568627451,0.0058823529411764705,0.005912990196078431,0.005943627450980392,0.0059742647058823525,0.006004901960784314,0.006035539215686275,0.006066176470588235,0.006096813725490196,0.006127450980392157,0.0061580882352941175,0.006188725490196078,0.006219362745098039,0.00625,0.006280637254901961,0.006311274509803922,0.006341911764705882,0.006372549019607843,0.006403186274509804,0.006433823529411764,0.006464460784313725,0.006495098039215687,0.006525735294117647,0.006556372549019608,0.006587009803921569,0.006617647058823529,0.00664828431372549,0.006678921568627451,0.006709558823529411,0.006740196078431373,0.0067708333333333336,0.006801470588235294,0.006832107843137255,0.006862745098039216,0.006893382352941176,0.006924019607843137,0.006954656862745098,0.006985294117647059,0.00701593137254902,0.0070465686274509805,0.007077205882352941,0.007107843137254902,0.0071384803921568625,0.007169117647058823,0.007199754901960784,0.007230392156862745,0.007261029411764706,0.007291666666666667,0.0073223039215686275,0.007352941176470588,0.007383578431372549,0.0074142156862745095,0.00744485294117647,0.007475490196078432,0.007506127450980392,0.007536764705882353,0.007567401960784314,0.007598039215686274,0.007628676470588235,0.007659313725490196,0.007689950980392156,0.007720588235294118,0.007751225490196079,0.007781862745098039,0.0078125]],"answers":{"pick":{"type":"choice","choice":"k254","confidence":0.0039,"probabilities":{"k0":0.0,"k1":0.0001,"k2":0.0001,"k3":0.0001,"k4":0.0002,"k5":0.0002,"k6":0.0002,"k7":0.0002,"k8":0.0003,"k9":0.0003,"k10":0.0003,"k11":0.0004,"k12":0.0004,"k13":0.0004,"k14":0.0005,"k15":0.0005,"k16":0.0005,"k17":0.0006,"k18":0.0006,"k19":0.0006,"k20":0.0006,"k21":0.0007,"k22":0.0007,"k23":0.0007,"k24":0.0008,"k25":0.0008,"k26":0.0008,"k27":0.0009,"k28":0.0009,"k29":0.0009,"k30":0.0009,"k31":0.001,"k32":0.001,"k33":0.001,"k34":0.0011,"k35":0.0011,"k36":0.0011,"k37":0.0012,"k38":0.0012,"k39":0.0012,"k40":0.0013,"k41":0.0013,"k42":0.0013,"k43":0.0013,"k44":0.0014,"k45":0.0014,"k46":0.0014,"k47":0.0015,"k48":0.0015,"k49":0.0015,"k50":0.0016,"k51":0.0016,"k52":0.0016,"k53":0.0017,"k54":0.0017,"k55":0.0017,"k56":0.0017,"k57":0.0018,"k58":0.0018,"k59":0.0018,"k60":0.0019,"k61":0.0019,"k62":0.0019,"k63":0.002,"k64":0.002,"k65":0.002,"k66":0.0021,"k67":0.0021,"k68":0.0021,"k69":0.0021,"k70":0.0022,"k71":0.0022,"k72":0.0022,"k73":0.0023,"k74":0.0023,"k75":0.0023,"k76":0.0024,"k77":0.0024,"k78":0.0024,"k79":0.0025,"k80":0.0025,"k81":0.0025,"k82":0.0025,"k83":0.0026,"k84":0.0026,"k85":0.0026,"k86":0.0027,"k87":0.0027,"k88":0.0027,"k89":0.0028,"k90":0.0028,"k91":0.0028,"k92":0.0028,"k93":0.0029,"k94":0.0029,"k95":0.0029,"k96":0.003,"k97":0.003,"k98":0.003,"k99":0.0031,"k100":0.0031,"k101":0.0031,"k102":0.0032,"k103":0.0032,"k104":0.0032,"k105":0.0032,"k106":0.0033,"k107":0.0033,"k108":0.0033,"k109":0.0034,"k110":0.0034,"k111":0.0034,"k112":0.0035,"k113":0.0035,"k114":0.0035,"k115":0.0036,"k116":0.0036,"k117":0.0036,"k118":0.0036,"k119":0.0037,"k120":0.0037,"k121":0.0037,"k122":0.0038,"k123":0.0038,"k124":0.0038,"k125":0.0039,"k126":0.0039,"k127":0.0039,"k128":0.004,"k129":0.004,"k130":0.004,"k131":0.004,"k132":0.0041,"k133":0.0041,"k134":0.0041,"k135":0.0042,"k136":0.0042,"k137":0.0042,"k138":0.0043,"k139":0.0043,"k140":0.0043,"k141":0.0044,"k142":0.0044,"k143":0.0044,"k144":0.0044,"k145":0.0045,"k146":0.0045,"k147":0.0045,"k148":0.0046,"k149":0.0046,"k150":0.0046,"k151":0.0047,"k152":0.0047,"k153":0.0047,"k154":0.0047,"k155":0.0048,"k156":0.0048,"k157":0.0048,"k158":0.0049,"k159":0.0049,"k160":0.0049,"k161":0.005,"k162":0.005,"k163":0.005,"k164":0.0051,"k165":0.0051,"k166":0.0051,"k167":0.0051,"k168":0.0052,"k169":0.0052,"k170":0.0052,"k171":0.0053,"k172":0.0053,"k173":0.0053,"k174":0.0054,"k175":0.0054,"k176":0.0054,"k177":0.0055,"k178":0.0055,"k179":0.0055,"k180":0.0055,"k181":0.0056,"k182":0.0056,"k183":0.0056,"k184":0.0057,"k185":0.0057,"k186":0.0057,"k187":0.0058,"k188":0.0058,"k189":0.0058,"k190":0.0059,"k191":0.0059,"k192":0.0059,"k193":0.0059,"k194":0.006,"k195":0.006,"k196":0.006,"k197":0.0061,"k198":0.0061,"k199":0.0061,"k200":0.0062,"k201":0.0062,"k202":0.0062,"k203":0.0063,"k204":0.0063,"k205":0.0063,"k206":0.0063,"k207":0.0064,"k208":0.0064,"k209":0.0064,"k210":0.0065,"k211":0.0065,"k212":0.0065,"k213":0.0066,"k214":0.0066,"k215":0.0066,"k216":0.0066,"k217":0.0067,"k218":0.0067,"k219":0.0067,"k220":0.0068,"k221":0.0068,"k222":0.0068,"k223":0.0069,"k224":0.0069,"k225":0.0069,"k226":0.007,"k227":0.007,"k228":0.007,"k229":0.007,"k230":0.0071,"k231":0.0071,"k232":0.0071,"k233":0.0072,"k234":0.0072,"k235":0.0072,"k236":0.0073,"k237":0.0073,"k238":0.0073,"k239":0.0074,"k240":0.0074,"k241":0.0074,"k242":0.0074,"k243":0.0075,"k244":0.0075,"k245":0.0075,"k246":0.0076,"k247":0.0076,"k248":0.0076,"k249":0.0077,"k250":0.0077,"k251":0.0077,"k252":0.0078,"k253":0.0078,"k254":0.0078}}},"output_tokens":3467},{"name":"score-1","request":{"state":"The task is mostly complete.","questions":{"rating":{"type":"score","instructions":"Completion level","criteria":["level 0"]}}},"record":{"state":"The task is mostly complete.","questions":[{"instr":"Completion level","options":["level 0"],"label":0}]},"meta":[{"id":"rating","type":"score","keys":["0"],"legend":{"0":"level 0"}}],"rows":[{"ids":[248060,760,3274,369,9712,4434,13,248061,32088,2119,248049,3187,220,15,248050,248062],"positions":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15],"readouts":[14,15]}],"input_tokens":16,"probabilities":[[1.0]],"answers":{"rating":{"type":"score","score":0.0,"legend":{"0":"level 0"},"probabilities":{"0":1.0},"confidence":1.0}},"output_tokens":48},{"name":"score-3","request":{"state":"The task is mostly complete.","questions":{"rating":{"type":"score","instructions":"Completion level","criteria":["level 0","level 1","level 2"]}}},"record":{"state":"The task is mostly complete.","questions":[{"instr":"Completion level","options":["level 0","level 1","level 2"],"label":0}]},"meta":[{"id":"rating","type":"score","keys":["0","1","2"],"legend":{"0":"level 0","1":"level 1","2":"level 2"}}],"rows":[{"ids":[248060,760,3274,369,9712,4434,13,248061,32088,2119,248049,3187,220,15,248050,248049,3187,220,16,248050,248049,3187,220,17,248050,248062],"positions":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25],"readouts":[14,19,24,25]}],"input_tokens":26,"probabilities":[[0.16666666666666666,0.3333333333333333,0.5]],"answers":{"rating":{"type":"score","score":1.3333,"legend":{"0":"level 0","1":"level 1","2":"level 2"},"probabilities":{"0":0.1667,"1":0.3333,"2":0.5},"confidence":0.0}},"output_tokens":89},{"name":"score-255","request":{"state":"The task is mostly complete.","questions":{"rating":{"type":"score","instructions":"Completion level","criteria":["level 0","level 1","level 2","level 3","level 4","level 5","level 6","level 7","level 8","level 9","level 10","level 11","level 12","level 13","level 14","level 15","level 16","level 17","level 18","level 19","level 20","level 21","level 22","level 23","level 24","level 25","level 26","level 27","level 28","level 29","level 30","level 31","level 32","level 33","level 34","level 35","level 36","level 37","level 38","level 39","level 40","level 41","level 42","level 43","level 44","level 45","level 46","level 47","level 48","level 49","level 50","level 51","level 52","level 53","level 54","level 55","level 56","level 57","level 58","level 59","level 60","level 61","level 62","level 63","level 64","level 65","level 66","level 67","level 68","level 69","level 70","level 71","level 72","level 73","level 74","level 75","level 76","level 77","level 78","level 79","level 80","level 81","level 82","level 83","level 84","level 85","level 86","level 87","level 88","level 89","level 90","level 91","level 92","level 93","level 94","level 95","level 96","level 97","level 98","level 99","level 100","level 101","level 102","level 103","level 104","level 105","level 106","level 107","level 108","level 109","level 110","level 111","level 112","level 113","level 114","level 115","level 116","level 117","level 118","level 119","level 120","level 121","level 122","level 123","level 124","level 125","level 126","level 127","level 128","level 129","level 130","level 131","level 132","level 133","level 134","level 135","level 136","level 137","level 138","level 139","level 140","level 141","level 142","level 143","level 144","level 145","level 146","level 147","level 148","level 149","level 150","level 151","level 152","level 153","level 154","level 155","level 156","level 157","level 158","level 159","level 160","level 161","level 162","level 163","level 164","level 165","level 166","level 167","level 168","level 169","level 170","level 171","level 172","level 173","level 174","level 175","level 176","level 177","level 178","level 179","level 180","level 181","level 182","level 183","level 184","level 185","level 186","level 187","level 188","level 189","level 190","level 191","level 192","level 193","level 194","level 195","level 196","level 197","level 198","level 199","level 200","level 201","level 202","level 203","level 204","level 205","level 206","level 207","level 208","level 209","level 210","level 211","level 212","level 213","level 214","level 215","level 216","level 217","level 218","level 219","level 220","level 221","level 222","level 223","level 224","level 225","level 226","level 227","level 228","level 229","level 230","level 231","level 232","level 233","level 234","level 235","level 236","level 237","level 238","level 239","level 240","level 241","level 242","level 243","level 244","level 245","level 246","level 247","level 248","level 249","level 250","level 251","level 252","level 253","level 254"]}}},"record":{"state":"The task is mostly complete.","questions":[{"instr":"Completion level","options":["level 0","level 1","level 2","level 3","level 4","level 5","level 6","level 7","level 8","level 9","level 10","level 11","level 12","level 13","level 14","level 15","level 16","level 17","level 18","level 19","level 20","level 21","level 22","level 23","level 24","level 25","level 26","level 27","level 28","level 29","level 30","level 31","level 32","level 33","level 34","level 35","level 36","level 37","level 38","level 39","level 40","level 41","level 42","level 43","level 44","level 45","level 46","level 47","level 48","level 49","level 50","level 51","level 52","level 53","level 54","level 55","level 56","level 57","level 58","level 59","level 60","level 61","level 62","level 63","level 64","level 65","level 66","level 67","level 68","level 69","level 70","level 71","level 72","level 73","level 74","level 75","level 76","level 77","level 78","level 79","level 80","level 81","level 82","level 83","level 84","level 85","level 86","level 87","level 88","level 89","level 90","level 91","level 92","level 93","level 94","level 95","level 96","level 97","level 98","level 99","level 100","level 101","level 102","level 103","level 104","level 105","level 106","level 107","level 108","level 109","level 110","level 111","level 112","level 113","level 114","level 115","level 116","level 117","level 118","level 119","level 120","level 121","level 122","level 123","level 124","level 125","level 126","level 127","level 128","level 129","level 130","level 131","level 132","level 133","level 134","level 135","level 136","level 137","level 138","level 139","level 140","level 141","level 142","level 143","level 144","level 145","level 146","level 147","level 148","level 149","level 150","level 151","level 152","level 153","level 154","level 155","level 156","level 157","level 158","level 159","level 160","level 161","level 162","level 163","level 164","level 165","level 166","level 167","level 168","level 169","level 170","level 171","level 172","level 173","level 174","level 175","level 176","level 177","level 178","level 179","level 180","level 181","level 182","level 183","level 184","level 185","level 186","level 187","level 188","level 189","level 190","level 191","level 192","level 193","level 194","level 195","level 196","level 197","level 198","level 199","level 200","level 201","level 202","level 203","level 204","level 205","level 206","level 207","level 208","level 209","level 210","level 211","level 212","level 213","level 214","level 215","level 216","level 217","level 218","level 219","level 220","level 221","level 222","level 223","level 224","level 225","level 226","level 227","level 228","level 229","level 230","level 231","level 232","level 233","level 234","level 235","level 236","level 237","level 238","level 239","level 240","level 241","level 242","level 243","level 244","level 245","level 246","level 247","level 248","level 249","level 250","level 251","level 252","level 253","level 254"],"label":0}]},"meta":[{"id":"rating","type":"score","keys":["0","1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63","64","65","66","67","68","69","70","71","72","73","74","75","76","77","78","79","80","81","82","83","84","85","86","87","88","89","90","91","92","93","94","95","96","97","98","99","100","101","102","103","104","105","106","107","108","109","110","111","112","113","114","115","116","117","118","119","120","121","122","123","124","125","126","127","128","129","130","131","132","133","134","135","136","137","138","139","140","141","142","143","144","145","146","147","148","149","150","151","152","153","154","155","156","157","158","159","160","161","162","163","164","165","166","167","168","169","170","171","172","173","174","175","176","177","178","179","180","181","182","183","184","185","186","187","188","189","190","191","192","193","194","195","196","197","198","199","200","201","202","203","204","205","206","207","208","209","210","211","212","213","214","215","216","217","218","219","220","221","222","223","224","225","226","227","228","229","230","231","232","233","234","235","236","237","238","239","240","241","242","243","244","245","246","247","248","249","250","251","252","253","254"],"legend":{"0":"level 0","1":"level 1","2":"level 2","3":"level 3","4":"level 4","5":"level 5","6":"level 6","7":"level 7","8":"level 8","9":"level 9","10":"level 10","11":"level 11","12":"level 12","13":"level 13","14":"level 14","15":"level 15","16":"level 16","17":"level 17","18":"level 18","19":"level 19","20":"level 20","21":"level 21","22":"level 22","23":"level 23","24":"level 24","25":"level 25","26":"level 26","27":"level 27","28":"level 28","29":"level 29","30":"level 30","31":"level 31","32":"level 32","33":"level 33","34":"level 34","35":"level 35","36":"level 36","37":"level 37","38":"level 38","39":"level 39","40":"level 40","41":"level 41","42":"level 42","43":"level 43","44":"level 44","45":"level 45","46":"level 46","47":"level 47","48":"level 48","49":"level 49","50":"level 50","51":"level 51","52":"level 52","53":"level 53","54":"level 54","55":"level 55","56":"level 56","57":"level 57","58":"level 58","59":"level 59","60":"level 60","61":"level 61","62":"level 62","63":"level 63","64":"level 64","65":"level 65","66":"level 66","67":"level 67","68":"level 68","69":"level 69","70":"level 70","71":"level 71","72":"level 72","73":"level 73","74":"level 74","75":"level 75","76":"level 76","77":"level 77","78":"level 78","79":"level 79","80":"level 80","81":"level 81","82":"level 82","83":"level 83","84":"level 84","85":"level 85","86":"level 86","87":"level 87","88":"level 88","89":"level 89","90":"level 90","91":"level 91","92":"level 92","93":"level 93","94":"level 94","95":"level 95","96":"level 96","97":"level 97","98":"level 98","99":"level 99","100":"level 100","101":"level 101","102":"level 102","103":"level 103","104":"level 104","105":"level 105","106":"level 106","107":"level 107","108":"level 108","109":"level 109","110":"level 110","111":"level 111","112":"level 112","113":"level 113","114":"level 114","115":"level 115","116":"level 116","117":"level 117","118":"level 118","119":"level 119","120":"level 120","121":"level 121","122":"level 122","123":"level 123","124":"level 124","125":"level 125","126":"level 126","127":"level 127","128":"level 128","129":"level 129","130":"level 130","131":"level 131","132":"level 132","133":"level 133","134":"level 134","135":"level 135","136":"level 136","137":"level 137","138":"level 138","139":"level 139","140":"level 140","141":"level 141","142":"level 142","143":"level 143","144":"level 144","145":"level 145","146":"level 146","147":"level 147","148":"level 148","149":"level 149","150":"level 150","151":"level 151","152":"level 152","153":"level 153","154":"level 154","155":"level 155","156":"level 156","157":"level 157","158":"level 158","159":"level 159","160":"level 160","161":"level 161","162":"level 162","163":"level 163","164":"level 164","165":"level 165","166":"level 166","167":"level 167","168":"level 168","169":"level 169","170":"level 170","171":"level 171","172":"level 172","173":"level 173","174":"level 174","175":"level 175","176":"level 176","177":"level 177","178":"level 178","179":"level 179","180":"level 180","181":"level 181","182":"level 182","183":"level 183","184":"level 184","185":"level 185","186":"level 186","187":"level 187","188":"level 188","189":"level 189","190":"level 190","191":"level 191","192":"level 192","193":"level 193","194":"level 194","195":"level 195","196":"level 196","197":"level 197","198":"level 198","199":"level 199","200":"level 200","201":"level 201","202":"level 202","203":"level 203","204":"level 204","205":"level 205","206":"level 206","207":"level 207","208":"level 208","209":"level 209","210":"level 210","211":"level 211","212":"level 212","213":"level 213","214":"level 214","215":"level 215","216":"level 216","217":"level 217","218":"level 218","219":"level 219","220":"level 220","221":"level 221","222":"level 222","223":"level 223","224":"level 224","225":"level 225","226":"level 226","227":"level 227","228":"level 228","229":"level 229","230":"level 230","231":"level 231","232":"level 232","233":"level 233","234":"level 234","235":"level 235","236":"level 236","237":"level 237","238":"level 238","239":"level 239","240":"level 240","241":"level 241","242":"level 242","243":"level 243","244":"level 244","245":"level 245","246":"level 246","247":"level 247","248":"level 248","249":"level 249","250":"level 250","251":"level 251","252":"level 252","253":"level 253","254":"level 254"}}],"rows":[{"ids":[248060,760,3274,369,9712,4434,13,248061,32088,2119,248049,3187,220,15,248050,248049,3187,220,16,248050,248049,3187,220,17,248050,248049,3187,220,18,248050,248049,3187,220,19,248050,248049,3187,220,20,248050,248049,3187,220,21,248050,248049,3187,220,22,248050,248049,3187,220,23,248050,248049,3187,220,24,248050,248049,3187,220,16,15,248050,248049,3187,220,16,16,248050,248049,3187,220,16,17,248050,248049,3187,220,16,18,248050,248049,3187,220,16,19,248050,248049,3187,220,16,20,248050,248049,3187,220,16,21,248050,248049,3187,220,16,22,248050,248049,3187,220,16,23,248050,248049,3187,220,16,24,248050,248049,3187,220,17,15,248050,248049,3187,220,17,16,248050,248049,3187,220,17,17,248050,248049,3187,220,17,18,248050,248049,3187,220,17,19,248050,248049,3187,220,17,20,248050,248049,3187,220,17,21,248050,248049,3187,220,17,22,248050,248049,3187,220,17,23,248050,248049,3187,220,17,24,248050,248049,3187,220,18,15,248050,248049,3187,220,18,16,248050,248049,3187,220,18,17,248050,248049,3187,220,18,18,248050,248049,3187,220,18,19,248050,248049,3187,220,18,20,248050,248049,3187,220,18,21,248050,248049,3187,220,18,22,248050,248049,3187,220,18,23,248050,248049,3187,220,18,24,248050,248049,3187,220,19,15,248050,248049,3187,220,19,16,248050,248049,3187,220,19,17,248050,248049,3187,220,19,18,248050,248049,3187,220,19,19,248050,248049,3187,220,19,20,248050,248049,3187,220,19,21,248050,248049,3187,220,19,22,248050,248049,3187,220,19,23,248050,248049,3187,220,19,24,248050,248049,3187,220,20,15,248050,248049,3187,220,20,16,248050,248049,3187,220,20,17,248050,248049,3187,220,20,18,248050,248049,3187,220,20,19,248050,248049,3187,220,20,20,248050,248049,3187,220,20,21,248050,248049,3187,220,20,22,248050,248049,3187,220,20,23,248050,248049,3187,220,20,24,248050,248049,3187,220,21,15,248050,248049,3187,220,21,16,248050,248049,3187,220,21,17,248050,248049,3187,220,21,18,248050,248049,3187,220,21,19,248050,248049,3187,220,21,20,248050,248049,3187,220,21,21,248050,248049,3187,220,21,22,248050,248049,3187,220,21,23,248050,248049,3187,220,21,24,248050,248049,3187,220,22,15,248050,248049,3187,220,22,16,248050,248049,3187,220,22,17,248050,248049,3187,220,22,18,248050,248049,3187,220,22,19,248050,248049,3187,220,22,20,248050,248049,3187,220,22,21,248050,248049,3187,220,22,22,248050,248049,3187,220,22,23,248050,248049,3187,220,22,24,248050,248049,3187,220,23,15,248050,248049,3187,220,23,16,248050,248049,3187,220,23,17,248050,248049,3187,220,23,18,248050,248049,3187,220,23,19,248050,248049,3187,220,23,20,248050,248049,3187,220,23,21,248050,248049,3187,220,23,22,248050,248049,3187,220,23,23,248050,248049,3187,220,23,24,248050,248049,3187,220,24,15,248050,248049,3187,220,24,16,248050,248049,3187,220,24,17,248050,248049,3187,220,24,18,248050,248049,3187,220,24,19,248050,248049,3187,220,24,20,248050,248049,3187,220,24,21,248050,248049,3187,220,24,22,248050,248049,3187,220,24,23,248050,248049,3187,220,24,24,248050,248049,3187,220,16,15,15,248050,248049,3187,220,16,15,16,248050,248049,3187,220,16,15,17,248050,248049,3187,220,16,15,18,248050,248049,3187,220,16,15,19,248050,248049,3187,220,16,15,20,248050,248049,3187,220,16,15,21,248050,248049,3187,220,16,15,22,248050,248049,3187,220,16,15,23,248050,248049,3187,220,16,15,24,248050,248049,3187,220,16,16,15,248050,248049,3187,220,16,16,16,248050,248049,3187,220,16,16,17,248050,248049,3187,220,16,16,18,248050,248049,3187,220,16,16,19,248050,248049,3187,220,16,16,20,248050,248049,3187,220,16,16,21,248050,248049,3187,220,16,16,22,248050,248049,3187,220,16,16,23,248050,248049,3187,220,16,16,24,248050,248049,3187,220,16,17,15,248050,248049,3187,220,16,17,16,248050,248049,3187,220,16,17,17,248050,248049,3187,220,16,17,18,248050,248049,3187,220,16,17,19,248050,248049,3187,220,16,17,20,248050,248049,3187,220,16,17,21,248050,248049,3187,220,16,17,22,248050,248049,3187,220,16,17,23,248050,248049,3187,220,16,17,24,248050,248049,3187,220,16,18,15,248050,248049,3187,220,16,18,16,248050,248049,3187,220,16,18,17,248050,248049,3187,220,16,18,18,248050,248049,3187,220,16,18,19,248050,248049,3187,220,16,18,20,248050,248049,3187,220,16,18,21,248050,248049,3187,220,16,18,22,248050,248049,3187,220,16,18,23,248050,248049,3187,220,16,18,24,248050,248049,3187,220,16,19,15,248050,248049,3187,220,16,19,16,248050,248049,3187,220,16,19,17,248050,248049,3187,220,16,19,18,248050,248049,3187,220,16,19,19,248050,248049,3187,220,16,19,20,248050,248049,3187,220,16,19,21,248050,248049,3187,220,16,19,22,248050,248049,3187,220,16,19,23,248050,248049,3187,220,16,19,24,248050,248049,3187,220,16,20,15,248050,248049,3187,220,16,20,16,248050,248049,3187,220,16,20,17,248050,248049,3187,220,16,20,18,248050,248049,3187,220,16,20,19,248050,248049,3187,220,16,20,20,248050,248049,3187,220,16,20,21,248050,248049,3187,220,16,20,22,248050,248049,3187,220,16,20,23,248050,248049,3187,220,16,20,24,248050,248049,3187,220,16,21,15,248050,248049,3187,220,16,21,16,248050,248049,3187,220,16,21,17,248050,248049,3187,220,16,21,18,248050,248049,3187,220,16,21,19,248050,248049,3187,220,16,21,20,248050,248049,3187,220,16,21,21,248050,248049,3187,220,16,21,22,248050,248049,3187,220,16,21,23,248050,248049,3187,220,16,21,24,248050,248049,3187,220,16,22,15,248050,248049,3187,220,16,22,16,248050,248049,3187,220,16,22,17,248050,248049,3187,220,16,22,18,248050,248049,3187,220,16,22,19,248050,248049,3187,220,16,22,20,248050,248049,3187,220,16,22,21,248050,248049,3187,220,16,22,22,248050,248049,3187,220,16,22,23,248050,248049,3187,220,16,22,24,248050,248049,3187,220,16,23,15,248050,248049,3187,220,16,23,16,248050,248049,3187,220,16,23,17,248050,248049,3187,220,16,23,18,248050,248049,3187,220,16,23,19,248050,248049,3187,220,16,23,20,248050,248049,3187,220,16,23,21,248050,248049,3187,220,16,23,22,248050,248049,3187,220,16,23,23,248050,248049,3187,220,16,23,24,248050,248049,3187,220,16,24,15,248050,248049,3187,220,16,24,16,248050,248049,3187,220,16,24,17,248050,248049,3187,220,16,24,18,248050,248049,3187,220,16,24,19,248050,248049,3187,220,16,24,20,248050,248049,3187,220,16,24,21,248050,248049,3187,220,16,24,22,248050,248049,3187,220,16,24,23,248050,248049,3187,220,16,24,24,248050,248049,3187,220,17,15,15,248050,248049,3187,220,17,15,16,248050,248049,3187,220,17,15,17,248050,248049,3187,220,17,15,18,248050,248049,3187,220,17,15,19,248050,248049,3187,220,17,15,20,248050,248049,3187,220,17,15,21,248050,248049,3187,220,17,15,22,248050,248049,3187,220,17,15,23,248050,248049,3187,220,17,15,24,248050,248049,3187,220,17,16,15,248050,248049,3187,220,17,16,16,248050,248049,3187,220,17,16,17,248050,248049,3187,220,17,16,18,248050,248049,3187,220,17,16,19,248050,248049,3187,220,17,16,20,248050,248049,3187,220,17,16,21,248050,248049,3187,220,17,16,22,248050,248049,3187,220,17,16,23,248050,248049,3187,220,17,16,24,248050,248049,3187,220,17,17,15,248050,248049,3187,220,17,17,16,248050,248049,3187,220,17,17,17,248050,248049,3187,220,17,17,18,248050,248049,3187,220,17,17,19,248050,248049,3187,220,17,17,20,248050,248049,3187,220,17,17,21,248050,248049,3187,220,17,17,22,248050,248049,3187,220,17,17,23,248050,248049,3187,220,17,17,24,248050,248049,3187,220,17,18,15,248050,248049,3187,220,17,18,16,248050,248049,3187,220,17,18,17,248050,248049,3187,220,17,18,18,248050,248049,3187,220,17,18,19,248050,248049,3187,220,17,18,20,248050,248049,3187,220,17,18,21,248050,248049,3187,220,17,18,22,248050,248049,3187,220,17,18,23,248050,248049,3187,220,17,18,24,248050,248049,3187,220,17,19,15,248050,248049,3187,220,17,19,16,248050,248049,3187,220,17,19,17,248050,248049,3187,220,17,19,18,248050,248049,3187,220,17,19,19,248050,248049,3187,220,17,19,20,248050,248049,3187,220,17,19,21,248050,248049,3187,220,17,19,22,248050,248049,3187,220,17,19,23,248050,248049,3187,220,17,19,24,248050,248049,3187,220,17,20,15,248050,248049,3187,220,17,20,16,248050,248049,3187,220,17,20,17,248050,248049,3187,220,17,20,18,248050,248049,3187,220,17,20,19,248050,248062],"positions":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255,256,257,258,259,260,261,262,263,264,265,266,267,268,269,270,271,272,273,274,275,276,277,278,279,280,281,282,283,284,285,286,287,288,289,290,291,292,293,294,295,296,297,298,299,300,301,302,303,304,305,306,307,308,309,310,311,312,313,314,315,316,317,318,319,320,321,322,323,324,325,326,327,328,329,330,331,332,333,334,335,336,337,338,339,340,341,342,343,344,345,346,347,348,349,350,351,352,353,354,355,356,357,358,359,360,361,362,363,364,365,366,367,368,369,370,371,372,373,374,375,376,377,378,379,380,381,382,383,384,385,386,387,388,389,390,391,392,393,394,395,396,397,398,399,400,401,402,403,404,405,406,407,408,409,410,411,412,413,414,415,416,417,418,419,420,421,422,423,424,425,426,427,428,429,430,431,432,433,434,435,436,437,438,439,440,441,442,443,444,445,446,447,448,449,450,451,452,453,454,455,456,457,458,459,460,461,462,463,464,465,466,467,468,469,470,471,472,473,474,475,476,477,478,479,480,481,482,483,484,485,486,487,488,489,490,491,492,493,494,495,496,497,498,499,500,501,502,503,504,505,506,507,508,509,510,511,512,513,514,515,516,517,518,519,520,521,522,523,524,525,526,527,528,529,530,531,532,533,534,535,536,537,538,539,540,541,542,543,544,545,546,547,548,549,550,551,552,553,554,555,556,557,558,559,560,561,562,563,564,565,566,567,568,569,570,571,572,573,574,575,576,577,578,579,580,581,582,583,584,585,586,587,588,589,590,591,592,593,594,595,596,597,598,599,600,601,602,603,604,605,606,607,608,609,610,611,612,613,614,615,616,617,618,619,620,621,622,623,624,625,626,627,628,629,630,631,632,633,634,635,636,637,638,639,640,641,642,643,644,645,646,647,648,649,650,651,652,653,654,655,656,657,658,659,660,661,662,663,664,665,666,667,668,669,670,671,672,673,674,675,676,677,678,679,680,681,682,683,684,685,686,687,688,689,690,691,692,693,694,695,696,697,698,699,700,701,702,703,704,705,706,707,708,709,710,711,712,713,714,715,716,717,718,719,720,721,722,723,724,725,726,727,728,729,730,731,732,733,734,735,736,737,738,739,740,741,742,743,744,745,746,747,748,749,750,751,752,753,754,755,756,757,758,759,760,761,762,763,764,765,766,767,768,769,770,771,772,773,774,775,776,777,778,779,780,781,782,783,784,785,786,787,788,789,790,791,792,793,794,795,796,797,798,799,800,801,802,803,804,805,806,807,808,809,810,811,812,813,814,815,816,817,818,819,820,821,822,823,824,825,826,827,828,829,830,831,832,833,834,835,836,837,838,839,840,841,842,843,844,845,846,847,848,849,850,851,852,853,854,855,856,857,858,859,860,861,862,863,864,865,866,867,868,869,870,871,872,873,874,875,876,877,878,879,880,881,882,883,884,885,886,887,888,889,890,891,892,893,894,895,896,897,898,899,900,901,902,903,904,905,906,907,908,909,910,911,912,913,914,915,916,917,918,919,920,921,922,923,924,925,926,927,928,929,930,931,932,933,934,935,936,937,938,939,940,941,942,943,944,945,946,947,948,949,950,951,952,953,954,955,956,957,958,959,960,961,962,963,964,965,966,967,968,969,970,971,972,973,974,975,976,977,978,979,980,981,982,983,984,985,986,987,988,989,990,991,992,993,994,995,996,997,998,999,1000,1001,1002,1003,1004,1005,1006,1007,1008,1009,1010,1011,1012,1013,1014,1015,1016,1017,1018,1019,1020,1021,1022,1023,1024,1025,1026,1027,1028,1029,1030,1031,1032,1033,1034,1035,1036,1037,1038,1039,1040,1041,1042,1043,1044,1045,1046,1047,1048,1049,1050,1051,1052,1053,1054,1055,1056,1057,1058,1059,1060,1061,1062,1063,1064,1065,1066,1067,1068,1069,1070,1071,1072,1073,1074,1075,1076,1077,1078,1079,1080,1081,1082,1083,1084,1085,1086,1087,1088,1089,1090,1091,1092,1093,1094,1095,1096,1097,1098,1099,1100,1101,1102,1103,1104,1105,1106,1107,1108,1109,1110,1111,1112,1113,1114,1115,1116,1117,1118,1119,1120,1121,1122,1123,1124,1125,1126,1127,1128,1129,1130,1131,1132,1133,1134,1135,1136,1137,1138,1139,1140,1141,1142,1143,1144,1145,1146,1147,1148,1149,1150,1151,1152,1153,1154,1155,1156,1157,1158,1159,1160,1161,1162,1163,1164,1165,1166,1167,1168,1169,1170,1171,1172,1173,1174,1175,1176,1177,1178,1179,1180,1181,1182,1183,1184,1185,1186,1187,1188,1189,1190,1191,1192,1193,1194,1195,1196,1197,1198,1199,1200,1201,1202,1203,1204,1205,1206,1207,1208,1209,1210,1211,1212,1213,1214,1215,1216,1217,1218,1219,1220,1221,1222,1223,1224,1225,1226,1227,1228,1229,1230,1231,1232,1233,1234,1235,1236,1237,1238,1239,1240,1241,1242,1243,1244,1245,1246,1247,1248,1249,1250,1251,1252,1253,1254,1255,1256,1257,1258,1259,1260,1261,1262,1263,1264,1265,1266,1267,1268,1269,1270,1271,1272,1273,1274,1275,1276,1277,1278,1279,1280,1281,1282,1283,1284,1285,1286,1287,1288,1289,1290,1291,1292,1293,1294,1295,1296,1297,1298,1299,1300,1301,1302,1303,1304,1305,1306,1307,1308,1309,1310,1311,1312,1313,1314,1315,1316,1317,1318,1319,1320,1321,1322,1323,1324,1325,1326,1327,1328,1329,1330,1331,1332,1333,1334,1335,1336,1337,1338,1339,1340,1341,1342,1343,1344,1345,1346,1347,1348,1349,1350,1351,1352,1353,1354,1355,1356,1357,1358,1359,1360,1361,1362,1363,1364,1365,1366,1367,1368,1369,1370,1371,1372,1373,1374,1375,1376,1377,1378,1379,1380,1381,1382,1383,1384,1385,1386,1387,1388,1389,1390,1391,1392,1393,1394,1395,1396,1397,1398,1399,1400,1401,1402,1403,1404,1405,1406,1407,1408,1409,1410,1411,1412,1413,1414,1415,1416,1417,1418,1419,1420,1421,1422,1423,1424,1425,1426,1427,1428,1429,1430,1431,1432,1433,1434,1435,1436,1437,1438,1439,1440,1441,1442,1443,1444,1445,1446,1447,1448,1449,1450,1451,1452,1453,1454,1455,1456,1457,1458,1459,1460,1461,1462,1463,1464,1465,1466,1467,1468,1469,1470,1471,1472,1473,1474,1475,1476,1477,1478,1479,1480,1481,1482,1483,1484,1485,1486,1487,1488,1489,1490,1491,1492,1493,1494,1495,1496,1497,1498,1499,1500,1501,1502,1503,1504,1505,1506,1507,1508,1509,1510,1511,1512,1513,1514,1515,1516,1517,1518,1519,1520,1521,1522,1523,1524,1525,1526,1527,1528,1529,1530,1531,1532,1533,1534,1535,1536,1537,1538,1539,1540,1541,1542,1543,1544,1545,1546,1547,1548,1549,1550,1551,1552,1553,1554,1555,1556,1557,1558,1559,1560,1561,1562,1563,1564,1565,1566,1567,1568,1569,1570,1571,1572,1573,1574,1575,1576,1577,1578,1579,1580,1581,1582,1583,1584,1585,1586,1587,1588,1589,1590,1591,1592,1593,1594,1595,1596,1597,1598,1599,1600,1601,1602,1603,1604,1605,1606,1607,1608,1609,1610,1611,1612,1613,1614,1615,1616,1617,1618,1619,1620,1621,1622,1623,1624,1625,1626,1627,1628,1629,1630,1631,1632,1633,1634,1635,1636,1637,1638,1639,1640,1641,1642,1643,1644,1645,1646,1647,1648,1649,1650,1651,1652,1653,1654,1655,1656,1657,1658,1659,1660,1661,1662,1663,1664,1665,1666,1667,1668,1669,1670,1671,1672,1673,1674,1675,1676,1677,1678,1679,1680,1681,1682,1683,1684,1685],"readouts":[14,19,24,29,34,39,44,49,54,59,65,71,77,83,89,95,101,107,113,119,125,131,137,143,149,155,161,167,173,179,185,191,197,203,209,215,221,227,233,239,245,251,257,263,269,275,281,287,293,299,305,311,317,323,329,335,341,347,353,359,365,371,377,383,389,395,401,407,413,419,425,431,437,443,449,455,461,467,473,479,485,491,497,503,509,515,521,527,533,539,545,551,557,563,569,575,581,587,593,599,606,613,620,627,634,641,648,655,662,669,676,683,690,697,704,711,718,725,732,739,746,753,760,767,774,781,788,795,802,809,816,823,830,837,844,851,858,865,872,879,886,893,900,907,914,921,928,935,942,949,956,963,970,977,984,991,998,1005,1012,1019,1026,1033,1040,1047,1054,1061,1068,1075,1082,1089,1096,1103,1110,1117,1124,1131,1138,1145,1152,1159,1166,1173,1180,1187,1194,1201,1208,1215,1222,1229,1236,1243,1250,1257,1264,1271,1278,1285,1292,1299,1306,1313,1320,1327,1334,1341,1348,1355,1362,1369,1376,1383,1390,1397,1404,1411,1418,1425,1432,1439,1446,1453,1460,1467,1474,1481,1488,1495,1502,1509,1516,1523,1530,1537,1544,1551,1558,1565,1572,1579,1586,1593,1600,1607,1614,1621,1628,1635,1642,1649,1656,1663,1670,1677,1684,1685]}],"input_tokens":1686,"probabilities":[[3.0637254901960784e-05,6.127450980392157e-05,9.191176470588235e-05,0.00012254901960784314,0.00015318627450980392,0.0001838235294117647,0.0002144607843137255,0.00024509803921568627,0.00027573529411764705,0.00030637254901960784,0.0003370098039215686,0.0003676470588235294,0.0003982843137254902,0.000428921568627451,0.00045955882352941176,0.0004901960784313725,0.0005208333333333333,0.0005514705882352941,0.0005821078431372549,0.0006127450980392157,0.0006433823529411765,0.0006740196078431372,0.000704656862745098,0.0007352941176470588,0.0007659313725490196,0.0007965686274509804,0.0008272058823529412,0.000857843137254902,0.0008884803921568627,0.0009191176470588235,0.0009497549019607843,0.000980392156862745,0.001011029411764706,0.0010416666666666667,0.0010723039215686275,0.0011029411764705882,0.0011335784313725491,0.0011642156862745098,0.0011948529411764707,0.0012254901960784314,0.0012561274509803922,0.001286764705882353,0.0013174019607843138,0.0013480392156862745,0.0013786764705882354,0.001409313725490196,0.001439950980392157,0.0014705882352941176,0.0015012254901960785,0.0015318627450980392,0.0015625,0.0015931372549019608,0.0016237745098039217,0.0016544117647058823,0.0016850490196078432,0.001715686274509804,0.0017463235294117648,0.0017769607843137255,0.0018075980392156864,0.001838235294117647,0.001868872549019608,0.0018995098039215686,0.0019301470588235295,0.00196078431372549,0.001991421568627451,0.002022058823529412,0.0020526960784313726,0.0020833333333333333,0.002113970588235294,0.002144607843137255,0.0021752450980392158,0.0022058823529411764,0.002236519607843137,0.0022671568627450982,0.002297794117647059,0.0023284313725490196,0.0023590686274509803,0.0023897058823529414,0.002420343137254902,0.0024509803921568627,0.0024816176470588234,0.0025122549019607845,0.002542892156862745,0.002573529411764706,0.0026041666666666665,0.0026348039215686276,0.0026654411764705883,0.002696078431372549,0.0027267156862745097,0.0027573529411764708,0.0027879901960784314,0.002818627450980392,0.002849264705882353,0.002879901960784314,0.0029105392156862746,0.0029411764705882353,0.002971813725490196,0.003002450980392157,0.0030330882352941177,0.0030637254901960784,0.003094362745098039,0.003125,0.003155637254901961,0.0031862745098039215,0.003216911764705882,0.0032475490196078433,0.003278186274509804,0.0033088235294117647,0.0033394607843137253,0.0033700980392156864,0.003400735294117647,0.003431372549019608,0.0034620098039215685,0.0034926470588235296,0.0035232843137254903,0.003553921568627451,0.0035845588235294116,0.0036151960784313727,0.0036458333333333334,0.003676470588235294,0.0037071078431372547,0.003737745098039216,0.0037683823529411765,0.003799019607843137,0.003829656862745098,0.003860294117647059,0.0038909313725490197,0.00392156862745098,0.003952205882352941,0.003982843137254902,0.004013480392156862,0.004044117647058824,0.004074754901960785,0.004105392156862745,0.004136029411764706,0.004166666666666667,0.004197303921568627,0.004227941176470588,0.004258578431372549,0.00428921568627451,0.004319852941176471,0.0043504901960784315,0.004381127450980392,0.004411764705882353,0.0044424019607843136,0.004473039215686274,0.004503676470588235,0.0045343137254901964,0.004564950980392157,0.004595588235294118,0.0046262254901960785,0.004656862745098039,0.0046875,0.0047181372549019605,0.004748774509803921,0.004779411764705883,0.004810049019607843,0.004840686274509804,0.004871323529411765,0.004901960784313725,0.004932598039215686,0.004963235294117647,0.0049938725490196074,0.005024509803921569,0.00505514705882353,0.00508578431372549,0.005116421568627451,0.005147058823529412,0.005177696078431372,0.005208333333333333,0.005238970588235294,0.005269607843137255,0.005300245098039216,0.005330882352941177,0.005361519607843137,0.005392156862745098,0.005422794117647059,0.005453431372549019,0.00548406862745098,0.0055147058823529415,0.005545343137254902,0.005575980392156863,0.0056066176470588236,0.005637254901960784,0.005667892156862745,0.005698529411764706,0.005729166666666666,0.005759803921568628,0.0057904411764705885,0.005821078431372549,0.00585171568627451,0.0058823529411764705,0.005912990196078431,0.005943627450980392,0.0059742647058823525,0.006004901960784314,0.006035539215686275,0.006066176470588235,0.006096813725490196,0.006127450980392157,0.0061580882352941175,0.006188725490196078,0.006219362745098039,0.00625,0.006280637254901961,0.006311274509803922,0.006341911764705882,0.006372549019607843,0.006403186274509804,0.006433823529411764,0.006464460784313725,0.006495098039215687,0.006525735294117647,0.006556372549019608,0.006587009803921569,0.006617647058823529,0.00664828431372549,0.006678921568627451,0.006709558823529411,0.006740196078431373,0.0067708333333333336,0.006801470588235294,0.006832107843137255,0.006862745098039216,0.006893382352941176,0.006924019607843137,0.006954656862745098,0.006985294117647059,0.00701593137254902,0.0070465686274509805,0.007077205882352941,0.007107843137254902,0.0071384803921568625,0.007169117647058823,0.007199754901960784,0.007230392156862745,0.007261029411764706,0.007291666666666667,0.0073223039215686275,0.007352941176470588,0.007383578431372549,0.0074142156862745095,0.00744485294117647,0.007475490196078432,0.007506127450980392,0.007536764705882353,0.007567401960784314,0.007598039215686274,0.007628676470588235,0.007659313725490196,0.007689950980392156,0.007720588235294118,0.007751225490196079,0.007781862745098039,0.0078125]],"answers":{"rating":{"type":"score","score":169.3333,"legend":{"0":"level 0","1":"level 1","2":"level 2","3":"level 3","4":"level 4","5":"level 5","6":"level 6","7":"level 7","8":"level 8","9":"level 9","10":"level 10","11":"level 11","12":"level 12","13":"level 13","14":"level 14","15":"level 15","16":"level 16","17":"level 17","18":"level 18","19":"level 19","20":"level 20","21":"level 21","22":"level 22","23":"level 23","24":"level 24","25":"level 25","26":"level 26","27":"level 27","28":"level 28","29":"level 29","30":"level 30","31":"level 31","32":"level 32","33":"level 33","34":"level 34","35":"level 35","36":"level 36","37":"level 37","38":"level 38","39":"level 39","40":"level 40","41":"level 41","42":"level 42","43":"level 43","44":"level 44","45":"level 45","46":"level 46","47":"level 47","48":"level 48","49":"level 49","50":"level 50","51":"level 51","52":"level 52","53":"level 53","54":"level 54","55":"level 55","56":"level 56","57":"level 57","58":"level 58","59":"level 59","60":"level 60","61":"level 61","62":"level 62","63":"level 63","64":"level 64","65":"level 65","66":"level 66","67":"level 67","68":"level 68","69":"level 69","70":"level 70","71":"level 71","72":"level 72","73":"level 73","74":"level 74","75":"level 75","76":"level 76","77":"level 77","78":"level 78","79":"level 79","80":"level 80","81":"level 81","82":"level 82","83":"level 83","84":"level 84","85":"level 85","86":"level 86","87":"level 87","88":"level 88","89":"level 89","90":"level 90","91":"level 91","92":"level 92","93":"level 93","94":"level 94","95":"level 95","96":"level 96","97":"level 97","98":"level 98","99":"level 99","100":"level 100","101":"level 101","102":"level 102","103":"level 103","104":"level 104","105":"level 105","106":"level 106","107":"level 107","108":"level 108","109":"level 109","110":"level 110","111":"level 111","112":"level 112","113":"level 113","114":"level 114","115":"level 115","116":"level 116","117":"level 117","118":"level 118","119":"level 119","120":"level 120","121":"level 121","122":"level 122","123":"level 123","124":"level 124","125":"level 125","126":"level 126","127":"level 127","128":"level 128","129":"level 129","130":"level 130","131":"level 131","132":"level 132","133":"level 133","134":"level 134","135":"level 135","136":"level 136","137":"level 137","138":"level 138","139":"level 139","140":"level 140","141":"level 141","142":"level 142","143":"level 143","144":"level 144","145":"level 145","146":"level 146","147":"level 147","148":"level 148","149":"level 149","150":"level 150","151":"level 151","152":"level 152","153":"level 153","154":"level 154","155":"level 155","156":"level 156","157":"level 157","158":"level 158","159":"level 159","160":"level 160","161":"level 161","162":"level 162","163":"level 163","164":"level 164","165":"level 165","166":"level 166","167":"level 167","168":"level 168","169":"level 169","170":"level 170","171":"level 171","172":"level 172","173":"level 173","174":"level 174","175":"level 175","176":"level 176","177":"level 177","178":"level 178","179":"level 179","180":"level 180","181":"level 181","182":"level 182","183":"level 183","184":"level 184","185":"level 185","186":"level 186","187":"level 187","188":"level 188","189":"level 189","190":"level 190","191":"level 191","192":"level 192","193":"level 193","194":"level 194","195":"level 195","196":"level 196","197":"level 197","198":"level 198","199":"level 199","200":"level 200","201":"level 201","202":"level 202","203":"level 203","204":"level 204","205":"level 205","206":"level 206","207":"level 207","208":"level 208","209":"level 209","210":"level 210","211":"level 211","212":"level 212","213":"level 213","214":"level 214","215":"level 215","216":"level 216","217":"level 217","218":"level 218","219":"level 219","220":"level 220","221":"level 221","222":"level 222","223":"level 223","224":"level 224","225":"level 225","226":"level 226","227":"level 227","228":"level 228","229":"level 229","230":"level 230","231":"level 231","232":"level 232","233":"level 233","234":"level 234","235":"level 235","236":"level 236","237":"level 237","238":"level 238","239":"level 239","240":"level 240","241":"level 241","242":"level 242","243":"level 243","244":"level 244","245":"level 245","246":"level 246","247":"level 247","248":"level 248","249":"level 249","250":"level 250","251":"level 251","252":"level 252","253":"level 253","254":"level 254"},"probabilities":{"0":0.0,"1":0.0001,"2":0.0001,"3":0.0001,"4":0.0002,"5":0.0002,"6":0.0002,"7":0.0002,"8":0.0003,"9":0.0003,"10":0.0003,"11":0.0004,"12":0.0004,"13":0.0004,"14":0.0005,"15":0.0005,"16":0.0005,"17":0.0006,"18":0.0006,"19":0.0006,"20":0.0006,"21":0.0007,"22":0.0007,"23":0.0007,"24":0.0008,"25":0.0008,"26":0.0008,"27":0.0009,"28":0.0009,"29":0.0009,"30":0.0009,"31":0.001,"32":0.001,"33":0.001,"34":0.0011,"35":0.0011,"36":0.0011,"37":0.0012,"38":0.0012,"39":0.0012,"40":0.0013,"41":0.0013,"42":0.0013,"43":0.0013,"44":0.0014,"45":0.0014,"46":0.0014,"47":0.0015,"48":0.0015,"49":0.0015,"50":0.0016,"51":0.0016,"52":0.0016,"53":0.0017,"54":0.0017,"55":0.0017,"56":0.0017,"57":0.0018,"58":0.0018,"59":0.0018,"60":0.0019,"61":0.0019,"62":0.0019,"63":0.002,"64":0.002,"65":0.002,"66":0.0021,"67":0.0021,"68":0.0021,"69":0.0021,"70":0.0022,"71":0.0022,"72":0.0022,"73":0.0023,"74":0.0023,"75":0.0023,"76":0.0024,"77":0.0024,"78":0.0024,"79":0.0025,"80":0.0025,"81":0.0025,"82":0.0025,"83":0.0026,"84":0.0026,"85":0.0026,"86":0.0027,"87":0.0027,"88":0.0027,"89":0.0028,"90":0.0028,"91":0.0028,"92":0.0028,"93":0.0029,"94":0.0029,"95":0.0029,"96":0.003,"97":0.003,"98":0.003,"99":0.0031,"100":0.0031,"101":0.0031,"102":0.0032,"103":0.0032,"104":0.0032,"105":0.0032,"106":0.0033,"107":0.0033,"108":0.0033,"109":0.0034,"110":0.0034,"111":0.0034,"112":0.0035,"113":0.0035,"114":0.0035,"115":0.0036,"116":0.0036,"117":0.0036,"118":0.0036,"119":0.0037,"120":0.0037,"121":0.0037,"122":0.0038,"123":0.0038,"124":0.0038,"125":0.0039,"126":0.0039,"127":0.0039,"128":0.004,"129":0.004,"130":0.004,"131":0.004,"132":0.0041,"133":0.0041,"134":0.0041,"135":0.0042,"136":0.0042,"137":0.0042,"138":0.0043,"139":0.0043,"140":0.0043,"141":0.0044,"142":0.0044,"143":0.0044,"144":0.0044,"145":0.0045,"146":0.0045,"147":0.0045,"148":0.0046,"149":0.0046,"150":0.0046,"151":0.0047,"152":0.0047,"153":0.0047,"154":0.0047,"155":0.0048,"156":0.0048,"157":0.0048,"158":0.0049,"159":0.0049,"160":0.0049,"161":0.005,"162":0.005,"163":0.005,"164":0.0051,"165":0.0051,"166":0.0051,"167":0.0051,"168":0.0052,"169":0.0052,"170":0.0052,"171":0.0053,"172":0.0053,"173":0.0053,"174":0.0054,"175":0.0054,"176":0.0054,"177":0.0055,"178":0.0055,"179":0.0055,"180":0.0055,"181":0.0056,"182":0.0056,"183":0.0056,"184":0.0057,"185":0.0057,"186":0.0057,"187":0.0058,"188":0.0058,"189":0.0058,"190":0.0059,"191":0.0059,"192":0.0059,"193":0.0059,"194":0.006,"195":0.006,"196":0.006,"197":0.0061,"198":0.0061,"199":0.0061,"200":0.0062,"201":0.0062,"202":0.0062,"203":0.0063,"204":0.0063,"205":0.0063,"206":0.0063,"207":0.0064,"208":0.0064,"209":0.0064,"210":0.0065,"211":0.0065,"212":0.0065,"213":0.0066,"214":0.0066,"215":0.0066,"216":0.0066,"217":0.0067,"218":0.0067,"219":0.0067,"220":0.0068,"221":0.0068,"222":0.0068,"223":0.0069,"224":0.0069,"225":0.0069,"226":0.007,"227":0.007,"228":0.007,"229":0.007,"230":0.0071,"231":0.0071,"232":0.0071,"233":0.0072,"234":0.0072,"235":0.0072,"236":0.0073,"237":0.0073,"238":0.0073,"239":0.0074,"240":0.0074,"241":0.0074,"242":0.0074,"243":0.0075,"244":0.0075,"245":0.0075,"246":0.0076,"247":0.0076,"248":0.0076,"249":0.0077,"250":0.0077,"251":0.0077,"252":0.0078,"253":0.0078,"254":0.0078},"confidence":0.0}},"output_tokens":6056},{"name":"mixed","request":{"model":"kev-latest","state":"The door is open. The light is green.","questions":{"pick":{"type":"choice","instructions":"What color is the light?","criteria":{"red":null,"green":"","blue":"blue"}},"yes":{"type":"noul","instructions":"The door is open."},"no":{"type":"noul","instructions":"The door is closed."},"described":{"type":"noul","instructions":"Can we enter?","criteria":{"false":"door closed","true":"door open"}},"score":{"type":"score","instructions":"How safe is entering?","criteria":["unsafe","uncertain","safe"]}}},"record":{"state":"The door is open. The light is green.","questions":[{"instr":"What color is the light?","options":["red","green","blue: blue"],"label":0},{"instr":"The door is open.","options":["no","yes"],"label":0},{"instr":"The door is closed.","options":["no","yes"],"label":0},{"instr":"Can we enter?","options":["no: door closed","yes: door open"],"label":0},{"instr":"How safe is entering?","options":["unsafe","uncertain","safe"],"label":0}]},"meta":[{"id":"pick","type":"choice","keys":["red","green","blue"]},{"id":"yes","type":"noul","keys":["false","true"]},{"id":"no","type":"noul","keys":["false","true"]},{"id":"described","type":"noul","keys":["false","true"]},{"id":"score","type":"score","keys":["0","1","2"],"legend":{"0":"unsafe","1":"uncertain","2":"safe"}}],"rows":[{"ids":[248060,760,5817,369,1724,13,561,3000,369,5983,13,248061,3710,1829,369,279,3000,30,248049,1114,248050,248049,12865,248050,248049,11855,25,6105,248050,248062],"positions":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29],"readouts":[20,23,28,29]},{"ids":[248060,760,5817,369,1724,13,561,3000,369,5983,13,248061,760,5817,369,1724,13,248049,2083,248050,248049,9405,248050,248062],"positions":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23],"readouts":[19,22,23]},{"ids":[248060,760,5817,369,1724,13,561,3000,369,5983,13,248061,760,5817,369,7629,13,248049,2083,248050,248049,9405,248050,248062],"positions":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23],"readouts":[19,22,23]},{"ids":[248060,760,5817,369,1724,13,561,3000,369,5983,13,248061,6503,567,3599,30,248049,2083,25,5817,7629,248050,248049,9405,25,5817,1724,248050,248062],"positions":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28],"readouts":[21,27,28]},{"ids":[248060,760,5817,369,1724,13,561,3000,369,5983,13,248061,4199,5902,369,15783,30,248049,36895,248050,248049,1299,7373,248050,248049,18112,248050,248062],"positions":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27],"readouts":[19,23,26,27]}],"input_tokens":91,"probabilities":[[0.16666666666666666,0.3333333333333333,0.5],[0.3333333333333333,0.6666666666666666],[0.3333333333333333,0.6666666666666666],[0.3333333333333333,0.6666666666666666],[0.16666666666666666,0.3333333333333333,0.5]],"answers":{"pick":{"type":"choice","choice":"blue","confidence":0.25,"probabilities":{"red":0.1667,"green":0.3333,"blue":0.5}},"yes":{"type":"noul","noul":0.6667},"no":{"type":"noul","noul":0.6667},"described":{"type":"noul","noul":0.6667},"score":{"type":"score","score":1.3333,"legend":{"0":"unsafe","1":"uncertain","2":"safe"},"probabilities":{"0":0.1667,"1":0.3333,"2":0.5},"confidence":0.0}},"output_tokens":209},{"name":"question-reordered","request":{"model":"kev-latest","state":"The door is open. The light is green.","questions":{"score":{"type":"score","instructions":"How safe is entering?","criteria":["unsafe","uncertain","safe"]},"described":{"type":"noul","instructions":"Can we enter?","criteria":{"false":"door closed","true":"door open"}},"no":{"type":"noul","instructions":"The door is closed."},"yes":{"type":"noul","instructions":"The door is open."},"pick":{"type":"choice","instructions":"What color is the light?","criteria":{"red":null,"green":"","blue":"blue"}}}},"record":{"state":"The door is open. The light is green.","questions":[{"instr":"How safe is entering?","options":["unsafe","uncertain","safe"],"label":0},{"instr":"Can we enter?","options":["no: door closed","yes: door open"],"label":0},{"instr":"The door is closed.","options":["no","yes"],"label":0},{"instr":"The door is open.","options":["no","yes"],"label":0},{"instr":"What color is the light?","options":["red","green","blue: blue"],"label":0}]},"meta":[{"id":"score","type":"score","keys":["0","1","2"],"legend":{"0":"unsafe","1":"uncertain","2":"safe"}},{"id":"described","type":"noul","keys":["false","true"]},{"id":"no","type":"noul","keys":["false","true"]},{"id":"yes","type":"noul","keys":["false","true"]},{"id":"pick","type":"choice","keys":["red","green","blue"]}],"rows":[{"ids":[248060,760,5817,369,1724,13,561,3000,369,5983,13,248061,4199,5902,369,15783,30,248049,36895,248050,248049,1299,7373,248050,248049,18112,248050,248062],"positions":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27],"readouts":[19,23,26,27]},{"ids":[248060,760,5817,369,1724,13,561,3000,369,5983,13,248061,6503,567,3599,30,248049,2083,25,5817,7629,248050,248049,9405,25,5817,1724,248050,248062],"positions":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28],"readouts":[21,27,28]},{"ids":[248060,760,5817,369,1724,13,561,3000,369,5983,13,248061,760,5817,369,7629,13,248049,2083,248050,248049,9405,248050,248062],"positions":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23],"readouts":[19,22,23]},{"ids":[248060,760,5817,369,1724,13,561,3000,369,5983,13,248061,760,5817,369,1724,13,248049,2083,248050,248049,9405,248050,248062],"positions":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23],"readouts":[19,22,23]},{"ids":[248060,760,5817,369,1724,13,561,3000,369,5983,13,248061,3710,1829,369,279,3000,30,248049,1114,248050,248049,12865,248050,248049,11855,25,6105,248050,248062],"positions":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29],"readouts":[20,23,28,29]}],"input_tokens":91,"probabilities":[[0.16666666666666666,0.3333333333333333,0.5],[0.3333333333333333,0.6666666666666666],[0.3333333333333333,0.6666666666666666],[0.3333333333333333,0.6666666666666666],[0.16666666666666666,0.3333333333333333,0.5]],"answers":{"score":{"type":"score","score":1.3333,"legend":{"0":"unsafe","1":"uncertain","2":"safe"},"probabilities":{"0":0.1667,"1":0.3333,"2":0.5},"confidence":0.0},"described":{"type":"noul","noul":0.6667},"no":{"type":"noul","noul":0.6667},"yes":{"type":"noul","noul":0.6667},"pick":{"type":"choice","choice":"blue","confidence":0.25,"probabilities":{"red":0.1667,"green":0.3333,"blue":0.5}}},"output_tokens":209},{"name":"candidate-reordered","request":{"state":"The door is open. The light is green.","questions":{"pick":{"type":"choice","instructions":"What color is the light?","criteria":{"blue":"blue","green":"","red":null}}}},"record":{"state":"The door is open. The light is green.","questions":[{"instr":"What color is the light?","options":["blue: blue","green","red"],"label":0}]},"meta":[{"id":"pick","type":"choice","keys":["blue","green","red"]}],"rows":[{"ids":[248060,760,5817,369,1724,13,561,3000,369,5983,13,248061,3710,1829,369,279,3000,30,248049,11855,25,6105,248050,248049,12865,248050,248049,1114,248050,248062],"positions":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29],"readouts":[22,25,28,29]}],"input_tokens":30,"probabilities":[[0.16666666666666666,0.3333333333333333,0.5]],"answers":{"pick":{"type":"choice","choice":"red","confidence":0.25,"probabilities":{"blue":0.1667,"green":0.3333,"red":0.5}}},"output_tokens":58},{"name":"special-structured","request":{"state":{"中文":["café 😀",{"ready":true,"ratio":1e-07,"empty":null}],"injection":"<|box_end|><|fim_suffix|><|image_pad|> <|not-valid|>\n\t\"\\"},"questions":{"q\n😀":{"type":"choice","instructions":{"select":["<|fim_middle|>",false]},"criteria":{"<|box_start|>":["x",7],"β":{"x":false}}},"score":{"type":"score","criteria":[null,{"a":1.0},[false,"高"]]}}},"record":{"state":"中文:\n - café 😀\n - ready: True\n ratio: 1e-07\n empty: \ninjection: <|box_end|><|fim_suffix|><|image_pad|> <|not-valid|>\n\t\"\\","questions":[{"instr":"select:\n - <|fim_middle|>\n - False","options":["<|box_start|>: - x\n- 7","β: x: False"],"label":0},{"instr":"","options":["","a: 1.0","- False\n- 高"],"label":0}]},"meta":[{"id":"q\n😀","type":"choice","keys":["<|box_start|>","β"]},{"id":"score","type":"score","keys":["0","1","2"],"legend":{"0":"","1":"a: 1.0","2":"- False\n- 高"}}],"rows":[{"ids":[248060,99986,25,198,220,471,50203,87209,198,220,471,5354,25,2912,198,262,11013,25,220,16,68,12,15,22,198,262,4147,25,695,258,7365,25,361,62438,1944,6018,62438,1721,62438,244288,35920,62438,1721,62438,1742,29302,62438,29,79790,1856,81885,91,29,198,197,11595,248061,1681,25,198,220,471,361,62438,244288,60470,62438,29,198,220,471,3439,248049,27,62438,1944,4747,62438,25961,471,830,198,12,220,22,248050,248049,49531,25,830,25,3439,248050,248062],"positions":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93],"readouts":[85,92,93]},{"ids":[248060,99986,25,198,220,471,50203,87209,198,220,471,5354,25,2912,198,262,11013,25,220,16,68,12,15,22,198,262,4147,25,695,258,7365,25,361,62438,1944,6018,62438,1721,62438,244288,35920,62438,1721,62438,1742,29302,62438,29,79790,1856,81885,91,29,198,197,11595,248061,248049,248050,248049,64,25,220,16,13,15,248050,248049,12,3439,198,12,220,95883,248050,248062],"positions":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75],"readouts":[58,66,74,75]}],"input_tokens":114,"probabilities":[[0.3333333333333333,0.6666666666666666],[0.16666666666666666,0.3333333333333333,0.5]],"answers":{"q\n😀":{"type":"choice","choice":"β","confidence":0.3333,"probabilities":{"<|box_start|>":0.3333,"β":0.6667}},"score":{"type":"score","score":1.3333,"legend":{"0":"","1":"a: 1.0","2":"- False\n- 高"},"probabilities":{"0":0.1667,"1":0.3333,"2":0.5},"confidence":0.0}},"output_tokens":163},{"name":"empty-scalars","request":{"state":null,"questions":{"q":{"type":"choice","criteria":{"first":null,"second":false}},"n":{"type":"noul","instructions":1.0}}},"record":{"state":"","questions":[{"instr":"","options":["first","second: False"],"label":0},{"instr":"1.0","options":["no","yes"],"label":0}]},"meta":[{"id":"q","type":"choice","keys":["first","second"]},{"id":"n","type":"noul","keys":["false","true"]}],"rows":[{"ids":[248060,248061,248049,3765,248050,248049,5394,25,3439,248050,248062],"positions":[0,1,2,3,4,5,6,7,8,9,10],"readouts":[4,9,10]},{"ids":[248060,248061,16,13,15,248049,2083,248050,248049,9405,248050,248062],"positions":[0,1,2,3,4,5,6,7,8,9,10,11],"readouts":[7,10,11]}],"input_tokens":22,"probabilities":[[0.3333333333333333,0.6666666666666666],[0.3333333333333333,0.6666666666666666]],"answers":{"q":{"type":"choice","choice":"second","confidence":0.3333,"probabilities":{"first":0.3333,"second":0.6667}},"n":{"type":"noul","noul":0.6667}},"output_tokens":74}]} diff --git a/tests/kev/data/unicode-processing.json b/tests/kev/data/unicode-processing.json new file mode 100644 index 00000000..ddbee663 --- /dev/null +++ b/tests/kev/data/unicode-processing.json @@ -0,0 +1,116 @@ +{ + "reference": "2074cc44256200bb37540103fa4572901faa10f3", + "transformers": "5.17.0", + "cases": [ + { + "name": "unicode-marks", + "request": { + "state": "😤❤️❤️❤️ é क् हिन्दी", + "questions": { + "q": { + "type": "noul", + "instructions": "❤️ café" + } + } + }, + "record": { + "state": "😤❤️❤️❤️ é क् हिन्दी", + "questions": [ + { + "instr": "❤️ café", + "options": [ + "no", + "yes" + ], + "label": 0 + } + ] + }, + "meta": [ + { + "id": "q", + "type": "noul", + "keys": [ + "false", + "true" + ] + } + ], + "rows": [ + { + "ids": [ + 248060, + 72790, + 97, + 209726, + 209726, + 209726, + 3825, + 46215, + 28640, + 81399, + 181077, + 28640, + 148651, + 42201, + 248061, + 209726, + 50203, + 248049, + 2083, + 248050, + 248049, + 9405, + 248050, + 248062 + ], + "positions": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23 + ], + "readouts": [ + 19, + 22, + 23 + ] + } + ], + "input_tokens": 24, + "probabilities": [ + [ + 0.3333333333333333, + 0.6666666666666666 + ] + ], + "answers": { + "q": { + "type": "noul", + "noul": 0.6667 + } + }, + "output_tokens": 22 + } + ] +} diff --git a/tests/kev/generate_head_fixture.py b/tests/kev/generate_head_fixture.py new file mode 100644 index 00000000..4b3eff3e --- /dev/null +++ b/tests/kev/generate_head_fixture.py @@ -0,0 +1,91 @@ +"""Run the pinned upstream PointerHead on deterministic hidden states, on CPU.""" + +import argparse +import ast +import hashlib +import json +import math +from pathlib import Path + +import torch + +SOURCE_SHA256 = "2ff3b2e67c048e3bc5a0d1a32aafe50804ed315a5564fce06bb4b95697c5271c" +HEAD_SHA256 = "dd633435998ecc751ac538717a3742e32149500fabf7d7276287dbf0693f347c" + + +def hidden(row): + return torch.tensor( + [((i * 37 + row * 19) % 101 - 50) / 64 for i in range(2560)], + dtype=torch.float32, + ) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--reference-model", type=Path, required=True) + parser.add_argument("--head", type=Path, required=True) + parser.add_argument("--out", type=Path, required=True) + args = parser.parse_args() + source = args.reference_model.read_bytes() + if ( + hashlib.sha256(source).hexdigest() != SOURCE_SHA256 + or hashlib.sha256(args.head.read_bytes()).hexdigest() != HEAD_SHA256 + ): + raise ValueError("reference source or head does not match the pinned release") + # Execute the actual pinned class without importing the unrelated backbone and + # training dependencies. The hash above covers the entire upstream model.py. + definition = next( + n + for n in ast.parse(source).body + if isinstance(n, ast.ClassDef) and n.name == "PointerHead" + ) + namespace = {"nn": torch.nn, "math": math} + exec( + compile( + ast.Module(body=[definition], type_ignores=[]), + str(args.reference_model), + "exec", + ), + namespace, + ) + meta = torch.load(args.head, map_location="cpu", weights_only=True) + head = namespace["PointerHead"](2560, 256) + head.load_state_dict(meta["head"], strict=True) + head.temperature = meta["temperature"] + torch.set_num_threads(1) + cases = [] + with torch.no_grad(): + for n in (1, 2, 7, 31, 255): + options = torch.stack([hidden(i + 1) for i in range(n)]) + head.train() + logits = head(hidden(0), options) + head.eval() + probabilities = head(hidden(0), options).softmax(-1) + cases.append( + { + "options": n, + "logits": logits.tolist(), + "probabilities": probabilities.tolist(), + } + ) + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text( + json.dumps( + { + "reference_revision": "2074cc44256200bb37540103fa4572901faa10f3", + "reference_model_sha256": SOURCE_SHA256, + "head_pt_sha256": HEAD_SHA256, + "torch": torch.__version__, + "device": "cpu", + "temperature": meta["temperature"], + "hidden_rule": "((i * 37 + row * 19) % 101 - 50) / 64; row 0 = decide", + "cases": cases, + }, + indent=2, + ) + + "\n" + ) + + +if __name__ == "__main__": + main() diff --git a/tests/kev/generate_planning_fixture.py b/tests/kev/generate_planning_fixture.py new file mode 100644 index 00000000..34b86e5f --- /dev/null +++ b/tests/kev/generate_planning_fixture.py @@ -0,0 +1,77 @@ +"""Regenerate the CPU geometry oracle from the pinned official implementation.""" + +import argparse +import hashlib +import importlib.util +import json +from pathlib import Path + +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument("--reference", type=Path, required=True) +parser.add_argument("--out", type=Path, required=True) +args = parser.parse_args() +source = args.reference / "kev/cuda_graphs.py" +assert ( + hashlib.sha256(source.read_bytes()).hexdigest() + == "5ce84f0e4e5ad2177314ed3ecc9564d45d14ad9eb422e0fb079c9e1cbbb011b6" +) +spec = importlib.util.spec_from_file_location("graphs", source) +g = importlib.util.module_from_spec(spec) +spec.loader.exec_module(g) +fixtures = [] +for split, lengths in [ + (1, [1]), + (29, [27, 12]), + (67, [66, 70]), + (52, [69]), + (7, [1679]), + (29, [12] * 5), + (29, [12] * 64), + (4096, [18] * 32), + (1025, [999, 32, 400, 33, 8]), + (29, [16, 200, 900, 16]), + (8000, [5000, 32, 500, 200]), + (7, [1679, 14, 1680]), +]: + state = g.bucket(split) if g.bucket(split) <= 1024 else split + graphed = g.bucket(max(lengths)) <= 1024 and split <= 4096 + prefix = g.pow2(max(16, split)) if graphed else split + groups = ( + g.length_groups(lengths, 32) + if graphed + else [ + list(range(len(lengths)))[i : i + max(1, 16384 // (split + max(lengths)))] + for i in range(0, len(lengths), max(1, 16384 // (split + max(lengths)))) + ] + ) + result = [None] * len(lengths) + for group in groups: + question = ( + g.bucket(max(lengths[i] for i in group)) + if graphed + else max(lengths[i] for i in group) + ) + cap = min(32, 32768 // (prefix + question)) if graphed else len(group) + limit = max( + n for n in range(1, cap + 1) if not graphed or g.count_bucket(n) <= cap + ) + for start in range(0, len(group), limit): + part = group[start : start + limit] + count = g.count_bucket(len(part)) if graphed else len(part) + for index, i in enumerate(part): + result[i] = { + "split": split, + "state": state, + "prefix": prefix, + "question": question, + "count": count, + "index": index, + } + fixtures.append({"split": split, "lengths": lengths, "plans": result}) +args.out.write_text( + json.dumps( + {"reference": "2074cc44256200bb37540103fa4572901faa10f3", "cases": fixtures}, + separators=(",", ":"), + ) + + "\n" +) diff --git a/tests/kev/generate_processing_fixture.py b/tests/kev/generate_processing_fixture.py new file mode 100644 index 00000000..230948e6 --- /dev/null +++ b/tests/kev/generate_processing_fixture.py @@ -0,0 +1,94 @@ +"""Capture Kev's pinned API/encode/rows_of oracle without loading a model.""" +import argparse +import hashlib +import json +import sys +from pathlib import Path + +SOURCE_HASHES = { + 'model.py': '2ff3b2e67c048e3bc5a0d1a32aafe50804ed315a5564fce06bb4b95697c5271c', + 'api.py': 'a6f53af5354f8ea17915abc78360fdf291a5108ce39a53a2d9135d48c603ae58', + 'checkpoint.py': '02062399977ee0da316dd52e03a79fa5e10134aa49c0b5ed801598c52042dc32', +} + + +def official(reference): + for name, expected in SOURCE_HASHES.items(): + assert hashlib.sha256((reference / 'kev' / name).read_bytes()).hexdigest() == expected, name + sys.path.insert(0, str(reference)) + from kev import api, model + return api, model + + +def cases(): + def choice(n): + return {'type': 'choice', 'instructions': 'Select the option matching the state.', + 'criteria': {f'k{i}': f'number {i}' for i in range(n)}} + result = [{'name': f'choice-{n}', 'request': {'state': f'The number is {n-1}.', + 'questions': {'pick': choice(n)}}} for n in (1, 2, 7, 31, 255)] + result += [{'name': f'score-{n}', 'request': {'state': 'The task is mostly complete.', + 'questions': {'rating': {'type': 'score', 'instructions': 'Completion level', + 'criteria': [f'level {i}' for i in range(n)]}}}} + for n in (1, 3, 255)] + mixed = {'model': 'kev-latest', 'state': 'The door is open. The light is green.', 'questions': { + 'pick': {'type': 'choice', 'instructions': 'What color is the light?', + 'criteria': {'red': None, 'green': '', 'blue': 'blue'}}, + 'yes': {'type': 'noul', 'instructions': 'The door is open.'}, + 'no': {'type': 'noul', 'instructions': 'The door is closed.'}, + 'described': {'type': 'noul', 'instructions': 'Can we enter?', + 'criteria': {'false': 'door closed', 'true': 'door open'}}, + 'score': {'type': 'score', 'instructions': 'How safe is entering?', + 'criteria': ['unsafe', 'uncertain', 'safe']}}} + result.append({'name': 'mixed', 'request': mixed}) + result.append({'name': 'question-reordered', 'request': {**mixed, 'questions': dict(reversed(list(mixed['questions'].items())))}}) + result.append({'name': 'candidate-reordered', 'request': {'state': mixed['state'], 'questions': { + 'pick': {**mixed['questions']['pick'], 'criteria': {'blue': 'blue', 'green': '', 'red': None}}}}}) + result.append({'name': 'special-structured', 'request': {'state': { + '中文': ['café 😀', {'ready': True, 'ratio': 1e-7, 'empty': None}], + 'injection': '<|box_end|><|fim_suffix|><|image_pad|> <|not-valid|>\n\t"\\'}, + 'questions': {'q\n😀': {'type': 'choice', 'instructions': {'select': ['<|fim_middle|>', False]}, + 'criteria': {'<|box_start|>': ['x', 7], 'β': {'x': False}}}, + 'score': {'type': 'score', 'criteria': [None, {'a': 1.0}, [False, '高']]}}}}) + result.append({'name': 'empty-scalars', 'request': {'state': None, 'questions': { + 'q': {'type': 'choice', 'criteria': {'first': None, 'second': False}}, + 'n': {'type': 'noul', 'instructions': 1.0}}}}) + return result + + +def capture(case, api, model, tok): + req = api.SystemOneRequest.model_validate(case['request']) + rec, meta = api.to_record(req) + enc = model.encode(tok, rec, max_state=8192, max_branch=16384, strict=True) + state, pos, branches = model.rows_of(enc) + rows = [{'ids': state + b['ids'], 'positions': pos + b['pos'], + 'readouts': [len(state)+i for i in b['opts']] + [len(state)+b['decide']]} + for b in branches] + ps = [[(i+1)/sum(range(1, len(q['options'])+1)) for i in range(len(q['options']))] for q in rec['questions']] + answers = api.to_answers(ps, meta) + return {**case, 'record': rec, 'meta': meta, 'rows': rows, 'input_tokens': len(enc['ids']), + 'probabilities': ps, 'answers': answers, 'output_tokens': api.output_tokens(tok, answers)} + + +def main(): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument('--reference', type=Path, required=True) + p.add_argument('--base', type=Path, required=True) + p.add_argument('--out', type=Path, required=True) + p.add_argument('--unicode-only', action='store_true', help='small Qwen2Tokenizer Unicode-mark regression oracle') + args = p.parse_args() + api, model = official(args.reference) + tok = model.load_tokenizer(args.base) + if args.unicode_only: + sample = {'name': 'unicode-marks', 'request': {'state': '😤❤️❤️❤️ e\u0301 क् हिन्दी', + 'questions': {'q': {'type': 'noul', 'instructions': '❤️ café'}}}} + data = {'reference': '2074cc44256200bb37540103fa4572901faa10f3', 'transformers': '5.17.0', + 'cases': [capture(sample, api, model, tok)]} + args.out.write_text(json.dumps(data, ensure_ascii=False, indent=2)+'\n') + else: + data = {'reference': '2074cc44256200bb37540103fa4572901faa10f3', 'source_hashes': SOURCE_HASHES, + 'cases': [capture(c, api, model, tok) for c in cases()]} + args.out.write_text(json.dumps(data, ensure_ascii=False, separators=(',', ':')) + '\n') + + +if __name__ == '__main__': + main() diff --git a/tests/kev/head.rs b/tests/kev/head.rs new file mode 100644 index 00000000..b7eefb83 --- /dev/null +++ b/tests/kev/head.rs @@ -0,0 +1,127 @@ +use omni_kev_native::{head::PointerHead, postprocess::probabilities}; +use safetensors::{ + Dtype, + tensor::{TensorView, serialize}, +}; +use std::{ + path::PathBuf, + sync::atomic::{AtomicU64, Ordering}, +}; + +static SERIAL: AtomicU64 = AtomicU64::new(0); + +struct Fixture(PathBuf); +impl Fixture { + fn new(weights: &[f32]) -> Self { + let path = std::env::temp_dir().join(format!( + "kev-head-{}-{}.safetensors", + std::process::id(), + SERIAL.fetch_add(1, Ordering::Relaxed) + )); + let data: Vec = weights.iter().flat_map(|v| v.to_le_bytes()).collect(); + let bias: Vec = [0.5f32, -0.25] + .iter() + .flat_map(|v| v.to_le_bytes()) + .collect(); + let tensors = [ + ( + "q.weight", + TensorView::new(Dtype::F32, vec![2, 2], &data).unwrap(), + ), + ( + "k.weight", + TensorView::new(Dtype::F32, vec![2, 2], &data).unwrap(), + ), + ( + "q.bias", + TensorView::new(Dtype::F32, vec![2], &bias).unwrap(), + ), + ( + "k.bias", + TensorView::new(Dtype::F32, vec![2], &bias).unwrap(), + ), + ]; + std::fs::write(&path, serialize(tensors, None).unwrap()).unwrap(); + Self(path) + } +} +impl Drop for Fixture { + fn drop(&mut self) { + let _ = std::fs::remove_file(&self.0); + } +} + +#[test] +fn trained_biases_scaling_calibration_and_candidate_order() { + let fixture = Fixture::new(&[1., 0., 0., 1.]); + let head = PointerHead::load(&fixture.0).unwrap(); + let logits = head + .logits(&[1., 2.], &[&[3., 4.], &[1., 0.], &[3., 4.]]) + .unwrap(); + let expected = [11.8125f32, 1.8125, 11.8125]; // dot after both learned biases + for (actual, expected) in logits.iter().zip(expected) { + assert!((actual - expected / 2f32.sqrt()).abs() < 1e-6); + } + let probs = probabilities(&logits, 2.4).unwrap(); + assert_eq!(probs[0], probs[2]); + assert!(probs[0] > probs[1]); + assert!((probs.iter().sum::() - 1.).abs() < 1e-6); + assert_eq!(probabilities(&[123.], 2.4).unwrap(), vec![1.]); + assert!(head.logits(&[1., 2.], &vec![&[0., 1.][..]; 255]).is_ok()); +} + +#[test] +fn rejects_invalid_weights_inputs_and_calibration() { + let fixture = Fixture::new(&[1., 0., 0., 1.]); + let head = PointerHead::load(&fixture.0).unwrap(); + assert!(head.logits(&[1.], &[&[1., 2.]]).is_err()); + assert!(head.logits(&[1., 2.], &[&[f32::NAN, 2.]]).is_err()); + assert!(head.logits(&[1., 2.], &[]).is_err()); + assert!(head.logits(&[1., 2.], &vec![&[0., 1.][..]; 256]).is_err()); + assert!(PointerHead::load(&Fixture::new(&[f32::NAN; 4]).0).is_err()); + std::fs::write(&fixture.0, b"invalid safetensors").unwrap(); + assert!(PointerHead::load(&fixture.0).is_err()); + for temperature in [0., -1., f64::NAN, f64::INFINITY, 1e-100, 1e100] { + assert!(probabilities(&[1., 2.], temperature).is_err()); + } + assert!(probabilities(&[f32::INFINITY], 1.).is_err()); + assert!(probabilities(&[], 1.).is_err()); +} + +fn hidden(row: usize) -> Vec { + (0..2560) + .map(|i| (((i * 37 + row * 19) % 101) as f32 - 50.) / 64.) + .collect() +} + +#[test] +#[ignore = "needs KEV_HEAD pointing to the exported pinned head.safetensors; CPU only"] +fn released_head_matches_pinned_upstream_class() { + let path = PathBuf::from(std::env::var_os("KEV_HEAD").expect("set KEV_HEAD")); + let head = PointerHead::load(&path).unwrap(); + let fixture: serde_json::Value = serde_json::from_str(include_str!("data/head.json")).unwrap(); + let mut max_logit_error = 0f64; + let mut max_probability_error = 0f64; + for case in fixture["cases"].as_array().unwrap() { + let options: Vec<_> = (1..=case["options"].as_u64().unwrap() as usize) + .map(hidden) + .collect(); + let refs: Vec<_> = options.iter().map(Vec::as_slice).collect(); + let logits = head.logits(&hidden(0), &refs).unwrap(); + let probs = probabilities(&logits, fixture["temperature"].as_f64().unwrap()).unwrap(); + for (&actual, expected) in logits.iter().zip(case["logits"].as_array().unwrap()) { + let expected = expected.as_f64().unwrap(); + let error = (actual as f64 - expected).abs(); + max_logit_error = max_logit_error.max(error); + assert!(error <= 1e-4 + 1e-5 * expected.abs(), "logit error {error}"); + } + for (&actual, expected) in probs.iter().zip(case["probabilities"].as_array().unwrap()) { + let error = (actual as f64 - expected.as_f64().unwrap()).abs(); + max_probability_error = max_probability_error.max(error); + assert!(error <= 1e-5, "probability error {error}"); + } + } + eprintln!( + "max logit error={max_logit_error:.8e}; max probability error={max_probability_error:.8e}" + ); +} diff --git a/tests/kev/native.rs b/tests/kev/native.rs new file mode 100644 index 00000000..05b07ba8 --- /dev/null +++ b/tests/kev/native.rs @@ -0,0 +1,91 @@ +//! Explicit GPU capture for three-way parity. Executed only under gpu run. +use crate::{checkpoint, executor::Executor, processing::Processor}; +use serde_json::{Value, json}; + +#[test] +#[ignore = "requires KEV_EXPORT, KEV_CUDA_LIB, KEV_NATIVE_OUT and a reserved CUDA GPU"] +fn native_corpus_capture() { + let dir = std::path::PathBuf::from(std::env::var("KEV_EXPORT").expect("KEV_EXPORT")); + let lib = std::path::PathBuf::from(std::env::var("KEV_CUDA_LIB").expect("KEV_CUDA_LIB")); + let out = std::path::PathBuf::from(std::env::var("KEV_NATIVE_OUT").expect("KEV_NATIVE_OUT")); + assert!( + !out.exists(), + "preserve previous evidence; choose a fresh output" + ); + checkpoint::verify(&dir).unwrap(); + let processor = Processor::load(&dir).unwrap(); + let mut executor = Executor::load(&dir, &lib).unwrap(); + let fixture: Value = serde_json::from_str(include_str!("data/processing.json")).unwrap(); + let mut captured = vec![]; + for case in fixture["cases"] + .as_array() + .unwrap() + .iter() + .chain(std::iter::repeat_n(&fixture["cases"][8], 3)) + { + let prepared = processor + .prepare(case["request"].as_object().unwrap()) + .unwrap(); + assert_eq!(json!(prepared.rows), case["rows"]); + let logits = executor.execute(&prepared.rows).unwrap(); + let probabilities: Vec<_> = logits + .iter() + .map(|row| { + crate::postprocess::probabilities(row, crate::contract::TEMPERATURE).unwrap() + }) + .collect(); + let body = prepared.context.finish(logits.clone()).unwrap(); + captured.push( + json!({"name":case["name"],"logits":logits,"probabilities":probabilities,"body":body}), + ); + std::fs::write( + &out, + serde_json::to_vec_pretty(&json!({"kind":"native","cases":captured})).unwrap(), + ) + .unwrap(); + } +} + +#[test] +#[ignore = "requires a frozen KEV_CORPUS and KEV_NATIVE_OUT, checkpoint and reserved CUDA GPU"] +fn native_service_capture() { + use std::io::Write; + let dir = std::path::PathBuf::from(std::env::var("KEV_EXPORT").expect("KEV_EXPORT")); + let lib = std::path::PathBuf::from(std::env::var("KEV_CUDA_LIB").expect("KEV_CUDA_LIB")); + let corpus = std::env::var("KEV_CORPUS").expect("KEV_CORPUS"); + let output = std::env::var("KEV_NATIVE_OUT").expect("KEV_NATIVE_OUT"); + let mut stream = std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(output) + .expect("choose a fresh evidence file"); + let fixture: Value = serde_json::from_slice(&std::fs::read(corpus).unwrap()).unwrap(); + checkpoint::verify(&dir).unwrap(); + let processor = Processor::load(&dir).unwrap(); + let mut executor = Executor::load(&dir, &lib).unwrap(); + for (i, case) in fixture["cases"].as_array().unwrap().iter().enumerate() { + let prepared = processor + .prepare(case["request"].as_object().unwrap()) + .unwrap(); + assert_eq!(json!(prepared.rows), case["rows"], "{}", case["name"]); + let logits = executor.execute(&prepared.rows).unwrap(); + let probabilities: Vec<_> = logits + .iter() + .map(|row| { + crate::postprocess::probabilities(row, crate::contract::TEMPERATURE).unwrap() + }) + .collect(); + let body = prepared.context.finish(logits.clone()).unwrap(); + assert_eq!(body["usage"]["input_tokens"], case["input_tokens"]); + serde_json::to_writer( + &mut stream, + &json!({"name":case["name"],"logits":logits,"probabilities":probabilities,"body":body}), + ) + .unwrap(); + writeln!(stream).unwrap(); + stream.flush().unwrap(); + if i % 25 == 0 { + eprintln!("native {} {}", i + 1, case["name"]); + } + } +} diff --git a/tests/kev/planning.rs b/tests/kev/planning.rs new file mode 100644 index 00000000..14fd5b1a --- /dev/null +++ b/tests/kev/planning.rs @@ -0,0 +1,44 @@ +use super::*; + +#[test] +fn geometry_matches_pinned_official_length_groups() { + let fixture: serde_json::Value = + serde_json::from_str(include_str!("data/planning.json")).unwrap(); + for case in fixture["cases"].as_array().unwrap() { + let lengths: Vec<_> = case["lengths"] + .as_array() + .unwrap() + .iter() + .map(|v| v.as_u64().unwrap() as usize) + .collect(); + let actual = geometry(case["split"].as_u64().unwrap() as usize, &lengths); + for (a, e) in actual.iter().zip(case["plans"].as_array().unwrap()) { + assert_eq!( + [a.split, a.state, a.question, a.prefix, a.count, a.index], + ["split", "state", "question", "prefix", "count", "index"] + .map(|k| e[k].as_u64().unwrap() as usize), + "{case}" + ); + } + } +} + +#[test] +fn state_boundary_and_common_prefix_are_validated() { + let row = |ids: Vec| Row { + positions: (0..ids.len() as i64).collect(), + ids, + readouts: vec![], + }; + assert!(plan(&[]).is_err()); + assert!( + plan(&[ + row(vec![248060, 1, 248061, 2]), + row(vec![248060, 3, 248061, 2]) + ]) + .is_err() + ); + assert!(plan(&[row(vec![248061, 2])]).is_err()); + assert!(plan(&[row(vec![248060, 248061, 2, 248061])]).is_err()); + assert!(plan(&[row(vec![248060, 248061, 2])]).is_ok()); +} diff --git a/tests/kev/processing.rs b/tests/kev/processing.rs new file mode 100644 index 00000000..af93f28d --- /dev/null +++ b/tests/kev/processing.rs @@ -0,0 +1,159 @@ +use omni_kev_native::{ + contract, + executor::validate_row, + postprocess, + processing::{Processor, Row, sanitize}, +}; +use omni_qwen3_5_native::json; +use serde_json::{Value, json}; + +fn fixture() -> Value { + serde_json::from_str(include_str!("data/processing.json")).unwrap() +} + +#[test] +fn official_api_rendering_and_answers() { + for case in fixture()["cases"].as_array().unwrap() { + let request = json::parse(&serde_json::to_vec(&case["request"]).unwrap()).unwrap(); + let compiled = contract::compile(&request).unwrap(); + assert_eq!(compiled.state, case["record"]["state"], "{}", case["name"]); + for (q, expected) in compiled + .questions + .iter() + .zip(case["record"]["questions"].as_array().unwrap()) + { + assert_eq!(q.instructions, expected["instr"]); + assert_eq!(json!(q.options), expected["options"]); + } + let ps: Vec> = serde_json::from_value(case["probabilities"].clone()).unwrap(); + assert_eq!( + postprocess::answers(&compiled.questions, &ps).unwrap(), + case["answers"], + "{}", + case["name"] + ); + } +} + +#[test] +fn sanitized_controls_and_json_scalars() { + assert_eq!( + sanitize("<|a_2|><|box_end|> <|not-valid|>"), + "<¦a_2¦><¦box_end¦> <|not-valid|>" + ); + assert_eq!( + contract::render(&json!([true, false, null, 1.0, 1e-7]), 0), + "- True\n- False\n- \n- 1.0\n- 1e-07" + ); + assert_eq!( + contract::render(&json!(["\u{1c}x", "\u{1f}y"]), 0), + "- x\n- y" + ); + assert_eq!( + postprocess::billing_json(&json!({"😀":"中文"})), + r#"{"\ud83d\ude00": "\u4e2d\u6587"}"# + ); +} + +#[test] +fn invalid_contract_and_readouts_rejected() { + for request in [ + json!({"questions":{"q":{"type":"noul"}}}), + json!({"model":"other","state":"","questions":{"q":{"type":"noul"}}}), + json!({"state":"","questions":{}}), + json!({"state":"","questions":{"q":{"type":"span"}}}), + json!({"state":"","questions":{"q":{"type":"choice","criteria":{}}}}), + json!({"state":"","questions":{"q":{"type":"score","criteria":[]}}}), + json!({"state":"","questions":{"q":{"type":"score","criteria":vec!["x";256]}}}), + json!({"state":"","questions":{"q":{"type":"noul","criteria":[]}}}), + ] { + assert!( + contract::compile(request.as_object().unwrap()).is_err(), + "{request}" + ); + } + assert!(json::parse(br#"{"state":1,"state":2}"#).is_err()); + let row: Row = serde_json::from_value(fixture()["cases"][0]["rows"][0].clone()).unwrap(); + validate_row(&row).unwrap(); + let mut bad = row.clone(); + bad.readouts.reverse(); + assert!(validate_row(&bad).is_err()); + let mut bad = row.clone(); + bad.positions[0] = 1; + assert!(validate_row(&bad).is_err()); + let mut bad = row; + bad.ids[0] = 999999; + assert!(validate_row(&bad).is_err()); +} + +#[test] +fn ties_and_whole_question_temperature() { + let request = + json!({"state":"", "questions":{"q":{"type":"choice","criteria":{"z":null,"a":null}}}}); + let compiled = contract::compile(request.as_object().unwrap()).unwrap(); + let answers = postprocess::answers(&compiled.questions, &[vec![0.5, 0.5]]).unwrap(); + assert_eq!(answers["q"]["choice"], "z"); + assert_eq!(answers["q"]["confidence"], 0.0); + let p = postprocess::probabilities(&[0.0, contract::TEMPERATURE as f32], contract::TEMPERATURE) + .unwrap(); + assert!((p[1] - 0.7310586).abs() < 1e-6); + assert!(postprocess::answers(&compiled.questions, &[vec![0.3, 0.3]]).is_err()); +} + +#[test] +#[ignore = "requires KEV_EXPORT with the pinned base tokenizer; CPU only"] +fn exact_tokens_readouts_and_billing_match_official() { + let dir = std::env::var("KEV_EXPORT").expect("KEV_EXPORT"); + let processor = Processor::load(std::path::Path::new(&dir)).unwrap(); + let unicode: Value = + serde_json::from_str(include_str!("data/unicode-processing.json")).unwrap(); + let external: Value = std::env::var("KEV_CORPUS") + .map(|path| serde_json::from_slice(&std::fs::read(path).unwrap()).unwrap()) + .unwrap_or_else(|_| json!({"cases":[]})); + for case in fixture()["cases"] + .as_array() + .unwrap() + .iter() + .chain(unicode["cases"].as_array().unwrap()) + .chain(external["cases"].as_array().unwrap()) + { + let request = json::parse(&serde_json::to_vec(&case["request"]).unwrap()).unwrap(); + let prepared = processor.prepare(&request).unwrap(); + let compiled = contract::compile(&request).unwrap(); + assert_eq!(compiled.state, case["record"]["state"], "{}", case["name"]); + assert_eq!( + compiled.questions.len(), + case["meta"].as_array().unwrap().len() + ); + for (q, expected) in compiled + .questions + .iter() + .zip(case["record"]["questions"].as_array().unwrap()) + { + assert_eq!(q.instructions, expected["instr"], "{}", case["name"]); + assert_eq!(json!(q.options), expected["options"], "{}", case["name"]); + } + assert_eq!(json!(prepared.rows), case["rows"], "{}", case["name"]); + assert_eq!( + prepared.context.input_tokens as u64, + case["input_tokens"].as_u64().unwrap() + ); + if case.get("probabilities").is_none() { + // Discovery corpora deliberately omit artificial CPU probabilities. + // Their real output/billing is checked by the service comparator. + assert_eq!(case["population"], "discovery"); + continue; + } + let ps: Vec> = serde_json::from_value(case["probabilities"].clone()).unwrap(); + let body = prepared.context.finish_probabilities(&ps).unwrap(); + assert_eq!(body["answers"], case["answers"]); + assert_eq!( + body["usage"]["output_tokens"], case["output_tokens"], + "{}", + case["name"] + ); + } + let over = + json!({"state":"x ".repeat(contract::MAX_STATE+1),"questions":{"q":{"type":"noul"}}}); + assert!(processor.prepare(over.as_object().unwrap()).is_err()); +} diff --git a/tests/kev/service_acceptance.py b/tests/kev/service_acceptance.py new file mode 100644 index 00000000..316cec93 --- /dev/null +++ b/tests/kev/service_acceptance.py @@ -0,0 +1,275 @@ +"""Pinned default-service acceptance. GPU subcommands MUST run under gpu run. + +The official server is unchanged: observers only save its unrounded response +inputs and count graph replays. No precision, cache, graph or kernel override. +""" +import argparse +import collections +import hashlib +import importlib.metadata +import importlib.util +import json +import os +import subprocess +import sys +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +REFERENCE = '2074cc44256200bb37540103fa4572901faa10f3' +SOURCES = { + 'v4/transfer-v4': 'ff374c49c6c9f15f8a56fb274b4a4857d20497eb8dd1ac07ce01560e682a5f2e', + 'v4/decision-v4': '8d5765d7aec4d08c61854f4664ca79ec2ba44ed092e9b967eaf61ed86c496d9c', + 'hard-v1': '3cdbf12a6b7c3b70c73e8c39f2677127ea61ef32f0ea75b55a43556d884b2d98', +} +GATES = { + 'probability_max_abs': 0.01, 'score_max_abs': 0.1, + 'choice_margin': 0.02, 'noul_distance': 0.01, + 'minimum_boundary_per_type': 10, + 'accuracy_drop': 0.01, 'per_type_accuracy_drop': 0.02, + 'brier_increase': 0.01, 'normalized_score_mae_increase': 0.005, +} + + +def module(name, path): + spec = importlib.util.spec_from_file_location(name, path) + result = importlib.util.module_from_spec(spec) + spec.loader.exec_module(result) + return result + + +def digest(path): + return hashlib.sha256(Path(path).read_bytes()).hexdigest() + + +def save(path, value): + Path(path).write_text(json.dumps(value, ensure_ascii=False, indent=2, allow_nan=False)+'\n') + + +def read_lines(path): + return [json.loads(line) for line in Path(path).read_text().split('\n') if line.strip()] + + +def request_digest(value): + return hashlib.sha256(json.dumps(value, ensure_ascii=False, sort_keys=True).encode()).hexdigest() + + +def prepare(args): + """Choose every task row before inference; discovery never uses native scores.""" + if args.out.exists(): + raise ValueError('preserve previous corpus') + fixture = module('kev_fixture', ROOT/'tests/kev/generate_processing_fixture.py') + api, model = fixture.official(args.reference) + tok = model.load_tokenizer(args.base) + cases = [{**c, 'population': 'contract', 'labels': {}, 'source': 'contract'} for c in fixture.cases()] + for suite, sha in SOURCES.items(): + path = args.reference/'evals'/suite/'development.jsonl' + assert digest(path) == sha, path + for record in read_lines(path): + meta = record['_meta'] + if meta['variant'] != 'clean': + continue + if suite == 'v4/decision-v4' and not any(q['type'] == 'score' for q in record['questions'].values()): + continue + if suite == 'hard-v1' and meta['source'] not in ('hard_ambiguous', 'hard_probability'): + continue + questions, labels = {}, {} + for qid, q in record['questions'].items(): + if suite == 'v4/decision-v4' and q['type'] != 'score': + continue + questions[qid] = {k: q[k] for k in ('type', 'instructions', 'criteria') if k in q} + keys = api.question_keys(q['type'], q.get('criteria')) + labels[qid] = keys.index(q['label']) if q['type'] == 'choice' else int(q['label']) + cases.append({'name': f'{suite}:{meta["id"]}', 'population': 'discovery' if suite == 'hard-v1' else 'task', + 'source': meta['source'], 'group': meta.get('group_id', meta['id']), 'labels': labels, + 'request': {'model': 'kev-latest', 'state': record['state'], 'questions': questions}}) + # Explicitly synthetic stress inputs, kept separate from labeled task quality. + # A fixed sweep is retained in full, even when it does not yield a boundary. + for i in range(101): + chance = 45 + i/10 + cases.append({'name': f'boundary-probe-{i:03}', 'population': 'discovery', 'source': 'synthetic_probability', 'labels': {}, + 'request': {'model': 'kev-latest', 'state': f'A lottery has exactly {chance:.1f}% winning tickets. One ticket is drawn uniformly at random. Its outcome has not been revealed.', + 'questions': {'outcome': {'type': 'choice', 'instructions': 'Predict the outcome of the ticket.', + 'criteria': {'win': 'The ticket wins.', 'lose': 'The ticket does not win.'}}, + 'wins': {'type': 'noul', 'instructions': 'The ticket wins.'}}}}) + mixed = next(c for c in cases if c['name'] == 'mixed') + cases += [{**mixed, 'name': f'mixed-repeat-{i}', 'population': 'repeat'} for i in range(3)] + captured = [] + for case in cases: + captured.append(fixture.capture(case, api, model, tok)) + assert len(captured[-1]['rows']) <= 64 + assert sum(len(row['ids']) for row in captured[-1]['rows']) <= 65536 + assert len({c['name'] for c in captured}) == len(captured) + counts = collections.Counter(c['population'] for c in captured) + save(args.out, {'reference': REFERENCE, 'sources': SOURCES, 'gates': GATES, 'counts': counts, 'cases': captured}) + print('frozen corpus', dict(counts), 'sha256', digest(args.out), flush=True) + + +def official_server(args): + """Run the official main with environment defaults, recording raw probabilities.""" + overrides = sorted(k for k in os.environ if k.startswith('KEV_')) + if overrides: + raise ValueError(f'official defaults require no KEV_* environment variables: {overrides}') + assert subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=args.reference, text=True).strip() == REFERENCE + assert not subprocess.check_output(['git', 'diff', 'HEAD', '--', 'kev'], cwd=args.reference) + sys.path.insert(0, str(args.reference)) + import torch + from kev import serve + from kev.checkpoint import fused_available + replay_count = 0 + original_replay = torch.cuda.CUDAGraph.replay + + def observed_replay(graph): + nonlocal replay_count + result = original_replay(graph) + replay_count += 1 + return result + + torch.cuda.CUDAGraph.replay = observed_replay + original_body = serve.Server._body + with args.out.open('x') as stream: + def observed_body(server, req, meta, probabilities, stats): + body = original_body(server, req, meta, probabilities, stats) + entry = {'request_digest': request_digest(req.model_dump()), 'probabilities': probabilities, 'body': body, + 'stats': stats, 'graph_replays': replay_count, 'graphs': server.model.graphs.stats(), + 'runtime': {'dtype': server.model.dtype, 'head_dtype': str(next(server.model.head.parameters()).dtype), + 'attention': server.model.lm.config._attn_implementation, + 'fused_available': fused_available(), + 'remaining_lora_modules': sum(hasattr(m, 'lora_A') for m in server.model.lm.modules()), + 'temperature': server.model.head.temperature, 'device': torch.cuda.get_device_name(), + 'capability': torch.cuda.get_device_capability(), 'torch_cuda': torch.version.cuda}} + stream.write(json.dumps(entry, ensure_ascii=False, allow_nan=False)+'\n') + stream.flush() + return body + serve.Server._body = observed_body + sys.argv = ['kev.serve', '--run', str(args.adapter), '--device', 'cuda', '--host', '127.0.0.1', '--port', str(args.port)] + serve.main() + + +def boundary_supplement(args): + """Predeclared full-range Noul sweep after the first pool's coverage failure.""" + if args.out.exists(): + raise ValueError('preserve previous corpus') + fixture = module('kev_fixture', ROOT/'tests/kev/generate_processing_fixture.py') + api, model = fixture.official(args.reference) + tok = model.load_tokenizer(args.base) + cases = [] + for template in ('tickets', 'balls'): + for i in range(1001): + chance = i/10 + state = (f'A lottery has exactly {chance:.1f}% winning tickets. One ticket is drawn uniformly at random. Its outcome has not been revealed.' + if template == 'tickets' else + f'A bag contains red and blue balls. Exactly {chance:.1f}% of the balls are red. One ball is drawn uniformly at random without looking. Its color has not been revealed.') + instruction = 'The ticket wins.' if template == 'tickets' else 'The drawn ball is red.' + case = {'name': f'boundary-supplement-{template}-{i:04}', 'population': 'discovery', + 'source': f'synthetic_{template}_full_range', 'labels': {}, + 'request': {'model': 'kev-latest', 'state': state, 'questions': {'event': {'type': 'noul', 'instructions': instruction}}}} + cases.append(fixture.capture(case, api, model, tok)) + save(args.out, {'reference': REFERENCE, 'sources': {}, 'gates': GATES, 'counts': {'discovery': len(cases)}, 'cases': cases}) + print('frozen supplement', len(cases), 'sha256', digest(args.out), flush=True) + + +def reference(args): + http = module('kev_http', ROOT/'recipe/kev/validate_http.py') + args.out.mkdir(parents=True, exist_ok=False) + corpus = json.loads(args.corpus.read_text()) + assert corpus['gates'] == GATES + exporter = module('kev_exporter', ROOT/'recipe/kev/export.py') + exporter.verify_files(args.adapter, exporter.CHECKPOINT_FILES) + exporter.verify_files(args.adapter/'base', exporter.BASE_FILES) + port = http.port() + url = f'http://127.0.0.1:{port}' + env = {k: v for k, v in os.environ.items() if not k.startswith('KEV_')} + command = [sys.executable, str(Path(__file__).resolve()), 'official-server', '--reference', str(args.reference), + '--adapter', str(args.adapter), '--out', str(args.out/'observations.jsonl'), '--port', str(port)] + save(args.out/'provenance.json', {'command': command, 'reference': REFERENCE, 'corpus_sha256': digest(args.corpus), + 'harness_sha256': digest(__file__), 'gates': GATES, + 'source': subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=ROOT, text=True).strip(), + 'source_diff': subprocess.check_output(['git', 'diff', 'HEAD'], cwd=ROOT, text=True), + 'dependencies': {d.metadata['Name']: d.version for d in importlib.metadata.distributions()}}) + with (args.out/'server.log').open('w') as log, (args.out/'responses.jsonl').open('x') as responses: + process = subprocess.Popen(command, env=env, stdout=log, stderr=subprocess.STDOUT) + try: + deadline = time.monotonic()+600 + while True: + if process.poll() is not None: + raise RuntimeError(f'official server exited {process.returncode}; see server.log') + try: + status, models = http.request(url+'/v1/models') + if status == 200: + break + except OSError: + pass + if time.monotonic() > deadline: + raise TimeoutError('official readiness deadline') + time.sleep(0.5) + save(args.out/'models-before.json', models) + for i, case in enumerate(corpus['cases']): + status, body = http.request(url+'/v1/systemone', case['request']) + responses.write(json.dumps({'name': case['name'], 'status': status, 'body': body}, ensure_ascii=False)+'\n') + responses.flush() + if status != 200: + raise ValueError((case['name'], status, body)) + if i % 25 == 0: + print('official', i+1, '/', len(corpus['cases']), case['name'], flush=True) + save(args.out/'models-after.json', http.request(url+'/v1/models')[1]) + finally: + process.terminate() + try: + process.wait(timeout=30) + except subprocess.TimeoutExpired: + process.kill() + process.wait() + observations = read_lines(args.out/'observations.jsonl') + responses = read_lines(args.out/'responses.jsonl') + assert len(observations) == len(responses) == len(corpus['cases']) + fixture = module('kev_fixture', ROOT/'tests/kev/generate_processing_fixture.py') + api, _ = fixture.official(args.reference) + captured = [] + for c, obs, response in zip(corpus['cases'], observations, responses): + assert obs['body'] == response['body'] + assert obs['request_digest'] == request_digest(api.SystemOneRequest.model_validate(c['request']).model_dump()) + assert obs['runtime']['dtype'] == 'bfloat16' and obs['runtime']['head_dtype'] == 'torch.float32' + assert obs['runtime']['remaining_lora_modules'] == 0 + captured.append({'name': c['name'], **obs}) + save(args.out/'official-service.json', {'cases': captured}) + boundary = [] + for case, record in zip(corpus['cases'], captured): + for meta, p in zip(case['meta'], record['probabilities']): + if meta['type'] == 'choice' and len(p) > 1: + ranked = sorted(p, reverse=True) + distance, limit = ranked[0]-ranked[1], GATES['choice_margin'] + elif meta['type'] == 'noul': + distance, limit = abs(p[1]-0.5), GATES['noul_distance'] + else: + continue + if distance <= limit: + boundary.append({'case': case['name'], 'question': meta['id'], 'type': meta['type'], + 'source': case['source'], 'distance': distance, 'probabilities': p}) + save(args.out/'boundary-membership.json', {'selection': 'official outputs only, before native', 'items': boundary}) + print('official default capture complete', len(captured), flush=True) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest='command', required=True) + prep = sub.add_parser('prepare') + for flag in ('reference', 'base', 'out'): + prep.add_argument('--'+flag, type=Path, required=True) + supplement = sub.add_parser('boundary-supplement') + for flag in ('reference', 'base', 'out'): + supplement.add_argument('--'+flag, type=Path, required=True) + server = sub.add_parser('official-server') + for flag in ('reference', 'adapter', 'out'): + server.add_argument('--'+flag, type=Path, required=True) + server.add_argument('--port', type=int, required=True) + ref = sub.add_parser('reference') + for flag in ('reference', 'adapter', 'corpus', 'out'): + ref.add_argument('--'+flag, type=Path, required=True) + args = parser.parse_args() + {'prepare': prepare, 'boundary-supplement': boundary_supplement, + 'official-server': official_server, 'reference': reference}[args.command](args) + + +if __name__ == '__main__': + main() diff --git a/tests/kev/service_kernels.py b/tests/kev/service_kernels.py new file mode 100644 index 00000000..b99cd06a --- /dev/null +++ b/tests/kev/service_kernels.py @@ -0,0 +1,334 @@ +"""Reserved-GPU regression for Kev's optional Torch 2.8 arithmetic. + +Use the pinned reference requirements and gpu run. --legacy-library compares +unchanged Qwen entry points against a previously built library, bit for bit. +""" + +import argparse +import ctypes as C +import inspect +import json +from pathlib import Path + +import torch +from torch.nn.attention import SDPBackend, sdpa_kernel +from transformers.models.qwen3_5.modeling_qwen3_5 import ( + Qwen3_5RMSNormGated, + torch_chunk_gated_delta_rule, +) + +P, I, F = C.c_void_p, C.c_int, C.c_float + + +def bind(lib, name, signature, result=I): + fn = getattr(lib, name) + fn.argtypes, fn.restype = signature, result + return fn + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--library", type=Path, required=True) + parser.add_argument("--legacy-library", type=Path, required=True) + parser.add_argument("--out", type=Path, required=True) + args = parser.parse_args() + assert not args.out.exists(), "preserve prior evidence" + assert torch.__version__ == "2.8.0+cu128" + torch.set_grad_enabled(False) + torch.manual_seed(27) + torch.backends.cuda.matmul.allow_tf32 = False + lib, old = C.CDLL(str(args.library)), C.CDLL(str(args.legacy_library)) + stream = torch.cuda.current_stream().cuda_stream + ptr = lambda t: t.data_ptr() + rand = lambda shape: torch.randn(shape, device="cuda", dtype=torch.bfloat16) + records = [] + + def check(name, actual, expected, tolerance=0): + torch.cuda.synchronize() + diff = actual.float() - expected.float() + relative = float(diff.norm() / expected.float().norm().clamp_min(1e-30)) + records.append( + { + "name": name, + "different": int((actual != expected).sum()), + "max_abs": float(diff.abs().max()), + "relative_l2": relative, + } + ) + assert relative <= tolerance, records[-1] + + # Default entry points stay bit exact; the new profile follows Torch 2.8. + for rows in (1, 16, 80, 160): + x, delta, w = rand((rows, 2560)), rand((rows, 2560)), rand((2560,)) + for added in (False, True): + outputs = [] + for library, prefix in ( + (old, "reference"), + (lib, "reference"), + (lib, "torch28"), + ): + residual, out = x.clone(), torch.empty_like(x) + name = "add_rms_norm" if added else "rms_norm" + fn = bind( + library, + "cs1_" + prefix + "_" + name, + [P] * (4 if added else 3) + [I, I, F, P], + ) + values = ( + [ptr(residual), ptr(delta), ptr(w), ptr(out)] + if added + else [ptr(residual), ptr(w), ptr(out)] + ) + assert fn(*values, rows, 2560, 1e-6, stream) == 0 + outputs.append(out) + check(f"legacy-norm-{rows}-{added}", outputs[1], outputs[0]) + z = (x + delta).float() if added else x.float() + expected = ( + z + * torch.rsqrt(z.square().mean(-1, keepdim=True) + 1e-6) + * (1 + w.float()) + ).bfloat16() + check(f"torch28-norm-{rows}-{added}", outputs[2], expected) + z, gate, weight = rand((rows, 32, 128)), rand((rows, 32, 128)), rand((128,)) + outputs = [] + for library, prefix in ((old, ""), (lib, ""), (lib, "torch28_")): + out = torch.empty_like(z) + fn = bind( + library, + "cs1_" + prefix + "gated_rms_norm", + [P, P, I, P, P, I, I, I, F, P], + ) + assert ( + fn( + ptr(z), + ptr(gate), + 4096, + ptr(weight), + ptr(out), + rows, + 32, + 128, + 1e-6, + stream, + ) + == 0 + ) + outputs.append(out) + check(f"legacy-gated-norm-{rows}", outputs[1], outputs[0]) + norm = Qwen3_5RMSNormGated(128).cuda().bfloat16() + norm.weight.copy_(weight) + expected = norm(z, gate) + check(f"torch28-gated-norm-{rows}", outputs[2], expected) + + # The existing reference attention-prep ABI retains its arithmetic. + # JEMM uses the same norm entry points at width 5120; Kev only exercises 2560. + for rows in (1, 16, 80, 160): + x, delta, w = rand((rows, 5120)), rand((rows, 5120)), rand((5120,)) + for added in (False, True): + outputs = [] + for library in (old, lib): + residual, out = x.clone(), torch.empty_like(x) + fn = bind( + library, + "cs1_reference_" + ("add_rms_norm" if added else "rms_norm"), + [P] * (4 if added else 3) + [I, I, F, P], + ) + values = ( + [ptr(residual), ptr(delta), ptr(w), ptr(out)] + if added + else [ptr(residual), ptr(w), ptr(out)] + ) + assert fn(*values, rows, 5120, 1e-6, stream) == 0 + outputs.append((out, residual)) + for name, before, after in zip(("out", "residual"), outputs[0], outputs[1]): + check(f"legacy-norm-5120-{rows}-{added}-{name}", after, before) + + # The existing reference attention-prep ABI retains its arithmetic. + for rows in (1, 32, 80): + packed = rand((rows, 2 * 16 * 256 + 4 * 256)) + qw, kw = rand((256,)), rand((256,)) + cos, sin = rand((rows, 32)), rand((rows, 32)) + outputs = [] + for library in (old, lib): + q, gate, k = ( + rand((rows, 16, 256)), + rand((rows, 16, 256)), + rand((rows, 4, 256)), + ) + fn = bind( + library, + "cs1_reference_attn_prep", + [P, P, I] + [P] * 7 + [I] * 5 + [F, P], + ) + assert ( + fn( + ptr(packed), + ptr(packed) + 2 * 16 * 256 * 2, + packed.shape[1], + ptr(qw), + ptr(kw), + ptr(cos), + ptr(sin), + ptr(q), + ptr(gate), + ptr(k), + rows, + 16, + 4, + 256, + 32, + 1e-6, + stream, + ) + == 0 + ) + outputs.append((q, gate, k)) + for name, a, b in zip(("q", "gate", "k"), outputs[0], outputs[1]): + check(f"legacy-attn-prep-{rows}-{name}", b, a) + + # FP32 GDN: chunk edges, nonzero initial state, final state and unrelated replay. + reference = inspect.unwrap(torch_chunk_gated_delta_rule) + for t in (1, 31, 32, 33, 63, 64, 65, 129, 233): + q, k, v = rand((1, t, 16, 128)), rand((1, t, 16, 128)), rand((1, t, 32, 128)) + g = -torch.rand(1, t, 32, device="cuda") * 0.25 + beta = torch.rand(1, t, 32, device="cuda", dtype=torch.bfloat16) + initial = torch.randn(1, 32, 128, 128, device="cuda") * 0.01 + outputs = [] + for library, profile in ((old, ""), (lib, ""), (lib, "_torch28")): + create = bind(library, "cs1_gdn_fp32_create" + profile, [C.c_size_t], P) + destroy = bind(library, "cs1_gdn_fp32_destroy", [P], None) + size = bind(library, "cs1_gdn_fp32_workspace_floats", [I, I], C.c_size_t) + handle = create(32 << 20) + assert handle + try: + work = torch.empty(size(t, 32), device="cuda") + out = torch.empty_like(v) + fn = bind(library, "cs1_gdn_prefill_fp32", [P] * 8 + [I, I, I, F, P]) + call = [ + handle, + ptr(q), + ptr(k), + ptr(v), + ptr(g), + ptr(beta), + ptr(out), + ptr(work), + t, + 32, + 16, + 128**-0.5, + stream, + ] + assert fn(*call) == 0 + outputs.append(out.clone()) + if profile: + expected, _ = reference( + q.repeat_interleave(2, 2), + k.repeat_interleave(2, 2), + v, + g, + beta, + use_qk_l2norm_in_kernel=True, + ) + check(f"torch28-gdn-{t}", out, expected, 1e-4) + state = torch.empty_like(initial) + fnx = bind( + library, + "cs1_gdn_prefill_fp32_x", + [P] * 8 + [I, I, I, F, P, P, P], + ) + xcall = call[:-1] + [ptr(initial), ptr(state), stream] + assert fnx(*xcall) == 0 + expected, expected_state = reference( + q.repeat_interleave(2, 2), + k.repeat_interleave(2, 2), + v, + g, + beta, + initial_state=initial, + output_final_state=True, + use_qk_l2norm_in_kernel=True, + ) + check(f"torch28-gdn-seeded-{t}", out, expected, 1e-4) + check(f"torch28-gdn-final-{t}", state, expected_state, 1e-4) + work.fill_(float("nan")) + assert fn(*call) == 0 + check(f"torch28-gdn-reset-{t}", out, outputs[-1]) + finally: + torch.cuda.synchronize() + destroy(handle) + check(f"legacy-gdn-{t}", outputs[1], outputs[0]) + + # Non-aligned key extents regress the 255-level Score's bias-stride failure. + size = bind(lib, "cs1_service_attention_bytes", [I, I, I], C.c_size_t) + attention = bind( + lib, "cs1_service_attention", [P, P, P, I, P, P] + [I] * 7 + [P, P] + ) + for nq, prefix, left, real in [ + (1, 16, 15, 1), + (3, 7, 0, 2), + (32, 32, 3, 27), + (80, 64, 12, 69), + (1679, 7, 0, 1679), + ]: + nk = prefix + nq + h, hk = 16, 4 + q, k, v, gate = ( + rand((nq, h, 256)), + rand((nk, hk, 256)), + rand((nk, hk, 256)), + rand((nq, h, 256)), + ) + out = torch.empty_like(q) + work = torch.empty(size(nq, nk, h), device="cuda", dtype=torch.uint8) + assert ( + attention( + ptr(q), + ptr(k), + ptr(v), + hk * 256, + ptr(gate), + ptr(out), + nq, + nk, + h, + hk, + prefix, + left, + real, + ptr(work), + stream, + ) + == 0 + ) + qi = torch.arange(nq, device="cuda")[:, None] + ki = torch.arange(nk, device="cuda")[None, :] + allow = ( + (ki >= left) & (ki <= prefix + qi) & ((ki < prefix) | (ki - prefix < real)) + ) | (ki == prefix + qi) + # Torch also requires a multiple-of-8 bias stride for efficient attention. + mask = torch.full( + (nq, (nk + 7) // 8 * 8), + torch.finfo(torch.bfloat16).min, + device="cuda", + dtype=torch.bfloat16, + ) + mask[:, :nk].masked_fill_(allow, 0) + with sdpa_kernel(SDPBackend.EFFICIENT_ATTENTION): + expected = torch.nn.functional.scaled_dot_product_attention( + q.transpose(0, 1)[None], + k.repeat_interleave(h // hk, 1).transpose(0, 1)[None], + v.repeat_interleave(h // hk, 1).transpose(0, 1)[None], + attn_mask=mask[:, :nk][None, None], + )[0].transpose(0, 1) + check(f"service-attention-{nq}-{nk}", out, expected * gate.sigmoid()) + args.out.write_text( + json.dumps( + {"torch": torch.__version__, "records": records, "passed": True}, indent=2 + ) + + "\n" + ) + + +if __name__ == "__main__": + main() diff --git a/tests/kev/test_export.py b/tests/kev/test_export.py new file mode 100644 index 00000000..dd82e839 --- /dev/null +++ b/tests/kev/test_export.py @@ -0,0 +1,273 @@ +"""CPU-only exporter gates and LoRA arithmetic; no checkpoint download.""" + +import hashlib +import importlib.util +import json +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +import torch +from safetensors.torch import load_file, save_file + +SPEC = importlib.util.spec_from_file_location( + "kev_export", Path(__file__).resolve().parents[2] / "recipe/kev/export.py" +) +exporter = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(exporter) + + +class ExportPathTests(unittest.TestCase): + """Real file I/O and export control flow with tiny synthetic base tensors.""" + + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.base, self.checkpoint = self.root / "base", self.root / "checkpoint" + self.base.mkdir() + self.checkpoint.mkdir() + self.out = self.root / "export" + self.shards = [name for name in exporter.BASE_FILES if name.endswith(".safetensors")] + self.weights = [{}, {}] + self.expected = {} + adapter = {} + self.targets = sorted(exporter.expected_targets()) + for i, target in enumerate(self.targets): + weight = (torch.arange(6).reshape(2, 3) + i).bfloat16() + self.weights[i % 2][target] = weight + # Exact powers of two: sum of 16 * (1/4 * 1/4), times alpha/r=2. + self.expected[target] = (weight.float() + 2).bfloat16() + prefix = target.removeprefix("model.language_model.").removesuffix(".weight") + adapter[f"base_model.model.{prefix}.lora_A.weight"] = torch.full((16, 3), 0.25) + adapter[f"base_model.model.{prefix}.lora_B.weight"] = torch.full((2, 16), 0.25) + plain = "model.language_model.norm.weight" + self.weights[1][plain] = torch.tensor([1.001, 2.003, 3.007]) + self.expected[plain] = self.weights[1][plain].bfloat16() + self.weights[0]["model.visual.weight"] = torch.ones(2, 3) + self.weights[1]["mtp.weight"] = torch.ones(2, 3) + for name in ("config.json", "tokenizer_config.json"): + (self.base / name).write_text("{}\n") + (self.base / "tokenizer.json").write_text(json.dumps({"added_tokens": [ + {"content": k, "id": v} for k, v in exporter.SPECIAL.items() + ]})) + (self.base / "model.safetensors.index.json").write_text(json.dumps({"weight_map": { + key: name for name, weights in zip(self.shards, self.weights) for key in weights + }})) + (self.checkpoint / "adapter_config.json").write_text("{}\n") + save_file(adapter, self.checkpoint / "adapter_model.safetensors") + self.head = { + "q.weight": torch.full((256, 2560), 0.1234567), + "k.weight": torch.full((256, 2560), -0.7654321), + "q.bias": torch.arange(256).float() / 31, + "k.bias": -torch.arange(256).float() / 43, + } + self.temperature = 2.406050072164233 + torch.save(dict( + base=exporter.BASE_ID, base_revision=exporter.BASE_REVISION, + lora=16, head_dim=256, option_isolation=False, + special_embeddings=False, weights_dtype="fp32", + temperature=self.temperature, head=self.head, + ), self.checkpoint / "head.pt") + + @staticmethod + def hashes(directory): + return {p.name: hashlib.sha256(p.read_bytes()).hexdigest() for p in directory.iterdir()} + + def run_export(self): + for name, weights in zip(self.shards, self.weights): + save_file(weights, self.base / name) + # Substitute only fixture hashes; keep target, dtype, shape and tokenizer + # validators, restricted pickle loading, merge, writers and rename real. + with patch.object(exporter, "BASE_FILES", self.hashes(self.base)), patch.object( + exporter, "CHECKPOINT_FILES", self.hashes(self.checkpoint) + ): + return exporter.export(self.base, self.checkpoint, self.out, merge_device="cpu") + + def test_complete_two_shard_export(self): + with patch.object(exporter.os, "rename", wraps=exporter.os.rename) as rename: + manifest = self.run_export() + rename.assert_called_once() + staging, destination = rename.call_args.args + self.assertEqual(destination, self.out) + self.assertEqual(staging.parent, self.out.parent) + self.assertFalse(staging.exists()) + self.assertEqual(manifest, json.loads((self.out / "kev_export.json").read_text())) + self.assertEqual(manifest["merged_targets"], self.targets) + self.assertEqual(manifest["temperature"], self.temperature) + self.assertEqual(manifest["special_tokens"], exporter.SPECIAL) + self.assertEqual(manifest["inputs"], { + "base": self.hashes(self.base), "checkpoint": self.hashes(self.checkpoint) + }) + self.assertEqual(manifest["files"], { + k: v for k, v in self.hashes(self.out).items() if k != "kev_export.json" + }) + self.assertEqual(set(manifest["files"]), { + *self.shards, "config.json", "tokenizer.json", "tokenizer_config.json", + "head.safetensors", "model.safetensors.index.json", + }) + weight_map = json.loads((self.out / "model.safetensors.index.json").read_text())["weight_map"] + self.assertEqual(weight_map, { + key: name for name, weights in zip(self.shards, self.weights) + for key in weights if key in self.expected + }) + for name in self.shards: + tensors = load_file(self.out / name) + self.assertEqual(set(tensors), {k for k, shard in weight_map.items() if shard == name}) + for key, tensor in tensors.items(): + self.assertEqual(tensor.dtype, torch.bfloat16) + self.assertTrue(torch.equal(tensor, self.expected[key]), key) + head = load_file(self.out / "head.safetensors") + self.assertEqual(set(head), set(self.head)) + for key, tensor in head.items(): + self.assertEqual(tensor.dtype, torch.float32) + self.assertTrue(torch.equal(tensor, self.head[key]), key) + for name in ("config.json", "tokenizer.json", "tokenizer_config.json"): + self.assertEqual((self.out / name).read_bytes(), (self.base / name).read_bytes()) + + def test_late_write_failure_removes_staging_without_publishing(self): + calls = [] + + def fail_second_shard(weights, path): + calls.append(path) + if len(calls) == 2: + self.assertTrue(calls[0].is_file()) + raise OSError("injected second-shard write failure") + save_file(weights, path) + + with patch.object(exporter, "save_file", side_effect=fail_second_shard): + with self.assertRaisesRegex(OSError, "second-shard"): + self.run_export() + self.assertEqual(len(calls), 2) + self.assertEqual(sorted(p.name for p in self.root.iterdir()), ["base", "checkpoint"]) + + +class ExportTests(unittest.TestCase): + def test_all_layers_and_projection_families_are_required(self): + targets = exporter.expected_targets() + self.assertEqual(len(targets), 248) + for name, count in (("gate_proj", 32), ("up_proj", 32), ("down_proj", 32), + ("q_proj", 8), ("k_proj", 8), ("v_proj", 8), ("o_proj", 8), + ("in_proj_qkv", 24), ("in_proj_a", 24), ("in_proj_z", 24), + ("in_proj_b", 24), ("out_proj", 24)): + self.assertEqual(sum(t.endswith(f".{name}.weight") for t in targets), count) + exporter.validate_targets(targets) + missing = "model.language_model.layers.0.linear_attn.in_proj_b.weight" + # The same count with a different target is also invalid. + with self.assertRaisesRegex(ValueError, "target mismatch"): + exporter.validate_targets((targets - {missing}) | {missing.replace(".0.", ".3.")}) + + def test_merge_includes_each_gdn_projection_with_one_rounding(self): + # Includes all five GDN projections omitted by a Cua-S1-only target list. + for module in ( + "in_proj_qkv", + "in_proj_a", + "in_proj_z", + "in_proj_b", + "out_proj", + ): + name = f"base_model.model.layers.0.linear_attn.{module}" + a = torch.arange(48, dtype=torch.float32).reshape(16, 3) / 31 + b = torch.arange(32, dtype=torch.float32).reshape(2, 16) / 43 + pair = exporter.adapter_pairs( + {f"{name}.lora_A.weight": a, f"{name}.lora_B.weight": b} + ) + target = f"model.language_model.layers.0.linear_attn.{module}.weight" + self.assertEqual(set(pair), {target}) + weight = torch.tensor( + [[1.1, -2.2, 3.3], [4.4, 0.1, -0.7]], dtype=torch.bfloat16 + ) + actual = exporter.merge_weight(weight, a, b) + # Independent elementwise FP64 sums; final BF16 rounding is exact here. + expected = weight.double() + 2 * torch.stack( + [ + torch.stack( + [ + sum(b[i, k].double() * a[k, j].double() for k in range(16)) + for j in range(3) + ] + ) + for i in range(2) + ] + ) + self.assertTrue(torch.equal(actual, expected.bfloat16())) + + def test_missing_unknown_nonfinite_and_shape_mismatches_fail(self): + key = "base_model.model.layers.0.mlp.down_proj.lora_A.weight" + for tensors in ( + {key: torch.ones(16, 2)}, + {"unknown": torch.ones(2, 2)}, + {key: torch.full((16, 2), float("nan"))}, + {key: torch.ones(16, 2).half()}, + ): + with self.assertRaises(ValueError): + exporter.adapter_pairs(tensors) + with self.assertRaisesRegex(ValueError, "shape"): + exporter.merge_weight(torch.ones(2, 3), torch.ones(4, 3), torch.ones(2, 4)) + + def test_wrong_hash_precedes_pickle_loading_and_output_creation(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + (root / "config.json").write_text("wrong revision") + with patch.object( + exporter.torch, + "load", + side_effect=AssertionError("must not deserialize"), + ): + with self.assertRaisesRegex(ValueError, "pinned release"): + exporter.export(root, root, root / "out") + self.assertFalse((root / "out").exists()) + self.assertEqual(sorted(p.name for p in root.iterdir()), ["config.json"]) + + def test_never_overwrites_existing_output(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + with self.assertRaisesRegex(ValueError, "already exists"): + exporter.export(root, root, root) + + def test_delimiters_use_base_ids(self): + with tempfile.TemporaryDirectory() as temp: + path = Path(temp) / "tokenizer.json" + tokens = [{"content": k, "id": v} for k, v in exporter.SPECIAL.items()] + path.write_text(json.dumps({"added_tokens": tokens})) + exporter.validate_tokenizer(path) + tokens[0]["id"] = 151659 + path.write_text(json.dumps({"added_tokens": tokens})) + with self.assertRaisesRegex(ValueError, "delimiter"): + exporter.validate_tokenizer(path) + + def test_head_metadata_and_calibration_are_required(self): + meta = dict( + base=exporter.BASE_ID, + base_revision=exporter.BASE_REVISION, + lora=16, + head_dim=256, + option_isolation=False, + special_embeddings=False, + weights_dtype="fp32", + temperature=2.406050072164233, + head={ + "q.weight": torch.zeros(256, 2560), + "k.weight": torch.ones(256, 2560), + "q.bias": torch.zeros(256), + "k.bias": torch.ones(256), + }, + ) + _, temperature = exporter.pointer_head(meta) + self.assertEqual(temperature, meta["temperature"]) + for change in ( + dict(temperature=0), + dict(temperature=float("nan")), + dict(special_embeddings=True), + dict(base_revision="wrong"), + dict(head={}), + dict(weights_dtype="bf16"), + ): + with self.assertRaises(ValueError): + exporter.pointer_head({**meta, **change}) + + +if __name__ == "__main__": + torch.set_num_threads(2) + unittest.main() diff --git a/tests/kev/test_service_acceptance.py b/tests/kev/test_service_acceptance.py new file mode 100644 index 00000000..7d1e4a41 --- /dev/null +++ b/tests/kev/test_service_acceptance.py @@ -0,0 +1,84 @@ +"""Regression checks for acceptance failures that must never be hidden by rounding.""" +import unittest + +from boundary_discovery import previous_keys +from compare_service import ( + compare, + distribution, + labeled_metrics, + question, + supplements, +) +from service_acceptance import GATES + + +class ServiceAcceptanceTests(unittest.TestCase): + def test_supplement_runs_cannot_be_silently_truncated(self): + for paths in ((['a'], [], []), (['a', 'b'], ['a'], ['a', 'b'])): + with self.assertRaisesRegex(ValueError, 'each run'): + supplements(*paths) + self.assertEqual(supplements(['a', 'b'], ['c', 'd'], ['e', 'f']), + [('a', 'c', 'e'), ('b', 'd', 'f')]) + + def test_discovery_deduplicates_source_questions_and_actual_tokens(self): + def case(name, ids, kind='noul', **extra): + return {'name': name, 'meta': [{'id': 'q', 'type': kind}], + 'rows': [{'ids': ids}], **extra} + cases = [case('suite-one:record', [1, 2]), case('suite-two:record', [1, 2]), + case('synthetic', [3, 4]), case('choice', [5, 6], 'choice'), + case('new-discovery', [7, 8], origin={'record_id': 'another', 'question_id': 'q'})] + identities, tokens = previous_keys([{'gates': GATES, 'cases': cases}]) + self.assertEqual(identities, {('record', 'q'), ('another', 'q')}) + self.assertEqual(tokens, {(1, 2), (3, 4), (7, 8)}) + with self.assertRaisesRegex(ValueError, 'gates changed'): + previous_keys([{'gates': {**GATES, 'noul_distance': .02}, 'cases': cases}]) + + def test_score_can_fail_while_every_probability_passes(self): + p, n = [0.0]*255, [0.0]*255 + p[0] = 1.0 + n[0], n[-1] = 0.9995, 0.0005 + result = question({'type': 'score', 'keys': list(range(255))}, p, n, {'score': 0}, {'score': 0.127}) + self.assertEqual(result['failures'], ['score']) + self.assertLess(result['probability_max_abs'], GATES['probability_max_abs']) + + def test_raw_score_failure_not_hidden_by_wire(self): + result = question({'type': 'score', 'keys': ['0', '1']}, [1., 0.], [.89999, .10001], {'score': 0}, {'score': .1}) + self.assertIn('score', result['failures']) + + def test_wire_score_gate_is_independent(self): + result = question({'type': 'score', 'keys': ['0', '1']}, [.5, .5], [.5, .5], {'score': .5}, {'score': .6001}) + self.assertEqual(result['failures'], ['score']) + + def test_probability_gate_not_rounded_away(self): + result = question({'type': 'noul', 'keys': ['false', 'true']}, [.8, .2], [.78999, .21001], {'noul': .2}, {'noul': .21}) + self.assertEqual(result['failures'], ['probability']) + + def test_boundary_flip_retained(self): + result = question({'type': 'noul', 'keys': ['false', 'true']}, [.501, .499], [.499, .501], {'noul': .499}, {'noul': .501}) + self.assertTrue(result['near_boundary'] and result['raw_flip'] and result['wire_flip']) + self.assertFalse(result['failures']) + + def test_confident_flip_fails(self): + result = question({'type': 'choice', 'keys': ['a', 'b']}, [.52, .48], [.49, .51], {'choice': 'a'}, {'choice': 'b'}) + self.assertIn('decision', result['failures']) + + def test_nonfinite_and_bad_distributions_rejected(self): + for p in ([float('nan'), 0], [float('inf'), 0], [-.1, 1.1], [.6, .6], [1]): + self.assertFalse(distribution(p, 2)) + + def test_missing_duplicate_reordered_cases_rejected(self): + corpus = {'cases': [{'name': 'a'}, {'name': 'b'}], 'gates': GATES} + for rows in ([], [{'name': 'b'}, {'name': 'a'}], [{'name': 'a'}, {'name': 'a'}]): + with self.assertRaisesRegex(ValueError, 'cases'): + compare(corpus, rows, rows) + + def test_quality_uses_labels_and_normalizes_score_range(self): + metrics = labeled_metrics('score', [.25, .5, .25], 2) + self.assertEqual(metrics['accuracy'], 0) + self.assertEqual(metrics['score_mae'], 1) + self.assertEqual(metrics['normalized_score_mae'], .5) + self.assertEqual(metrics['brier'], .875) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/kev/test_validate.py b/tests/kev/test_validate.py new file mode 100644 index 00000000..fcbe895c --- /dev/null +++ b/tests/kev/test_validate.py @@ -0,0 +1,67 @@ +"""Regression checks for the fixed acceptance gates; synthetic data, no inference.""" +import contextlib +import copy +import importlib.util +import io +import json +import tempfile +import unittest +from pathlib import Path +from types import SimpleNamespace + +ROOT = Path(__file__).resolve().parents[2] +SPEC = importlib.util.spec_from_file_location('kev_validate', ROOT / 'recipe/kev/validate.py') +validator = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(validator) + + +class AcceptanceTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.out = Path(self.temp.name) + self.corpus = json.loads((ROOT / 'tests/kev/data/processing.json').read_text())['cases'] + cases = [{'name': c['name'], 'probabilities': c['probabilities'], + 'body': {'usage': {'input_tokens': c['input_tokens']}}} for c in self.corpus] + self.data = {kind: {'cases': copy.deepcopy(cases)} for kind in ('official', 'fp32-merged', 'merged', 'native')} + + def run_compare(self): + for kind, value in self.data.items(): + value = copy.deepcopy(value) + if kind == 'native': + value['cases'] += [copy.deepcopy(value['cases'][8]) for _ in range(3)] + (self.out / f'{kind}.json').write_text(json.dumps(value)) + with contextlib.redirect_stdout(io.StringIO()): + return validator.compare(SimpleNamespace(out=self.out)) + + def test_probability_gate_is_not_widened(self): + self.data['native']['cases'][1]['probabilities'][0] = [1/3+0.0101, 2/3-0.0101] + self.assertEqual(self.run_compare(), 1) + report = json.loads((self.out / 'comparison.json').read_text()) + failures = report['comparisons']['official-to-native']['failures'] + self.assertTrue(any(f['gate'] == 'probability' for f in failures)) + + def test_near_boundary_flips_are_reported_without_failing_margin_gate(self): + for value in self.data.values(): + value['cases'][1]['probabilities'][0] = [0.499, 0.501] + value['cases'][8]['probabilities'][1] = [0.501, 0.499] + self.data['native']['cases'][1]['probabilities'][0] = [0.501, 0.499] + self.data['native']['cases'][8]['probabilities'][1] = [0.499, 0.501] + self.assertEqual(self.run_compare(), 0) + report = json.loads((self.out / 'comparison.json').read_text()) + boundary = report['comparisons']['official-to-native']['near_boundary_disagreements'] + self.assertEqual({b['gate'] for b in boundary}, {'choice', 'noul'}) + + def test_nonfinite_cannot_pass_comparison(self): + self.data['native']['cases'][1]['probabilities'][0][0] = float('nan') + with self.assertRaisesRegex(AssertionError, 'invalid probabilities'): + self.run_compare() + + def test_missing_case_cannot_pass_comparison(self): + self.data['official']['cases'].pop() + with self.assertRaisesRegex(AssertionError, 'incomplete corpus'): + self.run_compare() + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/qwen3_5/model.rs b/tests/qwen3_5/model.rs index e82c1958..5eac8f67 100644 --- a/tests/qwen3_5/model.rs +++ b/tests/qwen3_5/model.rs @@ -276,9 +276,23 @@ fn cpu_fixed_and_shared_refuse_other_profiles_before_device_work() { }; let refuse = |model: &mut Model, profile: &str| { let before = unsafe { calls() }; + // These public cache entry points must reject incompatible profiles + // before touching even a prefix's device storage. + let mut prefix = PrefixState { + owner: Arc::clone(&model.prefix_owner), + initialized: true, + len: 64, + cap: 64, + attn_kv: vec![], + gdn_state: vec![], + conv_tail: vec![], + bytes: 0, + }; for error in [ model.forward_fixed(&[1, 2, 3]).unwrap_err(), model.forward_shared(&prompts).unwrap_err(), + model.capture_fixed(&[1; 64], &mut prefix).unwrap_err(), + model.continue_fixed(&[2, 3], &prefix).unwrap_err(), model .forward_multimodal_readout(&input, &[2], &[(2, 4)]) .map(drop) @@ -307,6 +321,11 @@ fn cpu_fixed_and_shared_refuse_other_profiles_before_device_work() { model.fp32_gdn = None; model.reference = Some(cuda::api().language_reference().unwrap()); refuse(&mut model, "reference numerics"); + model.fp32_gdn = Some(cuda::api().torch28_gdn().unwrap()); + model.service_ops = Some(cuda::api().service_readout().unwrap()); + refuse(&mut model, "Kev service numerics"); + model.service_ops = None; + model.fp32_gdn = None; model.reference = None; model.lora = adapter; drop(model);