diff --git a/README.md b/README.md index baea3d3..1c7cf23 100644 --- a/README.md +++ b/README.md @@ -777,6 +777,10 @@ request for a model that is not available gets a `503` saying so. Add this repo as a `:git/url` dep, run `jolt kernels` and `jolt prepare` for the native library and `data/`, then: +jolt runs no prep step for a dependency, so `jolt kernels` runs in lev's +own checkout, the gitlib that `jolt path` names. A Windows build of the +program links no kernels, so the encoders are unavailable there. + ```clojure (require '[lev.agent :as ag] '[lev.server :as server] '[lev.config :as cfg]) (def agent (ag/load-agent (cfg/setting (cfg/context {}) "--data" "LEV_DATA" :data "data"))) @@ -1008,7 +1012,10 @@ entries, and the build task branches on OS. - **HTTP**: `jolt-lang/ring-chez-adapter` serves the API, and `org.clojars.askonomm/ruuter` from Clojars dispatches its routes. - **BLAS**: `cblas_sgemm` from the Accelerate framework on mac, OpenBLAS - on linux. + on linux, else the reference `libblas.so.3` (about 10x slower). It is an + optional native, so a program that uses lev as a library still starts + without one; the encoders then refuse to load, and + `lev.tensors/available?` says whether they can. On linux, install the OpenBLAS runtime package. diff --git a/deps.edn b/deps.edn index 1fcd916..2239213 100644 --- a/deps.edn +++ b/deps.edn @@ -23,11 +23,13 @@ ;; kernels: built from native/lev_kernels.c by the `kernels` task. ;; blas: Accelerate framework on mac, OpenBLAS elsewhere. ;; `jolt build` links the kernels statically from the archive so the binary - ;; is self-contained; run/repl/test load the shared object. + ;; is self-contained; run/repl/test load the shared object. By platform, so + ;; a Windows build of a program that uses lev as a library links none. :jolt/native [{:name "lev_kernels" :darwin ["native/liblev_kernels.dylib"] :linux ["native/liblev_kernels.so"] - :static {:archive "native/liblev_kernels.a"}} + :static {:darwin {:archive "native/liblev_kernels.a"} + :linux {:archive "native/liblev_kernels.a"}}} ;; llama.cpp behind native/lev_llm.c, built by `jolt llama` ;; (optional at runtime: without it only the thinker is missing) {:name "lev_llm" @@ -75,10 +77,16 @@ :linux [] :static {:darwin {:lib "MetalKit" :libdir "native/frameworks"}}} ;; on mac the static llm archive (ggml-cpu) also needs Accelerate - ;; at link time, through its stub like the frameworks above + ;; at link time, through its stub like the frameworks above. + ;; Optional, so a program that uses lev as a library starts on a + ;; host without it; the encoders then refuse to load + ;; (lev.tensors/available?). On linux libblas.so.3 is the + ;; fallback: the reference CBLAS, ~10x slower than OpenBLAS, or + ;; OpenBLAS itself where the alternatives point it there. {:name "blas" + :optional true :darwin ["/System/Library/Frameworks/Accelerate.framework/Accelerate"] - :linux ["libopenblas.so.0" "libopenblas.so"] + :linux ["libopenblas.so.0" "libopenblas.so" "libblas.so.3"] :static {:darwin {:lib "Accelerate" :libdir "native/frameworks"}}}] :tasks diff --git a/src/lev/tensors.clj b/src/lev/tensors.clj index c48fb34..d10f9dc 100644 --- a/src/lev/tensors.clj +++ b/src/lev/tensors.clj @@ -75,6 +75,12 @@ (set-threads* (int (max 1 n))) n) +(defn available? + "Are the kernels and a BLAS loaded? Both are declared natives, BLAS an + optional one: without them nothing here runs." + [] + (boolean (and (ffi/find-symbol "lev_set_threads") (ffi/find-symbol "cblas_sgemm")))) + (def ^:private sgemm-address "cblas_sgemm as the kernel library reaches it: the same BLAS entry the binding above calls, found once through the declared natives." diff --git a/src/lev/tinydecide.clj b/src/lev/tinydecide.clj index e8f7409..c609806 100644 --- a/src/lev/tinydecide.clj +++ b/src/lev/tinydecide.clj @@ -75,6 +75,8 @@ [dir] (when-not (release? dir) (unavailable dir (str dir " has no meta.json and model.bin (hf download TheREZOR/TinyDecide)"))) + (when-not (t/available?) + (unavailable dir "no kernels or BLAS loaded (jolt kernels; OpenBLAS on linux)")) (let [meta (json/read-str (slurp (str dir "/meta.json"))) cfg (get meta "cfg") tok (get meta "tokenizer")]