refactor(web): unbundle lexical model worker from compiled artifact

android/.gitignore

@@ -37,6 +37,8 @@ KMEA/**/assets/keymanandroid.js
 KMEA/**/assets/keyman.js.map
 KMEA/**/assets/keymanweb-webview.js
 KMEA/**/assets/keymanweb-webview.js.map
+KMEA/**/assets/worker-thread.js
+KMEA/**/assets/worker-thread.js.map
 KMEA/**/assets/sentry.min.js
 KMEA/**/assets/keyman-sentry.js
 KMEA/**/assets/kmwosk.css

android/KMEA/app/src/main/java/com/keyman/engine/KMManager.java

@@ -375,6 +375,7 @@ public int toInt() {
   // Keyman files
   protected static final String KMFilename_KeyboardHtml = "keyboard.html";
   protected static final String KMFilename_JSEngine = "keymanweb-webview.js";
+  protected static final String KMFilename_LMWorker = "worker-thread.js";
   protected static final String KMFilename_JSSentry = "keyman-sentry.js";
   protected static final String KMFilename_AndroidHost = "android-host.js";
   protected static final String KMFilename_KmwCss = "kmwosk.css";
@@ -1073,10 +1074,12 @@ private static void copyAssets(Context context) {
     AssetManager assetManager = context.getAssets();
 
     try {
+      // TODO: these files should be in a constant array, not enumerated separately here (see also iOS)
       // Copy KMW files
       copyAsset(context, KMFilename_KeyboardHtml, "", true);
 
       copyAsset(context, KMFilename_JSEngine, "", true);
+      copyAsset(context, KMFilename_LMWorker, "", true);
       copyAsset(context, KMFilename_JSSentry, "", true);
       copyAsset(context, KMFilename_AndroidHost, "", true);
       copyAsset(context, KMFilename_KmwCss, "", true);

android/KMEA/build.sh

@@ -55,6 +55,8 @@ do_build() {
   builder_echo "Copying Keyman Web artifacts"
   cp "$KEYMAN_ROOT/web/build/app/webview/$BUILDER_CONFIGURATION/keymanweb-webview.js" "$ENGINE_ASSETS/"
   cp "$KEYMAN_ROOT/web/build/app/webview/$BUILDER_CONFIGURATION/keymanweb-webview.js.map" "$ENGINE_ASSETS/"
+  cp "$KEYMAN_ROOT/web/build/app/webview/$BUILDER_CONFIGURATION/worker-thread.js" "$ENGINE_ASSETS/"
+  cp "$KEYMAN_ROOT/web/build/app/webview/$BUILDER_CONFIGURATION/worker-thread.js.map" "$ENGINE_ASSETS/"
   cp "$KEYMAN_ROOT/web/build/app/resources/osk/ajax-loader.gif" "$ENGINE_ASSETS/"
   cp "$KEYMAN_ROOT/web/build/app/resources/osk/kmwosk.css" "$ENGINE_ASSETS/"
   cp "$KEYMAN_ROOT/web/build/app/resources/osk/globe-hint.css" "$ENGINE_ASSETS/"

common/web/sentry-manager/src/index.ts

@@ -18,6 +18,7 @@ export class KeymanSentryManager {
   static STANDARD_ALIASABLE_FILES = {
     'keymanweb.js':             'keymanweb.js',
     'keymanweb-webview.js':     'keymanweb-webview.js',
+    'worker-thread.js':         'worker-thread.js',
     'kmwuibutton.js':           'kmwuibutton.js',
     'kmwuifloat.js':            'kmwuifloat.js',
     'kmwuitoggle.js':           'kmwuitoggle.js',

common/web/types/src/lexical-model-types.ts

@@ -259,8 +259,8 @@ export interface Transform {
    * transition event - say, the resulting effects of neighbor keys that may
    * have been missed due to "fat fingering".
    *
-   * Also note that the Transform reference cannot be preserved across WebWorker
-   * boundaries, but this ID may.
+   * Also note that the Transform reference cannot be preserved across
+   * WebPredictiveTextWorker boundaries, but this ID may.
    *
    * This is *separate* from any LMLayer-internal identification values.
    */

ios/.gitignore

@@ -18,6 +18,8 @@ engine/KMEI/KeymanEngine/resources/Keyman.bundle/Contents/Resources/kmwosk.css
 engine/KMEI/KeymanEngine/resources/Keyman.bundle/Contents/Resources/keyman.js.map
 engine/KMEI/KeymanEngine/resources/Keyman.bundle/Contents/Resources/keymanweb-webview.js
 engine/KMEI/KeymanEngine/resources/Keyman.bundle/Contents/Resources/keymanweb-webview.js.map
+engine/KMEI/KeymanEngine/resources/Keyman.bundle/Contents/Resources/worker-thread.js
+engine/KMEI/KeymanEngine/resources/Keyman.bundle/Contents/Resources/worker-thread.js.map
 engine/KMEI/KeymanEngine/resources/Keyman.bundle/Contents/Resources/keyman-sentry.js
 engine/KMEI/KeymanEngine/resources/Keyman.bundle/Contents/Resources/sentry.min.js
 Keyman 2*

ios/engine/KMEI/KeymanEngine/Classes/Resource Management/Storage.swift

@@ -225,6 +225,7 @@ class Storage {
 // MARK: - Copying
 extension Storage {
   func copyKMWFiles(from bundle: Bundle) throws {
+    // TODO: these should be in a constant array, not enumerated individually here (see also Android)
     try Storage.copy(from: bundle,
                      resourceName: Resources.kmwFilename,
                      dstDir: baseDir,
@@ -237,6 +238,18 @@ extension Storage {
                      resourceName: "keymanweb-webview.js",
                      dstDir: baseDir,
                      excludeFromBackup: true)
+    try Storage.copy(from: bundle,
+                     resourceName: "keymanweb-webview.js.map",
+                     dstDir: baseDir,
+                     excludeFromBackup: true)
+    try Storage.copy(from: bundle,
+                     resourceName: "worker-thread.js",
+                     dstDir: baseDir,
+                     excludeFromBackup: true)
+    try Storage.copy(from: bundle,
+                     resourceName: "worker-thread.js.map",
+                     dstDir: baseDir,
+                     excludeFromBackup: true)
     try Storage.copy(from: bundle,
                      resourceName: "keyman-sentry.js",
                      dstDir: baseDir,

ios/engine/build.sh

@@ -109,6 +109,9 @@ function update_bundle ( ) {
   cp "$KMW_RESOURCES/osk/kmwosk.css"            "$BUNDLE_PATH/kmwosk.css"
   cp "$KMW_RESOURCES/osk/keymanweb-osk.ttf"     "$BUNDLE_PATH/keymanweb-osk.ttf"
   cp "$KMW_PRODUCT/keymanweb-webview.js"        "$BUNDLE_PATH/keymanweb-webview.js"
+  cp "$KMW_PRODUCT/keymanweb-webview.js.map"    "$BUNDLE_PATH/keymanweb-webview.js.map"
+  cp "$KMW_PRODUCT/worker-thread.js"            "$BUNDLE_PATH/worker-thread.js"
+  cp "$KMW_PRODUCT/worker-thread.js.map"        "$BUNDLE_PATH/worker-thread.js.map"
   cp "$KEYMAN_ROOT/common/web/sentry-manager/build/lib/index.js"     "$BUNDLE_PATH/keyman-sentry.js"
 }
 

web/README.md

@@ -98,7 +98,7 @@ graph TD;
     Gestures["/web/src/engine/gesture-processor"];
     Gestures-->WebUtils;
 
-    subgraph PredText["PredText: WebWorker + its interface"]
+    subgraph PredText["PredText: WebPredictiveTextWorker + its interface"]
         LMLayer;
         LMWorker;
         Models;

web/src/app/browser/build.sh

@@ -49,13 +49,13 @@ compile_and_copy() {
   BUILD_ROOT="${KEYMAN_ROOT}/web/build/app/browser"
   SRC_ROOT="${KEYMAN_ROOT}/web/src/app/browser/src"
 
-  node_es_bundle "${SRC_ROOT}/debug-main.js" \
+  node_es_bundle "${SRC_ROOT}/main.js" \
     --out        "${BUILD_ROOT}/debug/keymanweb.js" \
     --charset    "utf8" \
     --sourceRoot "@keymanapp/keyman/web/build/app/browser/debug" \
     --target     "es6"
 
-  node_es_bundle "${SRC_ROOT}/release-main.js" \
+  node_es_bundle "${SRC_ROOT}/main.js" \
     --out        "${BUILD_ROOT}/release/keymanweb.js" \
     --charset    "utf8" \
     --profile    "${BUILD_ROOT}/filesize-profile.log" \
@@ -72,6 +72,14 @@ compile_and_copy() {
   mkdir -p "$KEYMAN_ROOT/web/build/app/resources/osk"
   cp -R "$KEYMAN_ROOT/web/src/resources/osk/." "$KEYMAN_ROOT/web/build/app/resources/osk/"
 
+  cp "${KEYMAN_ROOT}/web/src/engine/predictive-text/worker-thread/build/lib/worker-thread.js" "${BUILD_ROOT}/debug/worker-thread.js"
+  cp "${KEYMAN_ROOT}/web/src/engine/predictive-text/worker-thread/build/lib/worker-thread.js.map" "${BUILD_ROOT}/debug/worker-thread.js.map"
+  cp "${KEYMAN_ROOT}/web/src/engine/predictive-text/worker-thread/build/lib/worker-thread.d.ts" "${BUILD_ROOT}/debug/worker-thread.d.ts"
+
+  # TODO: rename to lm-worker.js? do we keep debug/release distinction through .js vs .min.js or via folder?
+  cp "${KEYMAN_ROOT}/web/src/engine/predictive-text/worker-thread/build/lib/worker-thread.min.js" "${BUILD_ROOT}/release/worker-thread.js"
+  cp "${KEYMAN_ROOT}/web/src/engine/predictive-text/worker-thread/build/lib/worker-thread.min.js.map" "${BUILD_ROOT}/release/worker-thread.js.map"
+
   # Update the build/publish copy of our build artifacts
   prepare
 

web/src/app/browser/src/debug-main.ts → web/src/app/browser/src/main.ts

@@ -1,5 +1,9 @@
+/*
+ * Keyman is copyright (C) SIL Global. MIT License.
+ */
+
 import { KeymanEngine } from './keymanEngine.js'
-import { SourcemappedWorker } from '@keymanapp/lexical-model-layer/web'
+import { WebPredictiveTextWorkerFactory } from '@keymanapp/lexical-model-layer/web'
 
 /**
 * Determine path and protocol of executing script, setting them as
@@ -9,4 +13,4 @@ const ss = (document.currentScript as HTMLScriptElement)?.src;
 const sPath = ss ? ss.substring(0, ss.lastIndexOf('/') + 1) : './';
 
 // @ts-ignore
-window['keyman'] = new KeymanEngine(SourcemappedWorker, sPath);
+window['keyman'] = new KeymanEngine(new WebPredictiveTextWorkerFactory(sPath + 'worker-thread.js'), sPath);

web/src/app/webview/build.sh

@@ -44,13 +44,13 @@ compile_and_copy() {
   BUILD_ROOT="${KEYMAN_ROOT}/web/build/app/webview"
   SRC_ROOT="${KEYMAN_ROOT}/web/src/app/webview/src"
 
-  node_es_bundle "${SRC_ROOT}/debug-main.js" \
+  node_es_bundle "${SRC_ROOT}/main.js" \
     --out        "${BUILD_ROOT}/debug/keymanweb-webview.js" \
     --charset    "utf8" \
     --sourceRoot "@keymanapp/keyman/web/build/app/webview/debug" \
     --target     "es6"
 
-  node_es_bundle "${SRC_ROOT}/release-main.js" \
+  node_es_bundle "${SRC_ROOT}/main.js" \
     --out        "${BUILD_ROOT}/release/keymanweb-webview.js" \
     --charset    "utf8" \
     --profile    "${BUILD_ROOT}/filesize-profile.log" \
@@ -83,6 +83,14 @@ compile_and_copy() {
       "$script" "$sourcemap" --clean
   done
 
+  cp "${KEYMAN_ROOT}/web/src/engine/predictive-text/worker-thread/build/lib/worker-thread.js" "${BUILD_ROOT}/debug/worker-thread.js"
+  cp "${KEYMAN_ROOT}/web/src/engine/predictive-text/worker-thread/build/lib/worker-thread.js.map" "${BUILD_ROOT}/debug/worker-thread.js.map"
+  cp "${KEYMAN_ROOT}/web/src/engine/predictive-text/worker-thread/build/lib/worker-thread.d.ts" "${BUILD_ROOT}/debug/worker-thread.d.ts"
+
+  # TODO: rename to lm-worker.js? do we keep debug/release distinction through .js vs .min.js or via folder?
+  cp "${KEYMAN_ROOT}/web/src/engine/predictive-text/worker-thread/build/lib/worker-thread.min.js" "${BUILD_ROOT}/release/worker-thread.js"
+  cp "${KEYMAN_ROOT}/web/src/engine/predictive-text/worker-thread/build/lib/worker-thread.min.js.map" "${BUILD_ROOT}/release/worker-thread.js.map"
+
   # For dependent test pages.
   builder_launch /web/src/test/manual/embed/android-harness/build.sh configure,build
 }

web/src/app/browser/src/release-main.ts → web/src/app/webview/src/main.ts

@@ -1,5 +1,9 @@
+/*
+ * Keyman is copyright (C) SIL Global. MIT License.
+ */
+
 import { KeymanEngine } from './keymanEngine.js'
-import { WebWorker } from '@keymanapp/lexical-model-layer/web'
+import { WebPredictiveTextWorkerFactory } from '@keymanapp/lexical-model-layer/web'
 
 /**
 * Determine path and protocol of executing script, setting them as
@@ -9,4 +13,4 @@ const ss = (document.currentScript as HTMLScriptElement)?.src;
 const sPath = ss ? ss.substring(0, ss.lastIndexOf('/') + 1) : './';
 
 // @ts-ignore
-window['keyman'] = new KeymanEngine(WebWorker, sPath);
+window['keyman'] = new KeymanEngine(new WebPredictiveTextWorkerFactory(sPath + 'worker-thread.js'), sPath);

web/src/engine/build.sh

@@ -67,7 +67,7 @@ run_tests() {
   # "TypeError [ERR_INVALID_URL_SCHEME]: The URL must be of scheme file"
   # when c8 tries to create a report from the raw coverage data. The reason
   # is a URL starting with `data:text/javascript` coming from
-  # `web/src/engine/predictive-text/worker-main/src/node/mappedWorker.ts`.
+  # `web/src/engine/predictive-text/worker-main/src/node/node-predictive-text-worker.ts`.
   #
   # So we first run all javascript tests except languageProcessor.tests.js
   # with coverage, and then in a second step run languageProcessor.tests.js

web/src/engine/predictive-text/build.sh

@@ -32,7 +32,7 @@ builder_describe_outputs \
   build:templates               "/web/src/engine/predictive-text/templates/build/obj/index.js" \
   build:wordbreakers            "/web/src/engine/predictive-text/wordbreakers/build/main/obj/index.js" \
   build:worker-main             "/web/src/engine/predictive-text/worker-main/build/lib/index.mjs" \
-  build:worker-thread           "/web/src/engine/predictive-text/worker-thread/build/lib/worker-main.wrapped.js"
+  build:worker-thread           "/web/src/engine/predictive-text/worker-thread/build/lib/worker-thread.js"
 
 #### Build action definitions ####
 

web/src/engine/predictive-text/worker-main/src/index.ts

@@ -1,7 +1,8 @@
-export { LMLayer } from './lmlayer.js';
-export { NodeWorker } from './node/node-worker.js';
-export { WebWorker } from './web/web-worker.js';
+/*
+ * Keyman is copyright (C) SIL Global. MIT License.
+ */
 
-export interface WorkerFactory {
-  constructInstance(): Worker
-}
+export { LMLayer } from './lmlayer.js';
+export { type WorkerFactory } from './worker-factory.js';
+export { NodePredictiveTextWorkerFactory } from './node/node-predictive-text-worker-factory.js';
+export { WebPredictiveTextWorkerFactory } from './web/web-predictive-text-worker-factory.js';

web/src/engine/predictive-text/worker-main/src/lmlayer.ts

@@ -230,7 +230,7 @@ export class LMLayer {
 
   /**
    * Clears out any computational resources in use by the LMLayer, including shutting
-   * down any internal WebWorkers.
+   * down any internal PredictiveTextWorkers.
    */
   public shutdown() {
     this._worker.terminate();

web/src/engine/predictive-text/worker-main/src/node/index.ts

@@ -1,7 +1,7 @@
-export { LMLayer } from '../lmlayer.js';
-export { NodeWorker } from './node-worker.js';
-export { SourcemappedWorker } from './sourcemappedWorker.js';
+/*
+ * Keyman is copyright (C) SIL Global. MIT License.
+ */
 
-export interface WorkerFactory {
-    constructInstance(): Worker
-  }
+export { LMLayer } from '../lmlayer.js';
+export { type WorkerFactory } from '../worker-factory.js';
+export { NodePredictiveTextWorkerFactory } from './node-predictive-text-worker-factory.js';

web/src/engine/predictive-text/worker-main/src/node/node-predictive-text-worker-factory.ts

@@ -0,0 +1,22 @@
+/*
+ * Keyman is copyright (C) SIL Global. MIT License.
+ */
+import { WorkerFactory } from "../worker-factory.js";
+import { NodePredictiveTextWorker } from "./node-predictive-text-worker.js";
+
+import * as path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const __filename__ = fileURLToPath(import.meta.url);
+const __dirname__ = path.dirname(__filename__);
+export const nodeLexicalModelWorkerPath = __dirname__ + "/../../../../worker-thread/build/lib/worker-thread.js";
+
+export class NodePredictiveTextWorkerFactory implements WorkerFactory {
+  private workerFilename: string;
+  constructor(workerFilename?: string) {
+    this.workerFilename = workerFilename ?? nodeLexicalModelWorkerPath;
+  }
+  constructInstance(): Worker {
+    return new NodePredictiveTextWorker(this.workerFilename);
+  }
+}

web/src/engine/predictive-text/worker-main/src/node/mappedWorker.ts → web/src/engine/predictive-text/worker-main/src/node/node-predictive-text-worker.ts

@@ -1,3 +1,10 @@
+/*
+ * Keyman is copyright (C) SIL Global. MIT License.
+ *
+ * NodePredictiveTextWorker mirrors the necessary interfaces for
+ * WebPredictiveTextWorker in Node. It is used only for unit tests.
+ */
+
 // We use a subset of the Worker interface here; compiling directly against the true
 // WebWorker type definitions would require us to implement more methods than we do.
 /// <reference path="../worker-interface.d.ts" />
@@ -7,10 +14,13 @@ import * as worker from 'node:worker_threads';
 import { Buffer } from 'node:buffer';
 import { URL } from 'node:url';
 
+
 /**
  * Defines mappings from Node Worker signatures to WebWorker signatures
+ *
+ * TODO: move this to a separate module, no need for it to be embedded string
  */
-const nodeWorkerToWebWorkerMappingSource = `
+const nodeWorkerToWebWorkerMappingSource = (filename: string) => (`
 import { parentPort } from 'node:worker_threads';
 import fs from 'node:fs';
 import vm from 'node:vm';
@@ -42,37 +52,24 @@ function importScripts(...args) {
 const self = globalThis;
 self.postMessage = postMessage;
 self.importScripts = importScripts;
-`;
+self.self = self; // make it global!
+// Start off by importing the main worker itself
+importScripts(${JSON.stringify(filename)});
+`);
 
 /**
  * Uses the Node version of Workers to provide proper, authentic separate-thread
  * 'sandboxing'.  Also intercepts and interprets certain WebWorker method signatures
  * necessary to run the WebWorker-oriented worker code.
- *
- * Alternatively, only after writing this did I discover this package:
- * https://github.com/developit/web-worker.  They also ran one notable issue I did:
- * Node 18.x, at least, does not support use of Node Blobs for construction of a
- * Worker: https://github.com/developit/web-worker/pull/32... unlike Web Workers.
- *
- * So... Base64-encoded Data URLs it is.
- *
- * What we have here is perfectly fine for now, but if we need more complicated
- * cross-platform Worker support in the future, it may be wise to swap to use of
- * that package.
  */
-export class MappedWorker extends worker.Worker implements Worker {
-  constructor(scriptStr: string) {
-    const concatenatedScript = `
-    ${nodeWorkerToWebWorkerMappingSource}
-
-    ${scriptStr}
-    `;
-    const buffer = Buffer.from(concatenatedScript);
+export class NodePredictiveTextWorker extends worker.Worker implements Worker {
+  constructor(workerFilename: string) {
+    const buffer = Buffer.from(nodeWorkerToWebWorkerMappingSource(workerFilename));
     const dataSrc = "data:text/javascript;base64," + buffer.toString('base64');
     //@ts-ignore
     super(new URL(dataSrc));
 
-    // WebWorkers have a defined `onmessage` function, rather than this.on('message', ...)
+    // Workers have a defined `onmessage` function, rather than this.on('message', ...)
     this.on('message', (ev) => {
       if(this.onmessage) {
         this.onmessage({data: ev[0]});