refactor(web): fully unbundles worker with engine-init configurable source path

web/src/app/browser/src/main.ts

@@ -1,5 +1,5 @@
 import { KeymanEngine } from './keymanEngine.js'
-import { WebWorker } from '@keymanapp/lexical-model-layer/web'
+import { WebWorkerFactory } from '@keymanapp/lexical-model-layer/web'
 
 /**
 * Determine path and protocol of executing script, setting them as
@@ -9,4 +9,4 @@ const ss = (document.currentScript as HTMLScriptElement)?.src;
 const sPath = ss ? ss.substring(0, ss.lastIndexOf('/') + 1) : './';
 
 // @ts-ignore
-window['keyman'] = new KeymanEngine(WebWorker, sPath);
+window['keyman'] = new KeymanEngine(new WebWorkerFactory(), sPath);

web/src/app/webview/src/main.ts

@@ -1,5 +1,5 @@
 import { KeymanEngine } from './keymanEngine.js'
-import { WebWorker } from '@keymanapp/lexical-model-layer/web'
+import { WebWorkerFactory } from '@keymanapp/lexical-model-layer/web'
 
 /**
 * Determine path and protocol of executing script, setting them as
@@ -9,4 +9,4 @@ const ss = (document.currentScript as HTMLScriptElement)?.src;
 const sPath = ss ? ss.substring(0, ss.lastIndexOf('/') + 1) : './';
 
 // @ts-ignore
-window['keyman'] = new KeymanEngine(WebWorker, sPath);
+window['keyman'] = new KeymanEngine(new WebWorkerFactory(), sPath);

web/src/engine/build.sh

@@ -61,6 +61,9 @@ do_build () {
 }
 
 run_tests() {
+  # Ensure test resources are properly built.
+  tsc -b ../test/auto/resources
+
   # Run javascript tests
   #
   # Trying to run languageProcessor.tests.js with c8 coverage fails with:

web/src/engine/predictive-text/worker-main/src/index.ts

@@ -1,7 +1,7 @@
 export { LMLayer } from './lmlayer.js';
-export { NodeWorker } from './node/node-worker.js';
-export { WebWorker } from './web/web-worker.js';
+export { NodeWorkerFactory } from './node/node-worker.js';
+export { WebWorkerFactory } from './web/web-worker.js';
 
 export interface WorkerFactory {
-  constructInstance(): Worker
+  constructInstance(workerSourcePath: string): Worker
 }

web/src/engine/predictive-text/worker-main/src/node/index.ts

@@ -1,6 +1,6 @@
 export { LMLayer } from '../lmlayer.js';
-export { NodeWorker } from './node-worker.js';
+export { NodeWorkerFactory } from './node-worker.js';
 
 export interface WorkerFactory {
-    constructInstance(): Worker
+    constructInstance(workerSource: string): Worker
   }

web/src/engine/predictive-text/worker-main/src/node/mappedWorker.ts

@@ -8,58 +8,57 @@ import { Buffer } from 'node:buffer';
 import { URL } from 'node:url';
 
 /**
- * Defines mappings from Node Worker signatures to WebWorker signatures
- *
- * TODO: move this to a separate module, no need for it to be embedded string
+ * Uses the Node version of Workers to provide proper, authentic separate-thread
+ * 'sandboxing'.  Also intercepts and interprets certain WebWorker method signatures
+ * necessary to run the WebWorker-oriented worker code.
  */
-const nodeWorkerToWebWorkerMappingSource = `
-import { parentPort } from 'node:worker_threads';
-import fs from 'node:fs';
-import vm from 'node:vm';
+export class MappedWorker extends worker.Worker implements Worker {
+  constructor(sourcePathString: string) {
+    /**
+     * Defines mappings from Node Worker signatures to WebWorker signatures
+     *
+     * TODO: move this to a separate module, no need for it to be embedded string
+     */
+    const nodeWorkerToWebWorkerMappingSource = `
+    import { parentPort } from 'node:worker_threads';
+    import fs from 'node:fs';
+    import vm from 'node:vm';
 
-function postMessage(...args) {
-  parentPort.postMessage.call(parentPort, args);
-}
+    // Useful for diagnosis, but leads to noisiness in test logs
+    // console.dir(import.meta);
 
-parentPort.on('message', (ev) => {
-  onmessage({data: ev});
-});
+    function postMessage(...args) {
+      parentPort.postMessage.call(parentPort, args);
+    }
 
-function importScripts(...args) {
-  function loadScriptInContext(scriptPath) {
-    let scriptStr = fs.readFileSync(scriptPath);
-    var script = new vm.Script(scriptStr, { filename: scriptPath });
-    script.runInThisContext();
-  }
-
-  for(let arg of args) {
-    loadScriptInContext(arg);
-  }
-}
+    parentPort.on('message', (ev) => {
+      onmessage({data: ev});
+    });
 
-/*
- * You'd think the method signature mapping would be implied from the first line,
- * but all three lines must be explicitly specified or the emulation will fail.
- */
-const self = globalThis;
-self.postMessage = postMessage;
-self.importScripts = importScripts;
-self.self = self; // make it global!
-// Start off by importing the main worker itself
-// importScripts('${import.meta.dirname}/../../../worker-thread/build/lib/worker-main.js');
-console.dir(import.meta);
-importScripts('${import.meta.dirname}/worker-main.js');
-`;
+    function importScripts(...args) {
+      function loadScriptInContext(scriptPath) {
+        let scriptStr = fs.readFileSync(scriptPath);
+        var script = new vm.Script(scriptStr, { filename: scriptPath });
+        script.runInThisContext();
+      }
 
+      for(let arg of args) {
+        loadScriptInContext(arg);
+      }
+    }
 
+    /*
+    * You'd think the method signature mapping would be implied from the first line,
+    * but all three lines must be explicitly specified or the emulation will fail.
+    */
+    const self = globalThis;
+    self.postMessage = postMessage;
+    self.importScripts = importScripts;
+    self.self = self; // make it global!
+    // Start off by importing the main worker itself
+    importScripts('${sourcePathString}');
+    `;
 
-/**
- * Uses the Node version of Workers to provide proper, authentic separate-thread
- * 'sandboxing'.  Also intercepts and interprets certain WebWorker method signatures
- * necessary to run the WebWorker-oriented worker code.
- */
-export class MappedWorker extends worker.Worker implements Worker {
-  constructor() {
     const buffer = Buffer.from(nodeWorkerToWebWorkerMappingSource);
     const dataSrc = "data:text/javascript;base64," + buffer.toString('base64');
     //@ts-ignore

web/src/engine/predictive-text/worker-main/src/node/node-worker.ts

@@ -4,8 +4,8 @@
 import { MappedWorker } from "./mappedWorker.js";
 
 // TODO: eliminate MappedWorker as its own thing
-export class NodeWorker {
-  static constructInstance(): Worker {
-    return new MappedWorker();
+export class NodeWorkerFactory {
+  constructInstance(workerSource: string): Worker {
+    return new MappedWorker(workerSource);
   }
 }

web/src/engine/predictive-text/worker-main/src/web/index.ts

@@ -1,6 +1,6 @@
 export { LMLayer } from '../lmlayer.js';
-export { WebWorker } from './web-worker.js';
+export { WebWorkerFactory } from './web-worker.js';
 
 export interface WorkerFactory {
-  constructInstance(): Worker
+  constructInstance(workerSource: string): Worker
 }

web/src/engine/predictive-text/worker-main/src/web/web-worker.ts

@@ -2,15 +2,8 @@
  * Keyman is copyright (C) SIL Global. MIT License.
  */
 
-export class WebWorker {
-  static constructInstance(): Worker {
-    return new Worker(this.workerURI());
-  }
-
-  static workerURI(): string {
-    // TODO: worker-thread generates worker-main.js,... whaaa
-    // TODO: worker-main.min.js?
-    // TODO: paths
-    return './worker-main.js';
+export class WebWorkerFactory {
+  constructInstance(workerSource: string): Worker {
+    return new Worker(workerSource);
   }
 }

web/src/engine/predictive-text/worker-main/unit_tests/headless/worker-dummy-integration.tests.js

@@ -4,8 +4,9 @@ import fs from 'fs';
 import { createRequire } from 'module';
 const require = createRequire(import.meta.url);
 
-import { LMLayer, NodeWorker as Worker } from '#./node/index.js';
+import { LMLayer, NodeWorkerFactory } from '#./node/index.js';
 import { capabilities, iGotDistractedByHazel } from '@keymanapp/common-test-resources/model-helpers.mjs';
+import { getWorkerPath } from 'keyman/test/resources';
 
 /*
  * Shows off the LMLayer API, using the full prediction interface.
@@ -20,7 +21,7 @@ describe('LMLayer using dummy model', function () {
   let worker;
 
   beforeEach(function() {
-    worker = Worker.constructInstance();
+    worker = (new NodeWorkerFactory()).constructInstance(getWorkerPath());
     lmLayer = new LMLayer(capabilities(), worker);
   });
 

web/src/engine/predictive-text/worker-main/unit_tests/headless/worker-trie-integration.tests.js

@@ -3,8 +3,9 @@ import { assert } from 'chai';
 import { createRequire } from 'module';
 const require = createRequire(import.meta.url);
 
-import { LMLayer, NodeWorker as Worker } from '#./node/index.js';
+import { LMLayer, NodeWorkerFactory } from '#./node/index.js';
 import { capabilities } from '@keymanapp/common-test-resources/model-helpers.mjs';
+import { getWorkerPath } from 'keyman/test/resources';
 
 /*
  * How to run the worlist
@@ -14,7 +15,7 @@ describe('LMLayer using the trie model', function () {
   let worker;
 
   beforeEach(function() {
-    worker = Worker.constructInstance();
+    worker = (new NodeWorkerFactory()).constructInstance(getWorkerPath());
     lmLayer = new LMLayer(capabilities(), worker, true);
   });
 

web/src/engine/predictive-text/worker-main/unit_tests/in_browser/cases/top-level-lmlayer.tests.ts

@@ -1,47 +1,18 @@
 import { assert } from 'chai';
 
-import { LMLayer, WebWorker }  from "@keymanapp/lexical-model-layer/web";
+import { LMLayer, WebWorkerFactory }  from "@keymanapp/lexical-model-layer/web";
 
 import { DEFAULT_BROWSER_TIMEOUT } from '@keymanapp/common-test-resources/test-timeouts.mjs';
-import { defaultCapabilities } from '../helpers.mjs';
+import { defaultCapabilities, workerPath } from '../helpers.mjs';
 
 describe('LMLayer', function () {
   this.timeout(DEFAULT_BROWSER_TIMEOUT);
 
   describe('[[constructor]]', function () {
     it('should construct with a single argument', function () {
-      let lmLayer = new LMLayer(defaultCapabilities, WebWorker.constructInstance(), true);
+      let lmLayer = new LMLayer(defaultCapabilities, (new WebWorkerFactory()).constructInstance(workerPath), true);
       assert.instanceOf(lmLayer, LMLayer);
       lmLayer.shutdown();
     });
   });
-
-  describe('#asBlobURI()', function () {
-    // #asBlobURI() requires browser APIs, hence why it cannot be tested headless in Node.
-    it('should take a function and convert it into a blob function', function (done) {
-      function dummyHandler() {
-        // Post something weird, so we can be reasonably certain the Web Worker is...
-        // well, working.
-        // WARNING: Do NOT refactor this string as a variable. It **MUST** remain a string
-        // in this function body, because the code in this function's body gets
-        // stringified!
-        postMessage('fhqwhgads');
-      }
-
-      // Note:  the full declaration exists; the code we want is wrapped within the func.
-      // So... let's just call the func.
-      const workerSrc = dummyHandler.toString() + "\ndummyHandler()";
-      let uri = WebWorker.asBlobURI(workerSrc);
-      assert.match(uri, /^blob:/);
-
-      let worker = new Worker(uri);
-      worker.onmessage = function thisShouldBeCalled(event) {
-        assert.propertyVal(event, 'data', 'fhqwhgads');
-        worker.terminate();
-        done();
-      };
-
-      worker.postMessage('test');
-    })
-  })
 });

web/src/engine/predictive-text/worker-main/unit_tests/in_browser/cases/worker-dummy-integration.tests.ts

@@ -1,9 +1,9 @@
 import { assert } from 'chai';
 
-import { LMLayer, WebWorker }   from "@keymanapp/lexical-model-layer/web";
+import { LMLayer, WebWorkerFactory }   from "@keymanapp/lexical-model-layer/web";
 
 import { DEFAULT_BROWSER_TIMEOUT } from '@keymanapp/common-test-resources/test-timeouts.mjs';
-import { defaultCapabilities } from '../helpers.mjs';
+import { defaultCapabilities, workerPath } from '../helpers.mjs';
 
 // Import assertions, even using 'with', aren't yet supported in Firefox's engine.
 // import hazelModel from '@keymanapp/common-test-resources/json/models/future_suggestions/i_got_distracted_by_hazel.json' with { type: 'json' };
@@ -28,13 +28,14 @@ describe('LMLayer using dummy model', function () {
     let loc = document.location;
     // config.testFile generally starts with a '/', with the path resembling the actual full local
     // filesystem for the drive.
-    domain = `${loc.protocol}/${loc.host}`
+    domain = `${loc.protocol}//${loc.host}`
 
     // Test-config setups will take care of the rest; the server-path will be rooted at the repo root.
     // With aliasing for resources/.
 
     // Since Firefox can't do JSON imports quite yet.
-    const hazelFixture = await fetch(new URL(`${domain}/resources/json/models/future_suggestions/i_got_distracted_by_hazel.json`));
+    console.log(new URL(`${domain}/common/test/resources/json/models/future_suggestions/i_got_distracted_by_hazel.json`));
+    const hazelFixture = await fetch(new URL(`${domain}/common/test/resources/json/models/future_suggestions/i_got_distracted_by_hazel.json`));
     hazelModel = await hazelFixture.json();
     hazelModel = hazelModel.map((set) => set.map((entry) => {
       return {
@@ -52,7 +53,7 @@ describe('LMLayer using dummy model', function () {
 
   describe('Prediction', function () {
     it('will predict future suggestions', function () {
-      var lmLayer = new LMLayer(defaultCapabilities, WebWorker.constructInstance(), true);
+      var lmLayer = new LMLayer(defaultCapabilities, (new WebWorkerFactory()).constructInstance(workerPath), true);
 
       var stripIDs = function(suggestions) {
         suggestions.forEach(function(suggestion) {
@@ -93,7 +94,7 @@ describe('LMLayer using dummy model', function () {
 
   describe('Wordbreaking', function () {
     it('will perform (default) wordbreaking and return word at caret', function () {
-      var lmLayer = new LMLayer(defaultCapabilities, WebWorker.constructInstance());
+      var lmLayer = new LMLayer(defaultCapabilities, (new WebWorkerFactory()).constructInstance(workerPath));
 
       // We're testing many as asynchronous messages in a row.
       // this would be cleaner using async/await syntax, but

web/src/engine/predictive-text/worker-main/unit_tests/in_browser/cases/worker-trie-integration.tests.ts

@@ -1,8 +1,8 @@
 import { assert } from 'chai';
 
-import { LMLayer, WebWorker }   from "@keymanapp/lexical-model-layer/web";
+import { LMLayer, WebWorkerFactory }   from "@keymanapp/lexical-model-layer/web";
 import { DEFAULT_BROWSER_TIMEOUT } from '@keymanapp/common-test-resources/test-timeouts.mjs';
-import { defaultCapabilities } from '../helpers.mjs';
+import { defaultCapabilities, workerPath } from '../helpers.mjs';
 
 // Import assertions, even using 'with', aren't yet supported in Firefox's engine.
 // import hazelModel from '@keymanapp/common-test-resources/json/models/future_suggestions/i_got_distracted_by_hazel.json' with { type: 'json' };
@@ -17,7 +17,7 @@ describe('LMLayer using the trie model', function () {
 
   before(async () => {
     let loc = document.location;
-    domain = `${loc.protocol}/${loc.host}`;
+    domain = `${loc.protocol}//${loc.host}`;
 
     // Test-config setups will take care of the rest; the server-path will be rooted at the repo root.
     // With aliasing for resources/.
@@ -29,7 +29,7 @@ describe('LMLayer using the trie model', function () {
       // Parameter 3 = true:  enables 'test mode', disables correction-search timeout.
       // This helps prevent the correction-search timeout from flaking out periodically during unit tests in
       // CI, since remote servers / devices are involved.
-      var lmLayer = new LMLayer(defaultCapabilities, WebWorker.constructInstance(), true);
+      var lmLayer = new LMLayer(defaultCapabilities, (new WebWorkerFactory()).constructInstance(workerPath), true);
 
       // We're testing many as asynchronous messages in a row.
       // this would be cleaner using async/await syntax, but
@@ -71,7 +71,7 @@ describe('LMLayer using the trie model', function () {
     //
     // https://community.software.sil.org/t/search-term-to-key-in-lexical-model-not-working-both-ways-by-default/3133
     it('should use the default searchTermToKey()', function () {
-      var lmLayer = new LMLayer(defaultCapabilities, WebWorker.constructInstance(), /* testMode */ true);
+      var lmLayer = new LMLayer(defaultCapabilities, (new WebWorkerFactory()).constructInstance(workerPath), /* testMode */ true);
 
       let loc = document.location;
       return lmLayer.loadModel(

web/src/engine/predictive-text/worker-main/unit_tests/in_browser/helpers.mjs

@@ -1,3 +1,5 @@
 export let defaultCapabilities = {
   maxLeftContextCodeUnits: 64
-};
+};
+
+export const workerPath = 'web/build/publish/debug/worker-main.js';

web/src/engine/predictive-text/worker-thread/src/tests/test-runner/cases/worker.tests.ts

@@ -6,7 +6,7 @@ describe('LMLayerWorker', function () {
 
   describe('Usage within a Web Worker', function () {
     it('should install itself in the worker context', function (done) {
-      let worker = new Worker(document.location.protocol + '//' + document.location.host + "/worker-main.js");
+      let worker = new Worker(document.location.protocol + '//' + document.location.host + "/web/src/engine/predictive-text/worker-thread/build/lib/worker-main.js");
       worker.onmessage = function thisShouldBeCalled(message) {
         done();
         worker.terminate();