Add a performance baseline harness for the data layer (#407)

.eslintignore

@@ -45,3 +45,6 @@ e2e-tests/test-results
 # A verbatim copy of the template's manifest, kept byte-for-byte so `npm run template:baseline` can
 # write it straight from `git show` rather than a copy plus whatever our tooling would impose on it
 scripts/merged-template-package.json
+
+# Captured text and generated datasets for the performance harness
+perf/.cache

.gitignore

@@ -40,6 +40,10 @@ temp-build
 # Playwright output
 e2e-tests/.cdp-*
 e2e-tests/.dev-*
+e2e-tests/.perf-*
 e2e-tests/.e2e-settings-backup*
 e2e-tests/playwright-report
 e2e-tests/test-results
+
+# Captured text and generated datasets for the performance harness
+perf/.cache

.prettierignore

@@ -45,3 +45,6 @@ e2e-tests/test-results
 # A verbatim copy of the template's manifest, kept byte-for-byte so `npm run template:baseline` can
 # write it straight from `git show` rather than a copy plus whatever our tooling would impose on it
 scripts/merged-template-package.json
+
+# Captured text and generated datasets for the performance harness
+perf/.cache

AGENTS.md

@@ -23,6 +23,14 @@ npm test                 # Run full Jest suite
 npm run test:coverage    # Run with coverage (100% threshold enforced)
 npm test -- path/to/file.test.ts                 # Run a single test file
 npm test -- --testNamePattern="pattern"          # Run tests matching name
+
+# Performance (results land in perf/results/)
+npm run perf:capture     # Capture the WEB sample's USJ from a launched app into perf/.cache
+npm run perf:datasets    # Generate the seeded dataset tiers (captures first if needed)
+npm run perf:bench       # Time the data layer's pure functions in Node
+npm run perf:app         # Production-build the extension and time it, WebView and extension host, in the app
+npm run perf:summary     # Render the newest results as markdown tables
+npm run perf:shape       # Compare analysis shape across tiers and stored projects
 ```
 
 Only a template merge moves the dependency baseline `npm run lint:dependencies` checks against; refresh it in that same commit with `npm run template:baseline`, which writes both the copy and the `MERGED_TEMPLATE_COMMIT` id recorded beside it in [scripts/check-dependency-scope.cjs](scripts/check-dependency-scope.cjs). [README.md](README.md) has the full procedure.

README.md

@@ -191,34 +191,21 @@ npm run test:coverage    # run with coverage report (output to coverage/)
 
 ### End-to-end tests
 
-E2E tests use [Playwright](https://playwright.dev/) and live in `e2e-tests/`. They launch Platform.Bible with this extension loaded and verify behavior through the real UI.
-
-**Prerequisites:**
-
-- `npm run build` must have been run (`dist/src/main.js` must exist)
-- `paranext-core` must have deps installed (e.g., with `npm run core:install`)
-
-**Smoke tests** (self-contained, good for CI — launches and tears down Platform.Bible automatically):
+E2E tests use [Playwright](https://playwright.dev/) and live in `e2e-tests/`. They launch Platform.Bible with this extension loaded and verify behavior through the real UI. Global setup builds the extension first; `paranext-core` must have deps installed (e.g., with `npm run core:install`).
 
 ```bash
-npm run test:e2e:smoke
+npm run test:e2e         # smoke tests, then CDP feature tests
+npm run test:e2e:smoke   # smoke tests only
+npm run test:e2e:cdp     # CDP feature tests only
 ```
 
-**CDP tests** (connect to an already-running app — faster for local development):
-
-1. Start Platform.Bible with remote debugging enabled:
-
-   ```bash
-   npm run start:cdp
-   ```
+Each tier launches its own Platform.Bible instance and tears it down afterward. To iterate against a warm instance, run `npm run start:cdp` first; the CDP tier reuses it and leaves it running.
 
-2. In a second terminal, run the tests:
+New feature tests should use `cdp.fixture` and navigate entirely through visible UI. See `e2e-tests/tests/_example/` for a reference template, and [e2e-tests/README.md](e2e-tests/README.md) for the full protocol.
 
-   ```bash
-   npm run test:e2e:cdp
-   ```
+### Performance
 
-New feature tests should use `cdp.fixture` and navigate entirely through visible UI. See `e2e-tests/tests/_example/` for a reference template.
+The performance harness times the data layer in Node and the extension in the app. See [perf/README.md](perf/README.md).
 
 ## Publishing
 

e2e-tests/README.md

@@ -23,6 +23,10 @@ To reproduce the Linux CI run locally before pushing, run `npm run test:e2e:head
 - `playwright*.config.ts` — fixture configs
 - `tests/` — tests, including a smoke test and a test template
 
+## Performance harness
+
+`playwright-perf.config.ts` and `tests/perf/` measure the extension rather than test it; see [perf/README.md](../perf/README.md).
+
 ## Key differences from `paranext-core/e2e-tests/`
 
 These tests are adapted from `paranext-core`'s e2e suite with changes to support testing a side-loaded extension rather than the core platform itself:

e2e-tests/global-setup-cdp.ts

@@ -320,7 +320,7 @@ async function cleanUpFailedLaunch(
  * @throws {Error} If no renderer page appears, tab titles do not resolve, or the extension does not
  *   activate within `timeout`.
  */
-async function waitForRendererSettled(timeout: number): Promise<void> {
+export async function waitForRendererSettled(timeout: number): Promise<void> {
   const deadline = Date.now() + timeout;
   const browser = await chromium.connectOverCDP(`http://localhost:${CDP_PORT}`, {
     timeout: 30_000,

e2e-tests/global-setup-perf.ts

@@ -0,0 +1,65 @@
+import type { FullConfig } from '@playwright/test';
+import fs from 'fs';
+import path from 'path';
+import { execSync } from 'child_process';
+import globalSetupCdp, { CDP_PORT } from './global-setup-cdp';
+import { isPortInUse, WEBSOCKET_PORT } from './global-setup';
+import globalTeardownCdp from './global-teardown-cdp';
+import { launchPackagedApp, PACKAGED_APP_PATH, stopPackagedApp } from './perf-app';
+
+/** The development host's storage for this extension's projects and drafts. */
+const STORAGE_DIR = path.resolve(
+  __dirname,
+  '../../paranext-core/dev-appdata/extensions/interlinearizer/user-data',
+);
+
+/** Where the developer's storage waits out a run. */
+const STORAGE_BACKUP_DIR = `${STORAGE_DIR}.perf-backup`;
+
+/** Puts back storage a run set aside, discarding what the run stored. */
+function restoreStorage(): void {
+  if (!fs.existsSync(STORAGE_BACKUP_DIR)) return;
+  fs.rmSync(STORAGE_DIR, { recursive: true, force: true });
+  fs.renameSync(STORAGE_BACKUP_DIR, STORAGE_DIR);
+}
+
+/**
+ * Launches the app as the CDP tier does, but over empty extension storage, so listing projects
+ * reads only what the run seeds and the developer's drafts and projects stay untouched. With
+ * `PERF_APP_PATH` set it builds the extension and launches that packaged app in its own home.
+ *
+ * @returns The run's teardown, which stops the app and gives back the storage set aside.
+ * @throws {Error} If Platform.Bible is already running, since it may own the storage a run sets
+ *   aside.
+ */
+export default async function globalSetupPerf(config: FullConfig): Promise<() => Promise<void>> {
+  if ((await isPortInUse(CDP_PORT)) || (await isPortInUse(WEBSOCKET_PORT))) {
+    throw new Error(
+      "Platform.Bible is already running. Quit it, or a previous run's app, so the run gets an app and storage of its own.",
+    );
+  }
+  if (PACKAGED_APP_PATH) {
+    const buildScript = process.env.E2E_BUILD_SCRIPT ?? 'build:production';
+    execSync(`npm run ${buildScript}`, { cwd: path.resolve(__dirname, '..'), stdio: 'inherit' });
+    await launchPackagedApp();
+    return stopPackagedApp;
+  }
+  // A crashed run leaves its backup in place; put it back before setting storage aside again.
+  restoreStorage();
+  fs.mkdirSync(STORAGE_DIR, { recursive: true });
+  fs.renameSync(STORAGE_DIR, STORAGE_BACKUP_DIR);
+  try {
+    fs.mkdirSync(STORAGE_DIR);
+    await globalSetupCdp(config);
+  } catch (error) {
+    restoreStorage();
+    throw error;
+  }
+  return async () => {
+    try {
+      await globalTeardownCdp(config);
+    } finally {
+      restoreStorage();
+    }
+  };
+}

e2e-tests/global-setup.ts

@@ -191,8 +191,10 @@ export async function bootstrapRendererDevServer(): Promise<void> {
   // extension the tests were not written against, and the selector mismatches that follow read as
   // real regressions rather than a build problem. A no-op rebuild costs seconds against a
   // multi-minute run, so always paying it is cheaper than ever debugging the stale case.
-  console.log('Building the extension...');
-  execSync('npm run build', { cwd: extensionRoot, stdio: 'inherit' });
+  // E2E_BUILD_SCRIPT names another build script, such as a production build.
+  const buildScript = process.env.E2E_BUILD_SCRIPT ?? 'build';
+  console.log(`Building the extension (npm run ${buildScript})...`);
+  execSync(`npm run ${buildScript}`, { cwd: extensionRoot, stdio: 'inherit' });
 
   // Ensure the paranext-core dev main bundle exists
   const devMainPath = path.join(coreDir, '.erb/dll/main.bundle.dev.js');

e2e-tests/perf-app.ts

@@ -0,0 +1,172 @@
+import { chromium } from '@playwright/test';
+import { spawn } from 'child_process';
+import fs from 'fs';
+import os from 'os';
+import path from 'path';
+import { CACHE_DIR } from '../perf/paths';
+import { CDP_PORT, waitForRendererSettled } from './global-setup-cdp';
+import { isPortInUse, waitForPort, WEBSOCKET_PORT } from './global-setup';
+import { E2E_SETTINGS_OVERRIDES } from './fixtures/helpers';
+import { killProcessTree, waitForPidExit } from './process-utils';
+
+/** Path to a packaged app's executable; set, it is what the harness launches. */
+export const PACKAGED_APP_PATH = process.env.PERF_APP_PATH;
+
+/** Home the packaged app runs under, apart from the developer's profile and kept across restarts. */
+export const PERF_HOME_DIR = path.join(CACHE_DIR, 'app-home');
+
+/** Display backend the app runs on under Linux, overridable with `PERF_OZONE_PLATFORM`. */
+export const OZONE_PLATFORM =
+  process.platform === 'linux' ? (process.env.PERF_OZONE_PLATFORM ?? 'x11') : undefined;
+
+const PID_FILE = path.join(__dirname, '.perf-app.pid');
+
+/** Where the launched app's output goes, overwritten by each launch. */
+export const PERF_APP_LOG_FILE = path.join(__dirname, '.perf-app.log');
+
+const READY_TIMEOUT = 5 * 60_000;
+
+const QUIT_TIMEOUT = 60_000;
+
+/** Prepares the packaged app's home and seeds the settings every e2e launch forces. */
+function seedSettings(): void {
+  // .NET reports a special folder that does not exist yet as an empty path. Windows resolves
+  // AppData under USERPROFILE; when it is missing, Electron cannot take its single-instance lock
+  // and the app quits as if another instance held it.
+  ['.config', '.local/share', 'AppData/Roaming', 'AppData/Local'].forEach((dir) =>
+    fs.mkdirSync(path.join(PERF_HOME_DIR, dir), { recursive: true }),
+  );
+  const file = path.join(PERF_HOME_DIR, '.platform.bible', 'data', 'settings.json');
+  fs.mkdirSync(path.dirname(file), { recursive: true });
+  const current: Record<string, unknown> = fs.existsSync(file)
+    ? JSON.parse(fs.readFileSync(file, 'utf-8'))
+    : {};
+  fs.writeFileSync(file, JSON.stringify({ ...current, ...E2E_SETTINGS_OVERRIDES }, undefined, 2));
+}
+
+/**
+ * Launches the packaged app with the built extension and remote debugging, then waits until the
+ * extension is ready.
+ *
+ * @returns Milliseconds from spawning the app to its being ready.
+ * @throws If the app is already running, exits during startup, or is not ready in time.
+ */
+export async function launchPackagedApp(): Promise<number> {
+  if (!PACKAGED_APP_PATH) throw new Error('PERF_APP_PATH names no packaged app');
+  if ((await isPortInUse(WEBSOCKET_PORT)) || (await isPortInUse(CDP_PORT)))
+    throw new Error('Platform.Bible is already running');
+  seedSettings();
+
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  const { ELECTRON_RUN_AS_NODE, ...restEnv } = process.env;
+  const logFd = fs.openSync(PERF_APP_LOG_FILE, 'w');
+  const started = Date.now();
+  let app: ReturnType<typeof spawn>;
+  try {
+    app = spawn(
+      PACKAGED_APP_PATH,
+      [
+        '--extensions',
+        path.resolve(__dirname, '../dist'),
+        `--remote-debugging-port=${CDP_PORT}`,
+        '--window-size',
+        '1280x960',
+        ...(process.platform === 'linux'
+          ? ['--no-sandbox', `--ozone-platform=${OZONE_PLATFORM}`]
+          : []),
+      ],
+      {
+        env: {
+          ...restEnv,
+          HOME: PERF_HOME_DIR,
+          // Windows takes the home folder from here, and its data provider ignores both variables.
+          USERPROFILE: PERF_HOME_DIR,
+          PLATFORM_BIBLE_PROJECT_ROOT_FOLDER: path.join(
+            PERF_HOME_DIR,
+            '.platform.bible',
+            'projects',
+            'Paratext 9 Projects',
+          ),
+          // Display authority usually lives in the real home, which the app cannot see.
+          ...(fs.existsSync(path.join(os.homedir(), '.Xauthority')) && !process.env.XAUTHORITY
+            ? { XAUTHORITY: path.join(os.homedir(), '.Xauthority') }
+            : {}),
+        },
+        stdio: ['ignore', logFd, logFd],
+        detached: true,
+      },
+    );
+  } finally {
+    fs.closeSync(logFd);
+  }
+  app.unref();
+  if (app.pid) fs.writeFileSync(PID_FILE, String(app.pid));
+
+  const exited = new Promise<never>((_resolve, reject) => {
+    app.once('exit', (code, signal) =>
+      reject(new Error(`The packaged app exited during startup (code=${code}, signal=${signal})`)),
+    );
+    app.once('error', reject);
+  });
+  exited.catch(() => {});
+  try {
+    await Promise.race([waitForPort(WEBSOCKET_PORT, READY_TIMEOUT), exited]);
+    await Promise.race([waitForPort(CDP_PORT, READY_TIMEOUT), exited]);
+    await Promise.race([waitForRendererSettled(READY_TIMEOUT), exited]);
+  } catch (error) {
+    await stopPackagedApp();
+    throw error;
+  }
+  return Date.now() - started;
+}
+
+/** Asks the app to quit as a user would, so it saves its layout. */
+async function requestQuit(): Promise<void> {
+  const browser = await chromium.connectOverCDP(`http://localhost:${CDP_PORT}`, {
+    timeout: 10_000,
+  });
+  try {
+    const page = browser
+      .contexts()
+      .flatMap((context) => context.pages())
+      .find((p) => !p.url().startsWith('devtools://'));
+    // The app may close the connection before the command answers.
+    await page?.evaluate(() => window.papi.commands.sendCommand('platform.quit')).catch(() => {});
+  } finally {
+    await browser.close().catch(() => {});
+  }
+}
+
+function isAlive(pid: number): boolean {
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+/** Stops the packaged app this harness launched, if it is running. */
+export async function stopPackagedApp(): Promise<void> {
+  if (!fs.existsSync(PID_FILE)) return;
+  const pid = Number(fs.readFileSync(PID_FILE, 'utf-8'));
+  try {
+    await requestQuit();
+  } catch {
+    // Not reachable over CDP; the kill below still stops it.
+  }
+  await waitForPidExit(pid, QUIT_TIMEOUT);
+  if (isAlive(pid)) {
+    killProcessTree(pid, 'SIGKILL');
+    await waitForPidExit(pid, 10_000);
+  }
+  // The next launch needs the websocket port free.
+  const deadline = Date.now() + QUIT_TIMEOUT;
+  while (Date.now() < deadline && (await isPortInUse(WEBSOCKET_PORT))) {
+    // eslint-disable-next-line no-await-in-loop
+    await new Promise<void>((resolve) => {
+      setTimeout(resolve, 250);
+    });
+  }
+  fs.rmSync(PID_FILE, { force: true });
+}

e2e-tests/playwright-cdp.config.ts

@@ -13,7 +13,7 @@ import { defineConfig } from '@playwright/test';
  */
 export default defineConfig({
   testDir: './tests',
-  testIgnore: ['**/smoke/**', '**/_example/**'],
+  testIgnore: ['**/smoke/**', '**/_example/**', '**/perf/**'],
   fullyParallel: false,
   forbidOnly: !!process.env.CI,
   // Retries help on the never-reset shared instance only because each test self-heals leftover

e2e-tests/playwright-perf.config.ts

@@ -0,0 +1,23 @@
+import { defineConfig } from '@playwright/test';
+
+/**
+ * Playwright configuration for the performance harness. It launches its own Platform.Bible as the
+ * CDP feature tier does, over empty extension storage, and never retries a spec: a retried
+ * measurement would be a different measurement.
+ */
+export default defineConfig({
+  testDir: './tests/perf',
+  fullyParallel: false,
+  workers: 1,
+  retries: 0,
+  reporter: [['list']],
+  timeout: 60 * 60_000,
+  expect: { timeout: 30_000 },
+  use: { actionTimeout: 30_000 },
+  globalSetup: './global-setup-perf.ts',
+  outputDir: './test-results/perf',
+  projects: [
+    { name: 'capture', testMatch: /capture-usj\.spec\.ts/ },
+    { name: 'app', testMatch: /app-perf\.spec\.ts/ },
+  ],
+});