Start the stone and session listener once, across processes

CHANGELOG.md

@@ -30,6 +30,26 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   no setter. A method's receiver (`self`, or whatever the def calls it), a tuple's items, group
   rows and a saved stack's values cannot be changed, and VS Code does not offer Set Value on them.
 
+### Fixed
+
+- **Starting GemDB while something else is starting it no longer reports a failure.** Pressing
+  Start, running a notebook cell or opening the GemDB Shell while another window or terminal was
+  bringing the database up could end in "Start session listener failed (exit code 1)", although the
+  database and its session listener were both running. GemDB now checks for and starts the session
+  listener under the same lock as the database, so one start waits for the other, and a listener
+  started by an older GemDB in another editor counts as started.
+- **GemDB waits for a database that is still starting before logging in to it.** After a crash
+  the database takes a while to recover before it accepts logins, and GemDB treated it as running
+  as soon as it appeared, so the first notebook cell, the GemDB Shell or a `gemdb` command could
+  fail to log in. GemDB and the `gemdb` command now wait for the database to finish starting, for up
+  to a minute, and then name the database's log if it has not.
+- **The GemDB Shell checks the database before starting it, as the window does.** Opening the
+  Shell with the database stopped started it without the checks the window makes first, so a
+  database the installed engine cannot read started and then refused every login, and one on a
+  network drive or a disk without room for it failed with the engine's own error. The Shell now
+  refuses with the same explanation the window gives, and applies the space limits a database
+  from an earlier GemDB is missing.
+
 ## [1.6.0] - 2026-10-07
 
 The engine is unchanged at GemStone 4.0.0.a4, but start with a fresh database: delete

CLAUDE.md

@@ -51,6 +51,11 @@ outstanding preparation, prompts for shared memory, starts the processes, files
 Grail in, and brings the MCP server up. New entry points that need a database
 should call it rather than checking and asking.
 
+Below it, the processes themselves start only through `ensureProcesses` in
+`processes.ts`, which checks and starts under the stone lock (NetLDI cannot be
+started any other way), and "ready" means gslist status `OK`, not "listed": a
+stone at `Startup` is listed and refuses logins, so wait on it, never start it.
+
 ## CI
 
 `.github/workflows/ci.yml`, two jobs split on what they need. `checks` runs

src/__integration__/breakpoint.test.ts

@@ -25,7 +25,7 @@ import {
   setVariableQuery,
   unquote,
 } from '../pauseVariables';
-import { isRunning, startNetldi, startStone, stopNetldi, stopStone } from '../processes';
+import { ensureProcesses, isRunning, stopNetldi, stopStone } from '../processes';
 import { fileOwner } from '../fileOwner';
 import { runPython, runPythonFile } from '../pythonQueries';
 import {
@@ -94,8 +94,7 @@ beforeAll(async () => {
   if (!fixture) return;
   createDatabaseWithPython(fixture);
   stageGrail(ext);
-  await startStone();
-  await startNetldi();
+  await ensureProcesses();
 });
 
 afterEach(() => setHaltHandler(undefined));

src/__integration__/database.test.ts

@@ -5,12 +5,12 @@ import { createDatabase } from '../database';
 import { isSharedMemoryConfigured } from '../osConfig';
 import { databaseExists, databaseStatPath, extentPath } from '../paths';
 import {
+  ensureProcesses,
   findNetldi,
   findStone,
   isListening,
   isRunning,
   listProcesses,
-  startNetldi,
   startStone,
   stopNetldi,
   stopStone,
@@ -81,8 +81,7 @@ describe.skipIf(!makeFixtureIsPossible())('a real database', () => {
   it('starts, and reports itself through gslist', async () => {
     // As startProcesses does before every start.
     ensureStatmonitor();
-    await startStone();
-    await startNetldi();
+    await ensureProcesses();
     ensureDatabaseAccount();
 
     const processes = listProcesses();

src/__integration__/fixtures/processContender.ts

@@ -0,0 +1,87 @@
+import * as fs from 'fs';
+import * as os from 'os';
+import * as path from 'path';
+import { spawn } from 'child_process';
+import { NETLDI_NAME } from '../../config';
+import { databaseLogPath } from '../../paths';
+import { engineEnvironment, ensureProcesses } from '../../processes';
+
+/**
+ * One process starting the database, run as a process of its own.
+ *
+ * processes.test.ts bundles this with the same `vscode` alias the shell bundle
+ * uses and starts two at once, because the race it covers is between
+ * processes: two windows, or a window and a GemDB Shell, that both find the
+ * database down. The root path arrives as GEMDB_ROOT_PATH, as it does for the
+ * real wrapper.
+ *
+ *   argv: <ensure|raw-netldi> <ready file> <go file> <report file> [rival log]
+ *
+ * `ensure` is what every start does today. `raw-netldi` is what GemDB 1.6.0
+ * did: `startnetldi` with no lock and no look first (#89).
+ *
+ * It says it is ready, waits for the go file so both start at the same
+ * moment, then appends one line saying what happened to the report.
+ *
+ * Released together, the raw `startnetldi` finishes before the other process
+ * has taken the lock and looked, so that one finds the listener up and the
+ * two never collide. Given a rival's log, the raw one waits after the go file
+ * until the rival logs its own `startnetldi`, so both commands run at once.
+ */
+const [mode, readyFile, goFile, reportFile, rivalLog] = process.argv.slice(2);
+const me = process.pid;
+
+const sleep = (ms: number): Promise<void> => new Promise((resolve) => setTimeout(resolve, ms));
+const report = (line: string): void => fs.appendFileSync(reportFile, `${line}\n`);
+
+/** `startnetldi` exactly as 1.6.0 ran it; resolves with its exit code and output. */
+function rawStartNetldi(): Promise<{ code: number | null; output: string }> {
+  const env = engineEnvironment();
+  return new Promise((resolve, reject) => {
+    const child = spawn(
+      path.join(env.GEMSTONE, 'bin', 'startnetldi'),
+      [
+        '-a',
+        os.userInfo().username,
+        '-g',
+        '-l',
+        path.join(databaseLogPath(), `${NETLDI_NAME}.log`),
+        NETLDI_NAME,
+      ],
+      { env: { ...process.env, ...env } },
+    );
+    let output = '';
+    child.stdout.on('data', (data: Buffer) => (output += data.toString()));
+    child.stderr.on('data', (data: Buffer) => (output += data.toString()));
+    child.on('close', (code) => resolve({ code, output }));
+    child.on('error', reject);
+  });
+}
+
+/** Wait until the rival logs that it is running `startnetldi`, or give up after a while. */
+async function rivalStartsNetldi(log: string): Promise<void> {
+  const deadline = Date.now() + 20_000;
+  while (Date.now() < deadline) {
+    if (fs.existsSync(log) && fs.readFileSync(log, 'utf8').includes('$ startnetldi')) return;
+    await sleep(2);
+  }
+}
+
+async function main(): Promise<void> {
+  fs.appendFileSync(readyFile, `${me}\n`);
+  while (!fs.existsSync(goFile)) await sleep(2);
+  if (rivalLog) await rivalStartsNetldi(rivalLog);
+
+  if (mode === 'ensure') {
+    const started = await ensureProcesses();
+    report(JSON.stringify({ mode, pid: me, ...started }));
+  } else {
+    const { code, output } = await rawStartNetldi();
+    report(JSON.stringify({ mode, pid: me, code, output: output.trim() }));
+  }
+}
+
+main().catch((e: unknown) => {
+  report(JSON.stringify({ mode, pid: me, error: String(e) }));
+  process.exitCode = 1;
+});

src/__integration__/grail.test.ts

@@ -5,7 +5,7 @@ import { createDatabase } from '../database';
 import { bundledGrailStamp, installGrail, recordGrailInstalled, stageGrail } from '../grail';
 import { isSharedMemoryConfigured } from '../osConfig';
 import { grailInstalled, grailStagedOnDisk } from '../paths';
-import { isRunning, startNetldi, startStone, stopNetldi, stopStone } from '../processes';
+import { ensureProcesses, isRunning, stopNetldi, stopStone } from '../processes';
 import {
   isErrorResult,
   isGrailInstalled,
@@ -61,8 +61,7 @@ beforeAll(async () => {
 
   createDatabase(fixture.engine);
   limitTestDatabase();
-  await startStone();
-  await startNetldi();
+  await ensureProcesses();
   // Grail installs into the gemdb account, as `ensureRunning` creates it.
   ensureDatabaseAccount();
 });

src/__integration__/mcp.test.ts

@@ -17,7 +17,7 @@ import {
   stopMcpServer,
 } from '../mcp';
 import { mcpInstalled, mcpRouterStatePath, mcpStagedOnDisk } from '../paths';
-import { isRunning, startNetldi, startStone, stopNetldi, stopStone } from '../processes';
+import { ensureProcesses, isRunning, stopNetldi, stopStone } from '../processes';
 import { GciSession, execute, logoutAll } from '../session';
 import { createDatabaseWithPython, Fixture, haveTestExtent, makeFixture } from './fixture';
 
@@ -73,8 +73,7 @@ beforeAll(async () => {
   // Grail's files on disk, because the router's worker gems inherit the
   // NetLDI's environment and resolve Python modules through GRAIL_DIR.
   stageGrail(ext);
-  await startStone();
-  await startNetldi();
+  await ensureProcesses();
 });
 
 afterAll(async () => {

src/__integration__/processes.test.ts

@@ -0,0 +1,189 @@
+import { afterAll, beforeAll, describe, expect, it } from 'vitest';
+import * as esbuild from 'esbuild';
+import * as fs from 'fs';
+import * as os from 'os';
+import * as path from 'path';
+import { execFile } from 'child_process';
+import { createDatabase } from '../database';
+import { isSharedMemoryConfigured } from '../osConfig';
+import {
+  isListening,
+  isRunning,
+  listProcesses,
+  startStone,
+  stopNetldi,
+  stopStone,
+} from '../processes';
+import { Fixture, limitTestDatabase, makeFixture } from './fixture';
+
+/**
+ * Two processes starting one database at the same moment.
+ *
+ * Two windows opened together, or a window and a GemDB Shell, both find the
+ * database down and both start it. Checked and started outside a lock, both
+ * ran `startnetldi`, and the loser reported a failure over a database that was
+ * up (#89). The lock that closes that race keeps separate processes apart, so
+ * an in-process test cannot exercise it: these bundle processContender.ts the
+ * way lock.test.ts bundles its contender, and race real node processes against
+ * a real engine.
+ *
+ * No Python is needed, so the database is the engine's own extent and the
+ * file skips on a missing engine alone.
+ */
+
+interface Report {
+  mode: 'ensure' | 'raw-netldi';
+  pid: number;
+  startedStone?: boolean;
+  startedNetldi?: boolean;
+  code?: number | null;
+  output?: string;
+  error?: string;
+}
+
+let fixture: Fixture | undefined;
+let bundleDir: string | undefined;
+let bundle = '';
+
+beforeAll(async () => {
+  fixture = makeFixture();
+  if (!fixture) return;
+  if (!(await isSharedMemoryConfigured())) {
+    throw new Error(
+      'Shared memory is below what the engine needs. Run "GemDB: Configure Shared Memory" ' +
+        'in the editor, or resources/setSharedMemoryDarwin.sh, before running these tests.',
+    );
+  }
+  createDatabase(fixture.engine);
+  limitTestDatabase();
+
+  bundleDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gemdb-contender-'));
+  bundle = path.join(bundleDir, 'processContender.cjs');
+  await esbuild.build({
+    entryPoints: [path.join(__dirname, 'fixtures', 'processContender.ts')],
+    bundle: true,
+    platform: 'node',
+    format: 'cjs',
+    outfile: bundle,
+    logLevel: 'silent',
+    alias: { vscode: path.join(__dirname, '..', 'cliVscode.ts') },
+  });
+});
+
+afterAll(async () => {
+  if (bundleDir) fs.rmSync(bundleDir, { recursive: true, force: true });
+  if (!fixture) return;
+  try {
+    if (isListening()) await stopNetldi();
+    if (isRunning()) await stopStone(true);
+  } finally {
+    fixture.remove();
+  }
+});
+
+/** See database.test.ts — skipIf is evaluated during collection. */
+function canMakeFixture(): boolean {
+  const probe = makeFixture();
+  if (!probe) return false;
+  probe.remove();
+  return true;
+}
+
+const sleep = (ms: number): Promise<void> => new Promise((resolve) => setTimeout(resolve, ms));
+
+/**
+ * Start one contender per mode, release them together, and return what each
+ * reported. A contender's log goes to a file beside the report, and a
+ * contender that exits non-zero rejects with that log, which is the only
+ * account of what it did.
+ */
+async function race(
+  modes: Report['mode'][],
+  opts: { rawWaitsForRival?: boolean } = {},
+): Promise<Report[]> {
+  const root = fixture!.root;
+  const dir = fs.mkdtempSync(path.join(root, 'race-'));
+  const ready = path.join(dir, 'ready');
+  const go = path.join(dir, 'go');
+  const reportFile = path.join(dir, 'report');
+  fs.writeFileSync(ready, '');
+  const logFile = (i: number): string => path.join(dir, `contender-${i}.log`);
+  const rivalLog = logFile(modes.indexOf('ensure'));
+  const exits = modes.map(
+    (mode, i) =>
+      new Promise<void>((resolve, reject) => {
+        const log = logFile(i);
+        const waitFor = mode === 'raw-netldi' && opts.rawWaitsForRival ? [rivalLog] : [];
+        const child = execFile(
+          process.execPath,
+          [bundle, mode, ready, go, reportFile, ...waitFor],
+          { env: { ...process.env, GEMDB_ROOT_PATH: root, GEMDB_SHELL_LOG: log } },
+          (error, _stdout, stderr) => {
+            if (!error) return resolve();
+            const logged = fs.existsSync(log) ? fs.readFileSync(log, 'utf8') : '';
+            reject(new Error(`${mode} contender failed: ${stderr || String(error)}\n${logged}`));
+          },
+        );
+        child.on('error', reject);
+      }),
+  );
+
+  // Release only once both are parked at the starting line, or the first
+  // would be done before the second had started.
+  const deadline = Date.now() + 20_000;
+  while (fs.readFileSync(ready, 'utf8').split('\n').filter(Boolean).length < modes.length) {
+    if (Date.now() > deadline) throw new Error('contenders never got ready');
+    await sleep(10);
+  }
+  fs.writeFileSync(go, '');
+  await Promise.all(exits);
+
+  return fs
+    .readFileSync(reportFile, 'utf8')
+    .split('\n')
+    .filter(Boolean)
+    .map((line) => JSON.parse(line) as Report);
+}
+
+/** What `gslist` lists, reduced to what the end state is judged on. */
+function listed(): { type: string; status: string }[] {
+  return listProcesses().map((p) => ({ type: p.type, status: p.status }));
+}
+
+describe.skipIf(!canMakeFixture())('starting the database from two processes at once', () => {
+  it('starts one stone and one session listener, and neither process fails', async () => {
+    const reports = await race(['ensure', 'ensure']);
+
+    expect(reports.filter((r) => r.error)).toEqual([]);
+    expect(reports).toHaveLength(2);
+    expect(reports.filter((r) => r.startedStone)).toHaveLength(1);
+    expect(listed()).toEqual(
+      expect.arrayContaining([
+        { type: 'stone', status: 'OK' },
+        { type: 'netldi', status: 'OK' },
+      ]),
+    );
+    expect(listed()).toHaveLength(2);
+  });
+
+  it('does not fail when a release that takes no lock starts the session listener alongside', async () => {
+    if (isListening()) await stopNetldi();
+    await startStone();
+    expect(isListening()).toBe(false);
+
+    const reports = await race(['raw-netldi', 'ensure'], { rawWaitsForRival: true });
+
+    const ensured = reports.find((r) => r.mode === 'ensure');
+    expect(ensured).toBeDefined();
+    expect(ensured?.error).toBeUndefined();
+    expect(ensured?.startedStone).toBe(false);
+    expect(reports.find((r) => r.mode === 'raw-netldi')?.error).toBeUndefined();
+    expect(listed()).toEqual(
+      expect.arrayContaining([
+        { type: 'stone', status: 'OK' },
+        { type: 'netldi', status: 'OK' },
+      ]),
+    );
+    expect(listed()).toHaveLength(2);
+  });
+});