From 4e59ba8d185e724237d04cb440aac31a978892a0 Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Sun, 18 Oct 2026 12:23:48 -0700 Subject: [PATCH 01/10] Measure Seatbelt system Keychains access on disposable macOS Co-authored-by: kai Agent-Model: codex/gpt-6-sol/medium Agent-Model: codex/gpt-6.1-sol/medium Agent-Model: claude/claude-haiku-5-5/default --- .../workflows/seatbelt-system-keychains.yml | 69 ++++++++++++++++++ .../seatbelt.system-keychains.smoke.test.mjs | 73 +++++++++++++++++++ 2 files changed, 142 insertions(+) create mode 100644 .github/workflows/seatbelt-system-keychains.yml create mode 100644 cli/__tests__/seatbelt.system-keychains.smoke.test.mjs diff --git a/.github/workflows/seatbelt-system-keychains.yml b/.github/workflows/seatbelt-system-keychains.yml new file mode 100644 index 0000000000..22ce1e9994 --- /dev/null +++ b/.github/workflows/seatbelt-system-keychains.yml @@ -0,0 +1,69 @@ +name: Seatbelt system Keychains smoke + +on: + pull_request: + paths: + - 'cli/src/lib/sandbox/seatbelt.js' + - 'cli/__tests__/seatbelt.system-keychains.smoke.test.mjs' + - '.github/workflows/seatbelt-system-keychains.yml' + +permissions: + contents: read + +jobs: + system-root: + runs-on: macos-15 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: '22' + cache: npm + cache-dependency-path: cli/package-lock.json + - run: npm ci --prefix cli + - name: Trust a disposable root and exercise the narrowed profile + shell: bash + run: | + set -euo pipefail + cert_dir="$(mktemp -d)" + cat > "$cert_dir/root.cnf" <<'CONF' + [req] + distinguished_name = dn + x509_extensions = v3_ca + prompt = no + [dn] + CN = Commonly disposable Seatbelt root + [v3_ca] + basicConstraints = critical,CA:TRUE + keyUsage = critical,keyCertSign,cRLSign + subjectKeyIdentifier = hash + CONF + cat > "$cert_dir/leaf.cnf" <<'CONF' + [req] + distinguished_name = dn + req_extensions = v3_req + prompt = no + [dn] + CN = 127.0.0.1 + [v3_req] + basicConstraints = critical,CA:FALSE + keyUsage = critical,digitalSignature,keyEncipherment + extendedKeyUsage = serverAuth + subjectAltName = @alt_names + [alt_names] + IP.1 = 127.0.0.1 + CONF + openssl req -new -x509 -nodes -newkey rsa:2048 -sha256 -days 1 \ + -keyout "$cert_dir/root.key" -out "$cert_dir/root.pem" -config "$cert_dir/root.cnf" + openssl req -new -nodes -newkey rsa:2048 -sha256 \ + -keyout "$cert_dir/leaf.key" -out "$cert_dir/leaf.csr" -config "$cert_dir/leaf.cnf" + openssl x509 -req -in "$cert_dir/leaf.csr" -CA "$cert_dir/root.pem" \ + -CAkey "$cert_dir/root.key" -CAcreateserial -out "$cert_dir/leaf.pem" \ + -days 1 -sha256 -extfile "$cert_dir/leaf.cnf" -extensions v3_req + sudo security add-trusted-cert -d -r trustRoot \ + -k /Library/Keychains/System.keychain "$cert_dir/root.pem" + export COMMONLY_DISPOSABLE_SYSTEM_ROOT_CERT="$cert_dir/leaf.pem" + export COMMONLY_DISPOSABLE_SYSTEM_ROOT_KEY="$cert_dir/leaf.key" + cd cli + node --experimental-vm-modules node_modules/.bin/jest \ + __tests__/seatbelt.system-keychains.smoke.test.mjs --runInBand diff --git a/cli/__tests__/seatbelt.system-keychains.smoke.test.mjs b/cli/__tests__/seatbelt.system-keychains.smoke.test.mjs new file mode 100644 index 0000000000..c4546209e4 --- /dev/null +++ b/cli/__tests__/seatbelt.system-keychains.smoke.test.mjs @@ -0,0 +1,73 @@ +import { spawn, spawnSync } from 'child_process'; +import { createServer } from 'https'; +import { + mkdirSync, mkdtempSync, readFileSync, rmSync, +} from 'fs'; +import { homedir } from 'os'; +import { join } from 'path'; +import { buildSeatbeltProfile } from '../src/lib/sandbox/seatbelt.js'; + +const macTest = process.platform === 'darwin' ? test : test.skip; +const root = process.platform === 'darwin' + ? mkdtempSync(join(homedir(), '.commonly-keychain-smoke-')) : null; +const workspace = root && join(root, 'workspace'); +const state = root && join(root, 'state'); +if (root) { + mkdirSync(workspace); + mkdirSync(state); +} +afterAll(() => { if (root) rmSync(root, { recursive: true, force: true }); }); + +// Measure the proposed deletion against a real Seatbelt profile before the +// production baseline changes. Do not install a root on an operator machine. +const narrowedProfile = () => buildSeatbeltProfile({ + workspacePath: workspace, executablePath: process.execPath, statePath: state, +}).replace(' (subpath "/Library/Keychains")\n', ''); + +macTest('narrowed profile refuses system Keychains listing and legacy file opens', () => { + const code = `const fs = require('fs'); + const paths = ['/Library/Keychains', '/Library/Keychains/System.keychain', + '/Library/Keychains/apsd.keychain']; + for (const path of paths) { + try { + if (path.endsWith('/Keychains')) fs.readdirSync(path); + else fs.closeSync(fs.openSync(path, 'r')); + console.log('ALLOWED'); + } catch (error) { console.log(error.code); } + }`; + const result = spawnSync('/usr/bin/sandbox-exec', [ + '-p', narrowedProfile(), process.execPath, '-e', code, + ], { cwd: workspace, encoding: 'utf8', timeout: 10_000 }); + expect(result.error).toBeUndefined(); + expect(result.status).toBe(0); + expect(result.stdout.trim().split('\n')).toEqual(['EPERM', 'EPERM', 'EPERM']); +}); + +const disposableRunnerTest = process.platform === 'darwin' + && process.env.COMMONLY_DISPOSABLE_SYSTEM_ROOT_CERT + && process.env.COMMONLY_DISPOSABLE_SYSTEM_ROOT_KEY ? test : test.skip; + +disposableRunnerTest('system-trusted local TLS still works without direct Keychains reads', async () => { + const certificate = readFileSync(process.env.COMMONLY_DISPOSABLE_SYSTEM_ROOT_CERT); + const key = readFileSync(process.env.COMMONLY_DISPOSABLE_SYSTEM_ROOT_KEY); + const server = createServer({ cert: certificate, key }, (_req, response) => response.end('tls-ok')); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + try { + const port = server.address().port; + const result = await new Promise((resolve, reject) => { + const child = spawn('/usr/bin/sandbox-exec', [ + '-p', narrowedProfile(), '/usr/bin/curl', '--fail', '--silent', '--show-error', + `https://127.0.0.1:${port}/`, + ], { cwd: workspace }); + let stdout = ''; + let stderr = ''; + child.stdout.on('data', (chunk) => { stdout += chunk; }); + child.stderr.on('data', (chunk) => { stderr += chunk; }); + child.on('error', reject); + child.on('close', (status) => resolve({ status, stdout, stderr })); + }); + expect(result).toEqual({ status: 0, stdout: 'tls-ok', stderr: '' }); + } finally { + await new Promise((resolve) => server.close(resolve)); + } +}, 15_000); From 5eaf22f0ed0892d1532cc5af193898e38933765d Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Sun, 18 Oct 2026 12:24:58 -0700 Subject: [PATCH 02/10] Expose disposable macOS trust and sandbox probe failures Co-authored-by: kai Agent-Model: codex/gpt-6-sol/medium Agent-Model: codex/gpt-6.1-sol/medium Agent-Model: claude/claude-haiku-5-5/default --- .github/workflows/seatbelt-system-keychains.yml | 2 ++ .../seatbelt.system-keychains.smoke.test.mjs | 16 +++++++++++++--- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/.github/workflows/seatbelt-system-keychains.yml b/.github/workflows/seatbelt-system-keychains.yml index 22ce1e9994..c250d3590a 100644 --- a/.github/workflows/seatbelt-system-keychains.yml +++ b/.github/workflows/seatbelt-system-keychains.yml @@ -62,6 +62,8 @@ jobs: -days 1 -sha256 -extfile "$cert_dir/leaf.cnf" -extensions v3_req sudo security add-trusted-cert -d -r trustRoot \ -k /Library/Keychains/System.keychain "$cert_dir/root.pem" + /usr/bin/curl --version | head -2 + security verify-cert -c "$cert_dir/leaf.pem" -p ssl -s 127.0.0.1 || true export COMMONLY_DISPOSABLE_SYSTEM_ROOT_CERT="$cert_dir/leaf.pem" export COMMONLY_DISPOSABLE_SYSTEM_ROOT_KEY="$cert_dir/leaf.key" cd cli diff --git a/cli/__tests__/seatbelt.system-keychains.smoke.test.mjs b/cli/__tests__/seatbelt.system-keychains.smoke.test.mjs index c4546209e4..2a161ef989 100644 --- a/cli/__tests__/seatbelt.system-keychains.smoke.test.mjs +++ b/cli/__tests__/seatbelt.system-keychains.smoke.test.mjs @@ -39,7 +39,7 @@ macTest('narrowed profile refuses system Keychains listing and legacy file opens '-p', narrowedProfile(), process.execPath, '-e', code, ], { cwd: workspace, encoding: 'utf8', timeout: 10_000 }); expect(result.error).toBeUndefined(); - expect(result.status).toBe(0); + expect({ status: result.status, stderr: result.stderr }).toEqual({ status: 0, stderr: '' }); expect(result.stdout.trim().split('\n')).toEqual(['EPERM', 'EPERM', 'EPERM']); }); @@ -54,10 +54,20 @@ disposableRunnerTest('system-trusted local TLS still works without direct Keycha await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); try { const port = server.address().port; + const curlArgs = ['--fail', '--silent', '--show-error', `https://127.0.0.1:${port}/`]; + const control = await new Promise((resolve, reject) => { + const child = spawn('/usr/bin/curl', curlArgs); + let stdout = ''; + let stderr = ''; + child.stdout.on('data', (chunk) => { stdout += chunk; }); + child.stderr.on('data', (chunk) => { stderr += chunk; }); + child.on('error', reject); + child.on('close', (status) => resolve({ status, stdout, stderr })); + }); + expect(control).toEqual({ status: 0, stdout: 'tls-ok', stderr: '' }); const result = await new Promise((resolve, reject) => { const child = spawn('/usr/bin/sandbox-exec', [ - '-p', narrowedProfile(), '/usr/bin/curl', '--fail', '--silent', '--show-error', - `https://127.0.0.1:${port}/`, + '-p', narrowedProfile(), '/usr/bin/curl', ...curlArgs, ], { cwd: workspace }); let stdout = ''; let stderr = ''; From f35ab9de7dc214486de3b16b8950ec7246ba6f96 Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Sun, 18 Oct 2026 12:26:02 -0700 Subject: [PATCH 03/10] Probe literal System.keychain allowance under Seatbelt Co-authored-by: kai Agent-Model: codex/gpt-6-sol/medium Agent-Model: codex/gpt-6.1-sol/medium Agent-Model: claude/claude-haiku-5-5/default --- .../seatbelt.system-keychains.smoke.test.mjs | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/cli/__tests__/seatbelt.system-keychains.smoke.test.mjs b/cli/__tests__/seatbelt.system-keychains.smoke.test.mjs index 2a161ef989..09587ec1c3 100644 --- a/cli/__tests__/seatbelt.system-keychains.smoke.test.mjs +++ b/cli/__tests__/seatbelt.system-keychains.smoke.test.mjs @@ -1,7 +1,7 @@ import { spawn, spawnSync } from 'child_process'; import { createServer } from 'https'; import { - mkdirSync, mkdtempSync, readFileSync, rmSync, + mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync, } from 'fs'; import { homedir } from 'os'; import { join } from 'path'; @@ -15,14 +15,16 @@ const state = root && join(root, 'state'); if (root) { mkdirSync(workspace); mkdirSync(state); + writeFileSync(join(workspace, 'openssl.cnf'), ''); } afterAll(() => { if (root) rmSync(root, { recursive: true, force: true }); }); -// Measure the proposed deletion against a real Seatbelt profile before the +// Measure the narrow literal against a real Seatbelt profile before the // production baseline changes. Do not install a root on an operator machine. const narrowedProfile = () => buildSeatbeltProfile({ workspacePath: workspace, executablePath: process.execPath, statePath: state, -}).replace(' (subpath "/Library/Keychains")\n', ''); +}).replace(' (subpath "/Library/Keychains")\n', + ' (literal "/Library/Keychains/System.keychain")\n'); macTest('narrowed profile refuses system Keychains listing and legacy file opens', () => { const code = `const fs = require('fs'); @@ -37,10 +39,13 @@ macTest('narrowed profile refuses system Keychains listing and legacy file opens }`; const result = spawnSync('/usr/bin/sandbox-exec', [ '-p', narrowedProfile(), process.execPath, '-e', code, - ], { cwd: workspace, encoding: 'utf8', timeout: 10_000 }); + ], { + cwd: workspace, encoding: 'utf8', timeout: 10_000, + env: { ...process.env, OPENSSL_CONF: join(workspace, 'openssl.cnf') }, + }); expect(result.error).toBeUndefined(); expect({ status: result.status, stderr: result.stderr }).toEqual({ status: 0, stderr: '' }); - expect(result.stdout.trim().split('\n')).toEqual(['EPERM', 'EPERM', 'EPERM']); + expect(result.stdout.trim().split('\n')).toEqual(['EPERM', 'ALLOWED', 'EPERM']); }); const disposableRunnerTest = process.platform === 'darwin' From 264e4d731933a689c39d1bf84976cdca0a75b62c Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Sun, 18 Oct 2026 12:26:53 -0700 Subject: [PATCH 04/10] Compare broad and literal Keychain profiles on disposable macOS Co-authored-by: kai Agent-Model: codex/gpt-6-sol/medium Agent-Model: codex/gpt-6.1-sol/medium Agent-Model: claude/claude-haiku-5-5/default --- .github/workflows/seatbelt-system-keychains.yml | 7 +++++++ cli/__tests__/seatbelt.system-keychains.smoke.test.mjs | 9 +++++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/.github/workflows/seatbelt-system-keychains.yml b/.github/workflows/seatbelt-system-keychains.yml index c250d3590a..c62126c255 100644 --- a/.github/workflows/seatbelt-system-keychains.yml +++ b/.github/workflows/seatbelt-system-keychains.yml @@ -69,3 +69,10 @@ jobs: cd cli node --experimental-vm-modules node_modules/.bin/jest \ __tests__/seatbelt.system-keychains.smoke.test.mjs --runInBand + - name: Show Keychains sandbox denials + if: failure() + shell: bash + run: | + sudo log show --last 5m --style compact \ + --predicate 'eventMessage CONTAINS[c] "Keychains"' 2>/dev/null \ + | tail -80 || true diff --git a/cli/__tests__/seatbelt.system-keychains.smoke.test.mjs b/cli/__tests__/seatbelt.system-keychains.smoke.test.mjs index 09587ec1c3..75db97367b 100644 --- a/cli/__tests__/seatbelt.system-keychains.smoke.test.mjs +++ b/cli/__tests__/seatbelt.system-keychains.smoke.test.mjs @@ -70,9 +70,9 @@ disposableRunnerTest('system-trusted local TLS still works without direct Keycha child.on('close', (status) => resolve({ status, stdout, stderr })); }); expect(control).toEqual({ status: 0, stdout: 'tls-ok', stderr: '' }); - const result = await new Promise((resolve, reject) => { + const sandboxCurl = (profile) => new Promise((resolve, reject) => { const child = spawn('/usr/bin/sandbox-exec', [ - '-p', narrowedProfile(), '/usr/bin/curl', ...curlArgs, + '-p', profile, '/usr/bin/curl', ...curlArgs, ], { cwd: workspace }); let stdout = ''; let stderr = ''; @@ -81,6 +81,11 @@ disposableRunnerTest('system-trusted local TLS still works without direct Keycha child.on('error', reject); child.on('close', (status) => resolve({ status, stdout, stderr })); }); + const broad = await sandboxCurl(buildSeatbeltProfile({ + workspacePath: workspace, executablePath: process.execPath, statePath: state, + })); + expect(broad).toEqual({ status: 0, stdout: 'tls-ok', stderr: '' }); + const result = await sandboxCurl(narrowedProfile()); expect(result).toEqual({ status: 0, stdout: 'tls-ok', stderr: '' }); } finally { await new Promise((resolve) => server.close(resolve)); From 2f38d8926352125688eab3b4a63c98beee01c02d Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Sun, 18 Oct 2026 12:35:01 -0700 Subject: [PATCH 05/10] Remove system Keychains folder from Seatbelt baseline Co-authored-by: kai Agent-Model: codex/gpt-6-sol/medium Agent-Model: codex/gpt-6.1-sol/medium Agent-Model: claude/claude-haiku-5-5/default --- .../workflows/seatbelt-system-keychains.yml | 64 +++---------- .../seatbelt.system-keychains.smoke.test.mjs | 91 +++++-------------- cli/src/lib/sandbox/seatbelt.js | 1 - 3 files changed, 35 insertions(+), 121 deletions(-) diff --git a/.github/workflows/seatbelt-system-keychains.yml b/.github/workflows/seatbelt-system-keychains.yml index c62126c255..71f27e790b 100644 --- a/.github/workflows/seatbelt-system-keychains.yml +++ b/.github/workflows/seatbelt-system-keychains.yml @@ -11,7 +11,7 @@ permissions: contents: read jobs: - system-root: + adapter-turns: runs-on: macos-15 steps: - uses: actions/checkout@v4 @@ -21,58 +21,16 @@ jobs: cache: npm cache-dependency-path: cli/package-lock.json - run: npm ci --prefix cli - - name: Trust a disposable root and exercise the narrowed profile - shell: bash + - name: Install tested OpenCode for a disposable adapter turn + run: npm install --prefix "$RUNNER_TEMP/opencode-smoke" opencode-ai@1.19.0 + - name: Exercise system Keychains denials and real adapter turns + env: + COMMONLY_OPENCODE_SMOKE_BINARY: ${{ runner.temp }}/opencode-smoke/node_modules/.bin/opencode run: | - set -euo pipefail - cert_dir="$(mktemp -d)" - cat > "$cert_dir/root.cnf" <<'CONF' - [req] - distinguished_name = dn - x509_extensions = v3_ca - prompt = no - [dn] - CN = Commonly disposable Seatbelt root - [v3_ca] - basicConstraints = critical,CA:TRUE - keyUsage = critical,keyCertSign,cRLSign - subjectKeyIdentifier = hash - CONF - cat > "$cert_dir/leaf.cnf" <<'CONF' - [req] - distinguished_name = dn - req_extensions = v3_req - prompt = no - [dn] - CN = 127.0.0.1 - [v3_req] - basicConstraints = critical,CA:FALSE - keyUsage = critical,digitalSignature,keyEncipherment - extendedKeyUsage = serverAuth - subjectAltName = @alt_names - [alt_names] - IP.1 = 127.0.0.1 - CONF - openssl req -new -x509 -nodes -newkey rsa:2048 -sha256 -days 1 \ - -keyout "$cert_dir/root.key" -out "$cert_dir/root.pem" -config "$cert_dir/root.cnf" - openssl req -new -nodes -newkey rsa:2048 -sha256 \ - -keyout "$cert_dir/leaf.key" -out "$cert_dir/leaf.csr" -config "$cert_dir/leaf.cnf" - openssl x509 -req -in "$cert_dir/leaf.csr" -CA "$cert_dir/root.pem" \ - -CAkey "$cert_dir/root.key" -CAcreateserial -out "$cert_dir/leaf.pem" \ - -days 1 -sha256 -extfile "$cert_dir/leaf.cnf" -extensions v3_req - sudo security add-trusted-cert -d -r trustRoot \ - -k /Library/Keychains/System.keychain "$cert_dir/root.pem" - /usr/bin/curl --version | head -2 - security verify-cert -c "$cert_dir/leaf.pem" -p ssl -s 127.0.0.1 || true - export COMMONLY_DISPOSABLE_SYSTEM_ROOT_CERT="$cert_dir/leaf.pem" - export COMMONLY_DISPOSABLE_SYSTEM_ROOT_KEY="$cert_dir/leaf.key" cd cli node --experimental-vm-modules node_modules/.bin/jest \ - __tests__/seatbelt.system-keychains.smoke.test.mjs --runInBand - - name: Show Keychains sandbox denials - if: failure() - shell: bash - run: | - sudo log show --last 5m --style compact \ - --predicate 'eventMessage CONTAINS[c] "Keychains"' 2>/dev/null \ - | tail -80 || true + __tests__/seatbelt.system-keychains.smoke.test.mjs \ + __tests__/adapters.opencode.seatbelt.smoke.test.mjs \ + __tests__/adapters.qwen.seatbelt.smoke.test.mjs \ + __tests__/seatbelt.claude-runtime.smoke.test.mjs \ + --runInBand diff --git a/cli/__tests__/seatbelt.system-keychains.smoke.test.mjs b/cli/__tests__/seatbelt.system-keychains.smoke.test.mjs index 75db97367b..0cc184d254 100644 --- a/cli/__tests__/seatbelt.system-keychains.smoke.test.mjs +++ b/cli/__tests__/seatbelt.system-keychains.smoke.test.mjs @@ -1,8 +1,5 @@ -import { spawn, spawnSync } from 'child_process'; -import { createServer } from 'https'; -import { - mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync, -} from 'fs'; +import { spawnSync } from 'child_process'; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'fs'; import { homedir } from 'os'; import { join } from 'path'; import { buildSeatbeltProfile } from '../src/lib/sandbox/seatbelt.js'; @@ -19,75 +16,35 @@ if (root) { } afterAll(() => { if (root) rmSync(root, { recursive: true, force: true }); }); -// Measure the narrow literal against a real Seatbelt profile before the -// production baseline changes. Do not install a root on an operator machine. -const narrowedProfile = () => buildSeatbeltProfile({ - workspacePath: workspace, executablePath: process.execPath, statePath: state, -}).replace(' (subpath "/Library/Keychains")\n', - ' (literal "/Library/Keychains/System.keychain")\n'); +const run = (code) => spawnSync('/usr/bin/sandbox-exec', [ + '-p', buildSeatbeltProfile({ + workspacePath: workspace, executablePath: process.execPath, statePath: state, + }), process.execPath, '-e', code, +], { + cwd: workspace, encoding: 'utf8', timeout: 15_000, + env: { ...process.env, OPENSSL_CONF: join(workspace, 'openssl.cnf') }, +}); -macTest('narrowed profile refuses system Keychains listing and legacy file opens', () => { - const code = `const fs = require('fs'); - const paths = ['/Library/Keychains', '/Library/Keychains/System.keychain', - '/Library/Keychains/apsd.keychain']; - for (const path of paths) { +macTest('public Seatbelt denies listing and opening system Keychains', () => { + const result = run(`const fs = require('fs'); + for (const path of ['/Library/Keychains', '/Library/Keychains/System.keychain', + '/Library/Keychains/apsd.keychain']) { try { if (path.endsWith('/Keychains')) fs.readdirSync(path); else fs.closeSync(fs.openSync(path, 'r')); console.log('ALLOWED'); } catch (error) { console.log(error.code); } - }`; - const result = spawnSync('/usr/bin/sandbox-exec', [ - '-p', narrowedProfile(), process.execPath, '-e', code, - ], { - cwd: workspace, encoding: 'utf8', timeout: 10_000, - env: { ...process.env, OPENSSL_CONF: join(workspace, 'openssl.cnf') }, - }); + }`); expect(result.error).toBeUndefined(); expect({ status: result.status, stderr: result.stderr }).toEqual({ status: 0, stderr: '' }); - expect(result.stdout.trim().split('\n')).toEqual(['EPERM', 'ALLOWED', 'EPERM']); + expect(result.stdout.trim().split('\n')).toEqual(['EPERM', 'EPERM', 'EPERM']); }); -const disposableRunnerTest = process.platform === 'darwin' - && process.env.COMMONLY_DISPOSABLE_SYSTEM_ROOT_CERT - && process.env.COMMONLY_DISPOSABLE_SYSTEM_ROOT_KEY ? test : test.skip; - -disposableRunnerTest('system-trusted local TLS still works without direct Keychains reads', async () => { - const certificate = readFileSync(process.env.COMMONLY_DISPOSABLE_SYSTEM_ROOT_CERT); - const key = readFileSync(process.env.COMMONLY_DISPOSABLE_SYSTEM_ROOT_KEY); - const server = createServer({ cert: certificate, key }, (_req, response) => response.end('tls-ok')); - await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); - try { - const port = server.address().port; - const curlArgs = ['--fail', '--silent', '--show-error', `https://127.0.0.1:${port}/`]; - const control = await new Promise((resolve, reject) => { - const child = spawn('/usr/bin/curl', curlArgs); - let stdout = ''; - let stderr = ''; - child.stdout.on('data', (chunk) => { stdout += chunk; }); - child.stderr.on('data', (chunk) => { stderr += chunk; }); - child.on('error', reject); - child.on('close', (status) => resolve({ status, stdout, stderr })); - }); - expect(control).toEqual({ status: 0, stdout: 'tls-ok', stderr: '' }); - const sandboxCurl = (profile) => new Promise((resolve, reject) => { - const child = spawn('/usr/bin/sandbox-exec', [ - '-p', profile, '/usr/bin/curl', ...curlArgs, - ], { cwd: workspace }); - let stdout = ''; - let stderr = ''; - child.stdout.on('data', (chunk) => { stdout += chunk; }); - child.stderr.on('data', (chunk) => { stderr += chunk; }); - child.on('error', reject); - child.on('close', (status) => resolve({ status, stdout, stderr })); - }); - const broad = await sandboxCurl(buildSeatbeltProfile({ - workspacePath: workspace, executablePath: process.execPath, statePath: state, - })); - expect(broad).toEqual({ status: 0, stdout: 'tls-ok', stderr: '' }); - const result = await sandboxCurl(narrowedProfile()); - expect(result).toEqual({ status: 0, stdout: 'tls-ok', stderr: '' }); - } finally { - await new Promise((resolve) => server.close(resolve)); - } -}, 15_000); +macTest('Node can complete HTTPS with the system Keychains folder denied', () => { + const result = run(`fetch('https://example.com') + .then((response) => { console.log(response.status); }) + .catch((error) => { console.error(error.message); process.exitCode = 1; });`); + expect(result.error).toBeUndefined(); + expect({ status: result.status, stdout: result.stdout.trim(), stderr: result.stderr }) + .toEqual({ status: 0, stdout: '200', stderr: '' }); +}); diff --git a/cli/src/lib/sandbox/seatbelt.js b/cli/src/lib/sandbox/seatbelt.js index 33419115ea..17f8f51b2e 100644 --- a/cli/src/lib/sandbox/seatbelt.js +++ b/cli/src/lib/sandbox/seatbelt.js @@ -55,7 +55,6 @@ const PLATFORM_BASELINE = String.raw` (subpath "/usr/lib") (subpath "/usr/share") (subpath "/Library/Preferences") - (subpath "/Library/Keychains") (subpath "/var/db") (subpath "/private/var/db")) (allow file-read* file-test-existence From b6a02d5cd067d1de514153a13c9e3fa563be5718 Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Sun, 18 Oct 2026 12:36:33 -0700 Subject: [PATCH 06/10] Use accessible OpenSSL config in macOS smoke Co-authored-by: kai Agent-Model: codex/gpt-6-sol/medium Agent-Model: codex/gpt-6.1-sol/medium Agent-Model: claude/claude-haiku-5-5/default --- .github/workflows/seatbelt-system-keychains.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/seatbelt-system-keychains.yml b/.github/workflows/seatbelt-system-keychains.yml index 71f27e790b..a5455ed914 100644 --- a/.github/workflows/seatbelt-system-keychains.yml +++ b/.github/workflows/seatbelt-system-keychains.yml @@ -26,6 +26,7 @@ jobs: - name: Exercise system Keychains denials and real adapter turns env: COMMONLY_OPENCODE_SMOKE_BINARY: ${{ runner.temp }}/opencode-smoke/node_modules/.bin/opencode + OPENSSL_CONF: /dev/null run: | cd cli node --experimental-vm-modules node_modules/.bin/jest \ @@ -33,4 +34,4 @@ jobs: __tests__/adapters.opencode.seatbelt.smoke.test.mjs \ __tests__/adapters.qwen.seatbelt.smoke.test.mjs \ __tests__/seatbelt.claude-runtime.smoke.test.mjs \ - --runInBand + --runInBand --testTimeout=60000 From a466abd4cd55833f21e3a28218b18f69c4a2b452 Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Sun, 18 Oct 2026 12:38:24 -0700 Subject: [PATCH 07/10] Admit exact OpenSSL config and avoid CLI version collision Co-authored-by: kai Agent-Model: codex/gpt-6-sol/medium Agent-Model: codex/gpt-6.1-sol/medium Agent-Model: claude/claude-haiku-5-5/default --- cli/src/lib/sandbox/seatbelt.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cli/src/lib/sandbox/seatbelt.js b/cli/src/lib/sandbox/seatbelt.js index 17f8f51b2e..780c929c48 100644 --- a/cli/src/lib/sandbox/seatbelt.js +++ b/cli/src/lib/sandbox/seatbelt.js @@ -45,7 +45,8 @@ const PLATFORM_BASELINE = String.raw` (path "/dev/null") (vnode-type CHARACTER-DEVICE))) -; Loader + standard system resources. +; Loader + standard system resources. Some Node builds read this one macOS +; OpenSSL config file before the CLI entrypoint runs. (allow file-read* file-test-existence (subpath "/Library/Apple") (subpath "/Library/Filesystems/NetFSPlugins") @@ -54,6 +55,7 @@ const PLATFORM_BASELINE = String.raw` (subpath "/private/var/db/timezone") (subpath "/usr/lib") (subpath "/usr/share") + (literal "/System/Library/OpenSSL/openssl.cnf") (subpath "/Library/Preferences") (subpath "/var/db") (subpath "/private/var/db")) From 7c8b681bedf8bc84771090403795c5edac3b9544 Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Sun, 18 Oct 2026 12:46:58 -0700 Subject: [PATCH 08/10] Exercise OpenSSL config path in macOS adapter smoke Co-authored-by: kai Agent-Model: codex/gpt-6-sol/medium Agent-Model: codex/gpt-6.1-sol/medium Agent-Model: claude/claude-haiku-5-5/default --- .github/workflows/seatbelt-system-keychains.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/seatbelt-system-keychains.yml b/.github/workflows/seatbelt-system-keychains.yml index a5455ed914..c2dba8667f 100644 --- a/.github/workflows/seatbelt-system-keychains.yml +++ b/.github/workflows/seatbelt-system-keychains.yml @@ -26,7 +26,6 @@ jobs: - name: Exercise system Keychains denials and real adapter turns env: COMMONLY_OPENCODE_SMOKE_BINARY: ${{ runner.temp }}/opencode-smoke/node_modules/.bin/opencode - OPENSSL_CONF: /dev/null run: | cd cli node --experimental-vm-modules node_modules/.bin/jest \ From 8ed79a9aa5679d1c40cd2a689e49f360d5a0bfe7 Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Fri, 30 Oct 2026 12:36:37 -0700 Subject: [PATCH 09/10] test(seatbelt): cover the Claude profile and its credential read (TASK-233) The system-Keychains denial now runs under the Claude runtime profile too, so a grant added to claudeRuntimeRules fails a test (Otto's mutation on eca2d8d79 passes today; this makes it red). A disposable-runner probe reads a generic-password item from the login keychain through /usr/bin/security under the Claude profile with System.keychain denied, with the old broad grant as a control. It runs only when COMMONLY_KEYCHAIN_PROBE=1. Also: checkout does not persist credentials, and the OpenSSL comment names the observed denial. Co-Authored-By: Claude Haiku 5.5 Co-authored-by: kai Agent-Model: claude/claude-haiku-5-5/default --- .../workflows/seatbelt-system-keychains.yml | 6 ++ ...atbelt.claude-keychain-read.smoke.test.mjs | 77 +++++++++++++++++++ .../seatbelt.system-keychains.smoke.test.mjs | 40 ++++++---- cli/src/lib/sandbox/seatbelt.js | 4 +- 4 files changed, 110 insertions(+), 17 deletions(-) create mode 100644 cli/__tests__/seatbelt.claude-keychain-read.smoke.test.mjs diff --git a/.github/workflows/seatbelt-system-keychains.yml b/.github/workflows/seatbelt-system-keychains.yml index c2dba8667f..b28a089f06 100644 --- a/.github/workflows/seatbelt-system-keychains.yml +++ b/.github/workflows/seatbelt-system-keychains.yml @@ -5,6 +5,7 @@ on: paths: - 'cli/src/lib/sandbox/seatbelt.js' - 'cli/__tests__/seatbelt.system-keychains.smoke.test.mjs' + - 'cli/__tests__/seatbelt.claude-keychain-read.smoke.test.mjs' - '.github/workflows/seatbelt-system-keychains.yml' permissions: @@ -15,6 +16,8 @@ jobs: runs-on: macos-15 steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - uses: actions/setup-node@v4 with: node-version: '22' @@ -26,10 +29,13 @@ jobs: - name: Exercise system Keychains denials and real adapter turns env: COMMONLY_OPENCODE_SMOKE_BINARY: ${{ runner.temp }}/opencode-smoke/node_modules/.bin/opencode + # Writes a probe item into this disposable runner's login keychain. + COMMONLY_KEYCHAIN_PROBE: '1' run: | cd cli node --experimental-vm-modules node_modules/.bin/jest \ __tests__/seatbelt.system-keychains.smoke.test.mjs \ + __tests__/seatbelt.claude-keychain-read.smoke.test.mjs \ __tests__/adapters.opencode.seatbelt.smoke.test.mjs \ __tests__/adapters.qwen.seatbelt.smoke.test.mjs \ __tests__/seatbelt.claude-runtime.smoke.test.mjs \ diff --git a/cli/__tests__/seatbelt.claude-keychain-read.smoke.test.mjs b/cli/__tests__/seatbelt.claude-keychain-read.smoke.test.mjs new file mode 100644 index 0000000000..2127abce54 --- /dev/null +++ b/cli/__tests__/seatbelt.claude-keychain-read.smoke.test.mjs @@ -0,0 +1,77 @@ +import { spawnSync } from 'child_process'; +import { mkdirSync, mkdtempSync, rmSync } from 'fs'; +import { homedir, tmpdir } from 'os'; +import { join } from 'path'; +import { randomBytes } from 'crypto'; +import { buildSeatbeltProfile } from '../src/lib/sandbox/seatbelt.js'; + +/** + * Does Claude's credential read survive the removal of the system Keychains + * folder from the baseline? Claude reads its OAuth item by running + * /usr/bin/security, whose default search list includes System.keychain. + * If Security.framework treats a denied search-list keychain as fatal, every + * public Claude seat would fail to log in after the change ships (TASK-233, + * Otto's Ask 2 on eca2d8d79). + * + * This writes a probe item into the login keychain, so it runs only when the + * disposable macOS job sets COMMONLY_KEYCHAIN_PROBE=1. It never runs on an + * operator host. + */ +const probeEnabled = process.platform === 'darwin' && process.env.COMMONLY_KEYCHAIN_PROBE === '1'; +const macProbe = probeEnabled ? test : test.skip; + +const SERVICE = 'commonly-seatbelt-probe'; +const loginKeychain = join(homedir(), 'Library', 'Keychains', 'login.keychain-db'); +const secret = randomBytes(16).toString('hex'); +const root = probeEnabled ? mkdtempSync(join(tmpdir(), 'commonly-keychain-probe-')) : null; +const workspace = root && join(root, 'workspace'); +const state = root && join(root, 'state'); +if (root) { + mkdirSync(workspace); + mkdirSync(state); +} + +beforeAll(() => { + if (!probeEnabled) return; + const added = spawnSync('/usr/bin/security', [ + 'add-generic-password', '-U', '-s', SERVICE, '-a', 'commonly', '-w', secret, loginKeychain, + ], { encoding: 'utf8' }); + if (added.status !== 0) throw new Error(`could not write the probe item: ${added.stderr}`); +}); + +afterAll(() => { + if (!probeEnabled) return; + spawnSync('/usr/bin/security', ['delete-generic-password', '-s', SERVICE, loginKeychain], { encoding: 'utf8' }); + rmSync(root, { recursive: true, force: true }); +}); + +// A confined Node spawns security, as the Claude CLI does. The profile is the +// Claude runtime profile, or that profile plus one extra rule for a control. +const readProbe = (extraRules = '') => { + const profile = `${buildSeatbeltProfile({ + workspacePath: workspace, executablePath: process.execPath, statePath: state, allowClaudeRuntimeAccess: true, + })}\n${extraRules}`; + return spawnSync('/usr/bin/sandbox-exec', [ + '-p', profile, process.execPath, '-e', + `const { spawnSync } = require('child_process'); + const r = spawnSync('/usr/bin/security', ['find-generic-password', '-s', '${SERVICE}', '-w'], { encoding: 'utf8' }); + process.stdout.write(String(r.stdout).trim()); process.stderr.write(String(r.stderr).trim()); process.exitCode = r.status;`, + ], { cwd: workspace, encoding: 'utf8', timeout: 20_000 }); +}; + +macProbe('the probe item is readable outside any sandbox (precondition)', () => { + const result = spawnSync('/usr/bin/security', ['find-generic-password', '-s', SERVICE, '-w'], { encoding: 'utf8' }); + expect({ status: result.status, stdout: result.stdout.trim() }).toEqual({ status: 0, stdout: secret }); +}); + +macProbe('the Claude runtime profile still reads its credential with System.keychain denied', () => { + const result = readProbe(); + expect({ status: result.status, stdout: result.stdout, stderr: result.stderr }).toEqual({ + status: 0, stdout: secret, stderr: '', + }); +}); + +macProbe('control: the old broad Keychains folder grant also reads it', () => { + const result = readProbe('(allow file-read* (subpath "/Library/Keychains"))'); + expect({ status: result.status, stdout: result.stdout }).toEqual({ status: 0, stdout: secret }); +}); diff --git a/cli/__tests__/seatbelt.system-keychains.smoke.test.mjs b/cli/__tests__/seatbelt.system-keychains.smoke.test.mjs index 0cc184d254..16d5b823cd 100644 --- a/cli/__tests__/seatbelt.system-keychains.smoke.test.mjs +++ b/cli/__tests__/seatbelt.system-keychains.smoke.test.mjs @@ -16,28 +16,38 @@ if (root) { } afterAll(() => { if (root) rmSync(root, { recursive: true, force: true }); }); -const run = (code) => spawnSync('/usr/bin/sandbox-exec', [ +// Two profiles reach the Keychain, and only the Claude one passes this flag. +// The denial must hold under both, or a grant added to claudeRuntimeRules +// would go unnoticed (TASK-233, Otto's mutation on eca2d8d79). +const PROFILES = [ + { label: 'public', options: {} }, + { label: 'Claude runtime', options: { allowClaudeRuntimeAccess: true } }, +]; + +const run = (code, options = {}) => spawnSync('/usr/bin/sandbox-exec', [ '-p', buildSeatbeltProfile({ - workspacePath: workspace, executablePath: process.execPath, statePath: state, + workspacePath: workspace, executablePath: process.execPath, statePath: state, ...options, }), process.execPath, '-e', code, ], { cwd: workspace, encoding: 'utf8', timeout: 15_000, env: { ...process.env, OPENSSL_CONF: join(workspace, 'openssl.cnf') }, }); -macTest('public Seatbelt denies listing and opening system Keychains', () => { - const result = run(`const fs = require('fs'); - for (const path of ['/Library/Keychains', '/Library/Keychains/System.keychain', - '/Library/Keychains/apsd.keychain']) { - try { - if (path.endsWith('/Keychains')) fs.readdirSync(path); - else fs.closeSync(fs.openSync(path, 'r')); - console.log('ALLOWED'); - } catch (error) { console.log(error.code); } - }`); - expect(result.error).toBeUndefined(); - expect({ status: result.status, stderr: result.stderr }).toEqual({ status: 0, stderr: '' }); - expect(result.stdout.trim().split('\n')).toEqual(['EPERM', 'EPERM', 'EPERM']); +describe.each(PROFILES)('$label Seatbelt', ({ options }) => { + macTest('denies listing and opening system Keychains', () => { + const result = run(`const fs = require('fs'); + for (const path of ['/Library/Keychains', '/Library/Keychains/System.keychain', + '/Library/Keychains/apsd.keychain']) { + try { + if (path.endsWith('/Keychains')) fs.readdirSync(path); + else fs.closeSync(fs.openSync(path, 'r')); + console.log('ALLOWED'); + } catch (error) { console.log(error.code); } + }`, options); + expect(result.error).toBeUndefined(); + expect({ status: result.status, stderr: result.stderr }).toEqual({ status: 0, stderr: '' }); + expect(result.stdout.trim().split('\n')).toEqual(['EPERM', 'EPERM', 'EPERM']); + }); }); macTest('Node can complete HTTPS with the system Keychains folder denied', () => { diff --git a/cli/src/lib/sandbox/seatbelt.js b/cli/src/lib/sandbox/seatbelt.js index 780c929c48..d0cbbee421 100644 --- a/cli/src/lib/sandbox/seatbelt.js +++ b/cli/src/lib/sandbox/seatbelt.js @@ -45,8 +45,8 @@ const PLATFORM_BASELINE = String.raw` (path "/dev/null") (vnode-type CHARACTER-DEVICE))) -; Loader + standard system resources. Some Node builds read this one macOS -; OpenSSL config file before the CLI entrypoint runs. +; Loader + standard system resources. Node opens this one macOS OpenSSL config +; file at startup; the macOS adapter job's Node was denied it without this rule. (allow file-read* file-test-existence (subpath "/Library/Apple") (subpath "/Library/Filesystems/NetFSPlugins") From f87b3d143eb3dc35caa5f1c8e263ed60b02129f8 Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Fri, 30 Oct 2026 12:45:08 -0700 Subject: [PATCH 10/10] test(seatbelt): exercise the OpenSSL config grant without an override (TASK-233) The HTTPS smoke pointed OPENSSL_CONF at an empty workspace file, so Node never read /System/Library/OpenSSL/openssl.cnf and removing the literal grant could leave it green. Drop the override and add a sandboxed read of that file under both profiles. Dropping the grant turns it red. Co-Authored-By: Claude Haiku 5.5 Co-authored-by: kai Agent-Model: claude/claude-haiku-5-5/default --- .../seatbelt.system-keychains.smoke.test.mjs | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/cli/__tests__/seatbelt.system-keychains.smoke.test.mjs b/cli/__tests__/seatbelt.system-keychains.smoke.test.mjs index 16d5b823cd..8dfc35440c 100644 --- a/cli/__tests__/seatbelt.system-keychains.smoke.test.mjs +++ b/cli/__tests__/seatbelt.system-keychains.smoke.test.mjs @@ -1,5 +1,5 @@ import { spawnSync } from 'child_process'; -import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'fs'; +import { mkdirSync, mkdtempSync, rmSync } from 'fs'; import { homedir } from 'os'; import { join } from 'path'; import { buildSeatbeltProfile } from '../src/lib/sandbox/seatbelt.js'; @@ -12,7 +12,6 @@ const state = root && join(root, 'state'); if (root) { mkdirSync(workspace); mkdirSync(state); - writeFileSync(join(workspace, 'openssl.cnf'), ''); } afterAll(() => { if (root) rmSync(root, { recursive: true, force: true }); }); @@ -30,7 +29,6 @@ const run = (code, options = {}) => spawnSync('/usr/bin/sandbox-exec', [ }), process.execPath, '-e', code, ], { cwd: workspace, encoding: 'utf8', timeout: 15_000, - env: { ...process.env, OPENSSL_CONF: join(workspace, 'openssl.cnf') }, }); describe.each(PROFILES)('$label Seatbelt', ({ options }) => { @@ -48,6 +46,16 @@ describe.each(PROFILES)('$label Seatbelt', ({ options }) => { expect({ status: result.status, stderr: result.stderr }).toEqual({ status: 0, stderr: '' }); expect(result.stdout.trim().split('\n')).toEqual(['EPERM', 'EPERM', 'EPERM']); }); + + // Node reads this file at startup with no OPENSSL_CONF override, so the + // literal grant in PLATFORM_BASELINE must let the read through (TASK-233). + macTest('reads the system OpenSSL config that Node loads at startup', () => { + const result = run(`const fs = require('fs'); + try { fs.readFileSync('/System/Library/OpenSSL/openssl.cnf'); console.log('READ'); } + catch (error) { console.log(error.code); }`, options); + expect(result.error).toBeUndefined(); + expect(result.stdout.trim()).toBe('READ'); + }); }); macTest('Node can complete HTTPS with the system Keychains folder denied', () => {