diff --git a/apps/sim/lib/credentials/secret-values.test.ts b/apps/sim/lib/credentials/secret-values.test.ts index 5193d1daf5f..59ba38e50e6 100644 --- a/apps/sim/lib/credentials/secret-values.test.ts +++ b/apps/sim/lib/credentials/secret-values.test.ts @@ -16,13 +16,11 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' vi.mock('@/lib/core/security/encryption', () => encryptionMock) const mockEncryptSecret = encryptionMockFns.mockEncryptSecret -const mockDecryptSecret = encryptionMockFns.mockDecryptSecret vi.mock('@/lib/credentials/environment', () => credentialsEnvironmentMock) import { deletePersonalSecret, deleteWorkspaceSecret, - readWorkspaceSecretValues, setWorkspaceSecret, updateWorkspaceSecretMetadata, } from '@/lib/credentials/secret-values' @@ -116,51 +114,6 @@ describe('secret value storage', () => { }) }) -describe('readWorkspaceSecretValues', () => { - beforeEach(() => { - resetDbChainMock() - mockDecryptSecret.mockImplementation(async (encrypted: string) => ({ - decrypted: `decrypted:${encrypted}`, - })) - }) - - it('decrypts only the requested names and omits absent or undecryptable ones', async () => { - queueTableRows(schemaMock.workspaceEnvironment, [ - { - id: 'env-1', - variables: { - VISIBLE_KEY: 'encrypted-visible', - BROKEN_KEY: 'encrypted-broken', - OTHER_KEY: 'encrypted-other', - }, - }, - ]) - mockDecryptSecret.mockImplementation(async (encrypted: string) => { - if (encrypted === 'encrypted-broken') throw new Error('cannot decrypt') - return { decrypted: `decrypted:${encrypted}` } - }) - - await expect( - readWorkspaceSecretValues({ - workspaceId: 'workspace-1', - names: ['VISIBLE_KEY', 'BROKEN_KEY', 'MISSING_KEY'], - }) - ).resolves.toEqual({ VISIBLE_KEY: 'decrypted:encrypted-visible' }) - expect(mockDecryptSecret).not.toHaveBeenCalledWith('encrypted-other') - }) - - it('never reads an inherited prototype member for a missing key', async () => { - queueTableRows(schemaMock.workspaceEnvironment, [ - { id: 'env-1', variables: { OTHER_KEY: 'encrypted-other' } }, - ]) - - await expect( - readWorkspaceSecretValues({ workspaceId: 'workspace-1', names: ['constructor', 'toString'] }) - ).resolves.toEqual({}) - expect(mockDecryptSecret).not.toHaveBeenCalled() - }) -}) - /** * The row-queue mocks resolve whatever was queued regardless of the predicate, so * the only way to pin a WHERE clause is to read the condition tree the `eq`/`and` diff --git a/apps/sim/lib/credentials/secret-values.ts b/apps/sim/lib/credentials/secret-values.ts index 9d73cf6a629..c1fe08c85e4 100644 --- a/apps/sim/lib/credentials/secret-values.ts +++ b/apps/sim/lib/credentials/secret-values.ts @@ -2,7 +2,7 @@ import { db } from '@sim/db' import { credential, environment, workspaceEnvironment } from '@sim/db/schema' import { generateId } from '@sim/utils/id' import { and, eq } from 'drizzle-orm' -import { decryptSecret, encryptSecret } from '@/lib/core/security/encryption' +import { encryptSecret } from '@/lib/core/security/encryption' import { lockPersonalEnvMap, lockWorkspaceEnvMap } from '@/lib/credentials/env-locks' import { createWorkspaceEnvCredentials, @@ -17,44 +17,6 @@ export interface SecretMutationResult { updatedAt: Date } -/** - * Decrypts the stored values for the requested workspace secret names. - * - * Exists for exactly one read path: rows a workspace marked visible (unredacted), - * whose values already print into every run log the caller can open. Every other - * secret read stays metadata-only — callers gate on the flag BEFORE asking. A - * name that is absent or fails to decrypt is omitted rather than failing the - * batch, since the value is optional on the wire. - */ -export async function readWorkspaceSecretValues(params: { - workspaceId: string - names: readonly string[] -}): Promise> { - if (params.names.length === 0) return {} - - const [row] = await db - .select({ variables: workspaceEnvironment.variables }) - .from(workspaceEnvironment) - .where(eq(workspaceEnvironment.workspaceId, params.workspaceId)) - .limit(1) - const variables = (row?.variables as Record | null) ?? {} - - const values: Record = {} - await Promise.all( - params.names.map(async (name) => { - const encrypted = Object.hasOwn(variables, name) ? variables[name] : undefined - if (!encrypted) return - try { - const { decrypted } = await decryptSecret(encrypted) - values[name] = decrypted - } catch { - // Omitted from the result; the caller's wire shape treats the value as optional. - } - }) - ) - return values -} - /** Stores one workspace secret without decrypting any existing value. */ export async function setWorkspaceSecret(params: { workspaceId: string diff --git a/apps/sim/lib/execution/remote-sandbox/execution-observer.ts b/apps/sim/lib/execution/remote-sandbox/execution-observer.ts index b5251dde53f..5eb9144a49b 100644 --- a/apps/sim/lib/execution/remote-sandbox/execution-observer.ts +++ b/apps/sim/lib/execution/remote-sandbox/execution-observer.ts @@ -1,8 +1,9 @@ import { AsyncLocalStorage } from 'node:async_hooks' +import type { DurableSecretProvenance } from '@/lib/execution/durable-secret-provenance' import type { SessionProcessIdentity } from '@/lib/execution/remote-sandbox/session-process' interface SandboxExecutionObserver { - sessionInputsSafe?(): boolean + sessionInputProvenance?(): boolean | DurableSecretProvenance hold(work: Promise): void unsettled(processId?: string): void claimProcess?(process: SessionProcessIdentity): Promise @@ -49,20 +50,25 @@ export async function prepareSandboxSessionAccess( } /** The trusted tool adapter supplies current input evidence while preserving execution ownership. */ -export function observeSandboxSessionInputs(safe: () => boolean, execute: () => T): T { +export function observeSandboxSessionInputs( + safe: () => boolean | DurableSecretProvenance, + execute: () => T +): T { const current = executionObserver.getStore() return executionObserver.run( { hold: (work) => current?.hold(work), unsettled: (id) => current?.unsettled(id), ...current, - sessionInputsSafe: safe, + sessionInputProvenance: safe, }, execute ) } -/** Unobserved arbitrary code cannot certify scratch files as safe. */ -export function sandboxSessionInputsSafe(): boolean { - return executionObserver.getStore()?.sessionInputsSafe?.() === true +/** Trusted input evidence is sampled immediately before the machine receives the bytes. */ +export function sandboxSessionInputProvenance(): DurableSecretProvenance { + const value = executionObserver.getStore()?.sessionInputProvenance?.() + if (typeof value === 'object') return value + return value === true ? { status: 'exact', entries: [] } : { status: 'unknown' } } diff --git a/apps/sim/lib/execution/remote-sandbox/index.ts b/apps/sim/lib/execution/remote-sandbox/index.ts index f3ea137224a..c4aef7154b2 100644 --- a/apps/sim/lib/execution/remote-sandbox/index.ts +++ b/apps/sim/lib/execution/remote-sandbox/index.ts @@ -20,7 +20,7 @@ import { prepareSandboxSessionAccess, reportUnsettledSandboxProcess, retainSandboxExecution, - sandboxSessionInputsSafe, + sandboxSessionInputProvenance, } from '@/lib/execution/remote-sandbox/execution-observer' import { withSandboxFilePublication } from '@/lib/execution/remote-sandbox/file-publication' import { @@ -55,7 +55,10 @@ import { SESSION_SANDBOX_IDLE_MS, } from '@/lib/execution/remote-sandbox/session' import { sessionCommandPath } from '@/lib/execution/remote-sandbox/session-cli' -import { recordSessionFileInput } from '@/lib/execution/remote-sandbox/session-file-provenance' +import { + readSessionSecretProvenance, + recordSessionFileInput, +} from '@/lib/execution/remote-sandbox/session-file-provenance' import { withSandboxSessionLock } from '@/lib/execution/remote-sandbox/session-lock' import type { CreateSandboxOptions, @@ -842,6 +845,18 @@ async function provisionWithinBudget( throwIfAborted(signal) } +/** Confidentiality checks also run on provider failures, before their diagnostics can escape. */ +async function acceptSessionOutputHistory( + session: SandboxSessionRequest | undefined, + machine: { providerId: SandboxProviderId; sandboxId: string } +): Promise { + if (!session) return + const provenance = await readSessionSecretProvenance(session.key, machine) + if (session.acceptOutputProvenance) await session.acceptOutputProvenance(provenance) + else if (provenance.status !== 'exact' || provenance.entries.length > 0) + throw new Error('Workbench output withheld because its secret provenance is unavailable') +} + async function executeInSandboxWithinBudget( // The budget wrapper always injects the signal; the required-signal type states that // invariant instead of a cast hiding it. @@ -887,13 +902,13 @@ async function executeInSandboxWithinBudget( // the finally below. Dependencies land before the inputs so user code and its // mounts always see a complete environment. // - if (req.session) + if (lease.session && req.session) await recordSessionFileInput( req.session.key, { providerId: created.providerId, sandboxId }, - sandboxSessionInputsSafe() && - !req.session.unprovenancedInputs && - !Object.keys(selected?.envs ?? {}).length + req.session.unprovenancedInputs || Object.keys(selected?.envs ?? {}).length + ? { status: 'unknown' } + : (req.session.inputProvenance?.() ?? sandboxSessionInputProvenance()) ) await provisionWithinBudget(sandbox, selected, signal) await writeSandboxInputs(sandbox, req.sandboxFiles, { @@ -1025,8 +1040,15 @@ async function executeInSandboxWithinBudget( if (cost && billableOutputError) { attachTrustedSandboxOutputCost(billableOutputError, cost) } - await privateInputFiles?.cleanup() - await lease.release() + try { + await privateInputFiles?.cleanup() + await lease.release() + } finally { + await acceptSessionOutputHistory(lease.session ? req.session : undefined, { + providerId: created.providerId, + sandboxId, + }) + } } } @@ -1076,13 +1098,13 @@ async function executeShellInSandboxWithinBudget( // Inside the try so a failed install or mount still releases the sandbox via // the finally below. The install shares the caller's budget rather than adding // to it — see the note in `executeInSandbox`. - if (req.session) + if (lease.session && req.session) await recordSessionFileInput( req.session.key, { providerId: created.providerId, sandboxId }, - sandboxSessionInputsSafe() && - !req.session.unprovenancedInputs && - !Object.keys(selected?.envs ?? {}).length + req.session.unprovenancedInputs || Object.keys(selected?.envs ?? {}).length + ? { status: 'unknown' } + : (req.session.inputProvenance?.() ?? sandboxSessionInputProvenance()) ) await provisionWithinBudget(sandbox, selected, signal) await writeSandboxInputs(sandbox, req.sandboxFiles, { @@ -1192,8 +1214,15 @@ async function executeShellInSandboxWithinBudget( if (cost && billableOutputError) { attachTrustedSandboxOutputCost(billableOutputError, cost) } - await privateInputFiles?.cleanup() - await lease.release() + try { + await privateInputFiles?.cleanup() + await lease.release() + } finally { + await acceptSessionOutputHistory(lease.session ? req.session : undefined, { + providerId: created.providerId, + sandboxId, + }) + } } } diff --git a/apps/sim/lib/execution/remote-sandbox/session-file-provenance.integration.ts b/apps/sim/lib/execution/remote-sandbox/session-file-provenance.integration.ts new file mode 100644 index 00000000000..ca71441e108 --- /dev/null +++ b/apps/sim/lib/execution/remote-sandbox/session-file-provenance.integration.ts @@ -0,0 +1,150 @@ +import { createHash } from 'node:crypto' +import { generateShortId } from '@sim/utils/id' +import { afterAll, describe, expect, it, vi } from 'vitest' + +const { redisUrl } = await vi.hoisted(async () => { + const { readTestRedisUrl } = await import('@sim/db/testing/test-infrastructure') + const redisUrl = readTestRedisUrl() + if (redisUrl) process.env.REDIS_URL = redisUrl + return { redisUrl } +}) + +import { closeRedisConnection, getRedisClient } from '@/lib/core/config/redis' +import { encryptSecret } from '@/lib/core/security/encryption' +import { + PROVENANCE_MAX_ENTRIES, + PROVENANCE_MAX_SERIALIZED_BYTES, +} from '@/lib/execution/provenance-limits' +import { + initializeSessionFileProvenance, + isSessionFileProvenanceClean, + readSessionSecretProvenance, + recordSessionFileInput, +} from '@/lib/execution/remote-sandbox/session-file-provenance' + +const keys: string[] = [] +function fixture() { + const session = `history-${generateShortId(16)}` + const machine = { providerId: 'e2b' as const, sandboxId: generateShortId(16) } + const key = `mothership:workbench-provenance:v2:${createHash('sha256') + .update(JSON.stringify([session, machine.providerId, machine.sandboxId])) + .digest('hex')}` + keys.push(key) + return { session, machine, key } +} +afterAll(async () => { + if (keys.length) await getRedisClient()?.del(...keys) + await closeRedisConnection() +}) + +describe.skipIf(!redisUrl)('physical workbench history with real Redis', () => { + it('atomically retains distinct encrypted inputs under concurrent calls and retries', async () => { + const { session, machine } = fixture() + await initializeSessionFileProvenance(session, machine) + const entries = await Promise.all( + Array.from({ length: 32 }, async (_, index) => ({ + name: `TOKEN_${index}`, + encryptedValue: (await encryptSecret(`synthetic-token-value-${index}`)).encrypted, + })) + ) + await Promise.all( + entries.map((entry) => + recordSessionFileInput(session, machine, { status: 'exact', entries: [entry] }) + ) + ) + await recordSessionFileInput(session, machine, { status: 'exact', entries }) + await recordSessionFileInput(session, machine, true) + await initializeSessionFileProvenance(session, machine) + const history = await readSessionSecretProvenance(session, machine) + expect(history.status).toBe('exact') + if (history.status !== 'exact') throw new Error('Expected exact history') + expect(history.entries).toHaveLength(32) + expect(JSON.stringify(history)).not.toContain('synthetic-token-value-') + expect(await isSessionFileProvenanceClean(session, machine)).toBe(false) + }) + it('keeps unknown permanent and never initializes recovered or expired history from current inputs', async () => { + const { session, machine, key } = fixture() + await recordSessionFileInput(session, machine, true) + await initializeSessionFileProvenance(session, machine) + expect(await readSessionSecretProvenance(session, machine)).toEqual({ status: 'unknown' }) + await getRedisClient()!.del(key) + await recordSessionFileInput(session, machine, true) + expect(await readSessionSecretProvenance(session, machine)).toEqual({ status: 'unknown' }) + }) + it('does not trust legacy history or another chat or physical machine', async () => { + const { session, machine, key } = fixture() + const legacy = key.replace(':v2:', ':v1:') + keys.push(legacy) + await getRedisClient()!.set(legacy, 'clean') + expect(await isSessionFileProvenanceClean(session, machine)).toBe(false) + await initializeSessionFileProvenance(session, machine) + expect(await isSessionFileProvenanceClean(session, machine)).toBe(true) + expect(await isSessionFileProvenanceClean(`${session}-other`, machine)).toBe(false) + expect( + await isSessionFileProvenanceClean(session, { ...machine, sandboxId: 'replacement' }) + ).toBe(false) + }) + it.each(['{"status":"exact","entries":{}}', '{"status":"exact","entries":[{}]}', 'not-json'])( + 'fails closed on malformed stored history %s', + async (value) => { + const { session, machine, key } = fixture() + await getRedisClient()!.set(key, value) + await recordSessionFileInput(session, machine, true) + expect(await readSessionSecretProvenance(session, machine)).toEqual({ status: 'unknown' }) + } + ) + it('folds many source bindings into one machine secret without spending the distinct-secret budget', async () => { + const { session, machine } = fixture() + await initializeSessionFileProvenance(session, machine) + const entries = Array.from({ length: PROVENANCE_MAX_ENTRIES + 1 }, (_, index) => ({ + encryptedValue: 'one-encrypted-secret', + sourceValueHash: `source-${index}`, + })) + await recordSessionFileInput(session, machine, { status: 'exact', entries }) + expect(await readSessionSecretProvenance(session, machine)).toEqual({ + status: 'exact', + entries: [{ encryptedValue: 'one-encrypted-secret' }], + }) + }) + it('makes cumulative entry overflow permanently unknown', async () => { + const { session, machine } = fixture() + await initializeSessionFileProvenance(session, machine) + const entries = Array.from({ length: PROVENANCE_MAX_ENTRIES }, (_, index) => ({ + encryptedValue: `bounded-ciphertext-${index}`, + })) + await recordSessionFileInput(session, machine, { status: 'exact', entries }) + expect((await readSessionSecretProvenance(session, machine)).status).toBe('exact') + await recordSessionFileInput(session, machine, { + status: 'exact', + entries: [{ encryptedValue: 'additional-distinct-ciphertext' }], + }) + await recordSessionFileInput(session, machine, true) + expect(await readSessionSecretProvenance(session, machine)).toEqual({ status: 'unknown' }) + }) + it('makes cumulative byte overflow permanently unknown even when each receipt fits', async () => { + const { session, machine } = fixture() + await initializeSessionFileProvenance(session, machine) + const halfBudget = PROVENANCE_MAX_SERIALIZED_BYTES / 2 + await recordSessionFileInput(session, machine, { + status: 'exact', + entries: [{ encryptedValue: 'a'.repeat(halfBudget) }], + }) + expect((await readSessionSecretProvenance(session, machine)).status).toBe('exact') + await recordSessionFileInput(session, machine, { + status: 'exact', + entries: [{ encryptedValue: 'b'.repeat(halfBudget) }], + }) + await recordSessionFileInput(session, machine, true) + expect(await readSessionSecretProvenance(session, machine)).toEqual({ status: 'unknown' }) + }) + it('withholds classification when an encrypted history exceeds its byte budget', async () => { + const { session, machine } = fixture() + await initializeSessionFileProvenance(session, machine) + await recordSessionFileInput(session, machine, { + status: 'exact', + entries: [{ encryptedValue: 'x'.repeat(PROVENANCE_MAX_SERIALIZED_BYTES) }], + }) + await recordSessionFileInput(session, machine, true) + expect(await readSessionSecretProvenance(session, machine)).toEqual({ status: 'unknown' }) + }) +}) diff --git a/apps/sim/lib/execution/remote-sandbox/session-file-provenance.test.ts b/apps/sim/lib/execution/remote-sandbox/session-file-provenance.test.ts index 7346d255c6e..e55f10fd094 100644 --- a/apps/sim/lib/execution/remote-sandbox/session-file-provenance.test.ts +++ b/apps/sim/lib/execution/remote-sandbox/session-file-provenance.test.ts @@ -1,101 +1,24 @@ -import { createHash } from 'node:crypto' import { redisConfigMockFns } from '@sim/testing/mocks/redis-config.mock' -import { generateShortId } from '@sim/utils/id' -import Redis from 'ioredis' -import { afterAll, beforeEach, describe, expect, it } from 'vitest' - -const records = new Map() -const memory = { - set: async (key: string, value: string) => { - if (!records.has(key)) records.set(key, value) - }, - get: async (key: string) => records.get(key) ?? null, - eval: async (_script: string, _count: number, key: string, input: string) => { - records.set(key, records.get(key) === 'clean' && input === 'clean' ? 'clean' : 'unknown') - }, -} -const redis = process.env.MSHIP_TEST_REDIS_SOCKET - ? new Redis({ - path: process.env.MSHIP_TEST_REDIS_SOCKET, - lazyConnect: true, - retryStrategy: () => null, - maxRetriesPerRequest: 1, - }) - : undefined -redis?.on('error', () => {}) -let storage: typeof memory | Redis | null = redis ?? memory -redisConfigMockFns.mockGetRedisClient.mockImplementation(() => storage) - +import { beforeEach, describe, expect, it } from 'vitest' import { initializeSessionFileProvenance, - isSessionFileProvenanceClean, + readSessionSecretProvenance, recordSessionFileInput, } from '@/lib/execution/remote-sandbox/session-file-provenance' -let session = '' -const ownedKeys = new Set() const machine = { providerId: 'e2b', sandboxId: 'machine' } as const -beforeEach(() => { - records.clear() - storage = redis ?? memory - session = `scratch-test-${generateShortId(16)}` - for (const [scope, provider, id] of [ - [session, 'e2b', 'machine'], - [`${session}-other`, 'e2b', 'machine'], - [session, 'e2b', 'replacement'], - [session, 'modal', 'machine'], - ]) - ownedKeys.add( - `mothership:workbench-provenance:v1:${createHash('sha256') - .update(JSON.stringify([scope, provider, id])) - .digest('hex')}` - ) -}) -afterAll(async () => { - if (redis) { - const keys = [...ownedKeys] - if (keys.length) await redis.del(...keys) - redis.disconnect() - } -}) +beforeEach(() => redisConfigMockFns.mockGetRedisClient.mockReturnValue(null)) -describe('physical session input history', () => { - it('permits fresh safe code, but a prior secret stays unknown through retries and a clean later call', async () => { - await initializeSessionFileProvenance(session, machine) - await recordSessionFileInput(session, machine, true) - expect(await isSessionFileProvenanceClean(session, machine)).toBe(true) - await recordSessionFileInput(session, machine, false) - await recordSessionFileInput(session, machine, true) - await initializeSessionFileProvenance(session, machine) - expect(await isSessionFileProvenanceClean(session, machine)).toBe(false) - }) - it('never certifies a recovered machine with absent history from a current clean input', async () => { - expect(await isSessionFileProvenanceClean(session, machine)).toBe(false) - await recordSessionFileInput(session, machine, true) - expect(await isSessionFileProvenanceClean(session, machine)).toBe(false) - }) - it('binds evidence to exact chat, provider and physical machine rather than a reused path', async () => { - await initializeSessionFileProvenance(session, machine) - expect(await isSessionFileProvenanceClean(`${session}-other`, machine)).toBe(false) - expect( - await isSessionFileProvenanceClean(session, { ...machine, sandboxId: 'replacement' }) - ).toBe(false) - expect(await isSessionFileProvenanceClean(session, { ...machine, providerId: 'modal' })).toBe( - false +describe('unavailable workbench evidence storage', () => { + it('refuses allocation, input receipt and output classification when Redis is unavailable', async () => { + await expect(initializeSessionFileProvenance('chat', machine)).rejects.toThrow( + 'storage is unavailable' ) - await initializeSessionFileProvenance(session, { ...machine, sandboxId: 'replacement' }) - expect( - await isSessionFileProvenanceClean(session, { ...machine, sandboxId: 'replacement' }) - ).toBe(true) - }) - it('fails closed when evidence storage or physical identity is absent', async () => { - storage = null - await expect(isSessionFileProvenanceClean(session, machine)).rejects.toThrow( + await expect(recordSessionFileInput('chat', machine, true)).rejects.toThrow( 'storage is unavailable' ) - storage = redis ?? memory - await expect(initializeSessionFileProvenance(session, { providerId: 'e2b' })).rejects.toThrow( - 'physical identity' + await expect(readSessionSecretProvenance('chat', machine)).rejects.toThrow( + 'storage is unavailable' ) }) }) diff --git a/apps/sim/lib/execution/remote-sandbox/session-file-provenance.ts b/apps/sim/lib/execution/remote-sandbox/session-file-provenance.ts index f4a98836af0..3234861477e 100644 --- a/apps/sim/lib/execution/remote-sandbox/session-file-provenance.ts +++ b/apps/sim/lib/execution/remote-sandbox/session-file-provenance.ts @@ -1,15 +1,72 @@ import { createHash } from 'node:crypto' +import { isRecordLike, omit } from '@sim/utils/object' import { getRedisClient } from '@/lib/core/config/redis' +import { + type DurableSecretProvenance, + normalizeDurableSecretProvenanceEntries, +} from '@/lib/execution/durable-secret-provenance' +import { + PROVENANCE_MAX_ENTRIES, + PROVENANCE_MAX_SERIALIZED_BYTES, +} from '@/lib/execution/provenance-limits' import { resolveProvider } from '@/lib/execution/remote-sandbox/provider' import type { SessionFileIdentity } from '@/lib/execution/remote-sandbox/session-file-observer' const TTL_SECONDS = 24 * 60 * 60 +const UNKNOWN = '{"status":"unknown"}' +const EMPTY = '{"status":"exact","entries":[]}' const RECORD_INPUT = ` local previous = redis.call('GET', KEYS[1]) -local next = 'unknown' -if previous == 'clean' and ARGV[1] == 'clean' then next = 'clean' end +local next = ARGV[3] +local maxBytes = tonumber(ARGV[4]) +local maxEntries = tonumber(ARGV[5]) +if previous and #previous <= maxBytes and #ARGV[1] <= maxBytes then + local previousOk, history = pcall(cjson.decode, previous) + local inputOk, input = pcall(cjson.decode, ARGV[1]) + if previousOk and inputOk and type(history) == 'table' and type(input) == 'table' + and history.status == 'exact' and input.status == 'exact' + and type(history.entries) == 'table' and type(input.entries) == 'table' then + local entries = {} + local seen = {} + local secretValues = {} + local secretValueCount = 0 + local valid = (#history.entries > 0 or previous == ARGV[6]) + and (#input.entries > 0 or ARGV[1] == ARGV[6]) + for _, source in ipairs({history.entries, input.entries}) do + for index, _ in pairs(source) do + if type(index) ~= 'number' or index < 1 or index > #source or index % 1 ~= 0 then + valid = false; break + end + end + if not valid then break end + for _, entry in ipairs(source) do + if type(entry) ~= 'table' or type(entry.encryptedValue) ~= 'string' then + valid = false + break + end + local id = cjson.encode({entry.encryptedValue, entry.name or '', + entry.sourceUserId or '', entry.sourceWorkspaceId or ''}) + if not seen[id] then + seen[id] = true + if not secretValues[entry.encryptedValue] then + secretValues[entry.encryptedValue] = true + secretValueCount = secretValueCount + 1 + if secretValueCount > maxEntries then valid = false; break end + end + table.insert(entries, entry) + end + end + if not valid then break end + end + if valid then + if #entries == 0 then next = ARGV[6] + else next = cjson.encode({status='exact', entries=entries}) end + if #next > maxBytes then next = ARGV[3] end + end + end +end redis.call('SET', KEYS[1], next, 'EX', ARGV[2]) -return next +return 1 ` function key(sessionKey: string, machine: SessionFileIdentity) { @@ -17,7 +74,7 @@ function key(sessionKey: string, machine: SessionFileIdentity) { const digest = createHash('sha256') .update(JSON.stringify([sessionKey, machine.providerId, machine.sandboxId])) .digest('hex') - return `mothership:workbench-provenance:v1:${digest}` + return `mothership:workbench-provenance:v2:${digest}` } function redis() { const client = getRedisClient() @@ -30,40 +87,85 @@ export async function initializeSessionFileProvenance( sessionKey: string, machine: SessionFileIdentity ) { - await redis().set(key(sessionKey, machine), 'clean', 'EX', TTL_SECONDS, 'NX') + await redis().set(key(sessionKey, machine), EMPTY, 'EX', TTL_SECONDS, 'NX') } -/** Record inputs before writing or executing; missing history never becomes clean on a retry. */ +/** + * Atomically widen encrypted machine history before classified input enters it. + * Source-value hashes have already narrowed the input selection and do not bind a machine's lifetime. + */ export async function recordSessionFileInput( sessionKey: string, machine: SessionFileIdentity, - exactEmpty: boolean + input: boolean | DurableSecretProvenance ) { + const provenance = + typeof input === 'boolean' + ? input + ? { status: 'exact' as const, entries: [] } + : { status: 'unknown' as const } + : input + const normalized = + provenance.status === 'exact' + ? normalizeDurableSecretProvenanceEntries(provenance.entries) + : undefined + const entries = + normalized && + normalizeDurableSecretProvenanceEntries( + normalized.map((entry) => omit(entry, ['sourceValueHash'])) + ) + const encoded = entries ? JSON.stringify({ status: 'exact', entries }) : UNKNOWN await redis().eval( RECORD_INPUT, 1, key(sessionKey, machine), - exactEmpty ? 'clean' : 'unknown', - TTL_SECONDS + Buffer.byteLength(encoded, 'utf8') <= PROVENANCE_MAX_SERIALIZED_BYTES ? encoded : UNKNOWN, + TTL_SECONDS, + UNKNOWN, + PROVENANCE_MAX_SERIALIZED_BYTES, + PROVENANCE_MAX_ENTRIES, + EMPTY ) } +/** Missing, legacy, expired or malformed history can never certify an existing machine. */ +export async function readSessionSecretProvenance( + sessionKey: string, + machine: SessionFileIdentity +): Promise { + const value = await redis().get(key(sessionKey, machine)) + if (!value || Buffer.byteLength(value, 'utf8') > PROVENANCE_MAX_SERIALIZED_BYTES) + return { status: 'unknown' } + try { + const parsed: unknown = JSON.parse(value) + if (!isRecordLike(parsed) || parsed.status !== 'exact') return { status: 'unknown' } + const entries = normalizeDurableSecretProvenanceEntries(parsed.entries) + return entries ? { status: 'exact', entries } : { status: 'unknown' } + } catch { + return { status: 'unknown' } + } +} + /** Physical identity, not a caller path, binds the lifetime of this evidence. */ export async function isSessionFileProvenanceClean( sessionKey: string, machine: SessionFileIdentity ) { - return (await redis().get(key(sessionKey, machine))) === 'clean' + const history = await readSessionSecretProvenance(sessionKey, machine) + return history.status === 'exact' && history.entries.length === 0 } /** Records classified API input on the existing physical machine without creating one. */ -export async function recordExistingSessionFileInput(sessionKey: string, exactEmpty: boolean) { +export async function recordExistingSessionFileInput( + sessionKey: string, + provenance: boolean | DurableSecretProvenance +) { const provider = resolveProvider() const sandbox = await provider.findSessionSandbox?.(sessionKey, {}) if (!sandbox) throw new Error('The active workbench is unavailable') await recordSessionFileInput( sessionKey, { providerId: provider.id, sandboxId: sandbox.sandboxId }, - exactEmpty + provenance ) } diff --git a/apps/sim/lib/execution/remote-sandbox/session-file-snapshot.ts b/apps/sim/lib/execution/remote-sandbox/session-file-snapshot.ts index 3b05db38e80..0acdfe1cafd 100644 --- a/apps/sim/lib/execution/remote-sandbox/session-file-snapshot.ts +++ b/apps/sim/lib/execution/remote-sandbox/session-file-snapshot.ts @@ -102,7 +102,12 @@ export async function openSessionFileSnapshot( accessSignal.throwIfAborted() if (copied.timedOut) throw new Error('Workbench upload snapshot timed out') if (copied.exitCode !== 0) { - throw new Error(copied.stderr.trim() || 'Could not prepare the workbench upload file') + const reason = [ + 'Scratch file resolves outside the permitted sandbox directories', + 'Upload source must be a regular file', + 'Upload source exceeds the workspace file size limit', + ].find((message) => copied.stderr.trim().split('\n').at(-1)?.includes(message)) + throw new Error(reason ?? 'Could not prepare the workbench upload file') } const size = await sandbox.getFileSize(staged) accessSignal.throwIfAborted() diff --git a/apps/sim/lib/execution/remote-sandbox/session-files.test.ts b/apps/sim/lib/execution/remote-sandbox/session-files.test.ts index a06bc4ed385..76704e78ee6 100644 --- a/apps/sim/lib/execution/remote-sandbox/session-files.test.ts +++ b/apps/sim/lib/execution/remote-sandbox/session-files.test.ts @@ -35,6 +35,8 @@ vi.mock('@/lib/execution/remote-sandbox/session-lock', () => ({ })) import { observeSandboxExecution } from '@/lib/execution/remote-sandbox/execution-observer' +import { SandboxOutputLimitError } from '@/lib/execution/remote-sandbox/output-limits' +import { readSessionSecretProvenance } from '@/lib/execution/remote-sandbox/session-file-provenance' import { readSessionSandboxFile, writeSessionSandboxFile, @@ -63,6 +65,24 @@ describe('workbench file cancellation', () => { run.mockResolvedValue({ stdout: '', stderr: '', exitCode: 0 }) }) + it('distinguishes a file-size failure without returning provider diagnostics', async () => { + read.mockRejectedValueOnce(new SandboxOutputLimitError(4 * 1024 * 1024 + 1, 4 * 1024 * 1024)) + expect(await readSessionSandboxFile('chat', 'large.txt')).toEqual({ + outcome: 'error', + detail: 'Workbench file exceeds the maximum read size of 4194304 bytes', + }) + }) + + it('distinguishes a provenance outage from a missing file without returning storage diagnostics', async () => { + vi.mocked(readSessionSecretProvenance).mockRejectedValueOnce( + new Error('SYNTHETIC_PRIVATE_DIAGNOSTIC') + ) + expect(await readSessionSandboxFile('chat', 'input.txt')).toEqual({ + outcome: 'error', + detail: 'Workbench file secret provenance is unavailable', + }) + }) + it('does not write if Stop arrives during the sandbox lookup', async () => { const controller = new AbortController() find.mockImplementation(async () => { @@ -112,6 +132,7 @@ describe('workbench file cancellation', () => { expect(await readSessionSandboxFile('chat', 'input.csv')).toEqual({ outcome: 'read', content: 'data', + secretProvenance: { status: 'exact', entries: [] }, }) expect(read).toHaveBeenLastCalledWith( '/home/user/input.csv', @@ -444,4 +465,5 @@ describe('workbench file cancellation', () => { vi.mock('@/lib/execution/remote-sandbox/session-file-provenance', () => ({ initializeSessionFileProvenance: vi.fn(), recordSessionFileInput: vi.fn(), + readSessionSecretProvenance: vi.fn(async () => ({ status: 'exact', entries: [] })), })) diff --git a/apps/sim/lib/execution/remote-sandbox/session-files.ts b/apps/sim/lib/execution/remote-sandbox/session-files.ts index ae38e01f31b..0204356e6ba 100644 --- a/apps/sim/lib/execution/remote-sandbox/session-files.ts +++ b/apps/sim/lib/execution/remote-sandbox/session-files.ts @@ -1,16 +1,21 @@ import { posix } from 'node:path' import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' -import { PayloadSizeLimitError } from '@/lib/core/utils/stream-limits' +import { isPayloadSizeLimitError, PayloadSizeLimitError } from '@/lib/core/utils/stream-limits' +import type { DurableSecretProvenance } from '@/lib/execution/durable-secret-provenance' import { prepareSandboxSessionAccess } from '@/lib/execution/remote-sandbox/execution-observer' import { withSandboxFilePublication } from '@/lib/execution/remote-sandbox/file-publication' +import { isSandboxOutputLimitError } from '@/lib/execution/remote-sandbox/output-limits' import { resolveProvider } from '@/lib/execution/remote-sandbox/provider' import { ensureSessionSandbox, SESSION_SANDBOX_IDLE_MS, } from '@/lib/execution/remote-sandbox/session' import type { SessionFileObserver } from '@/lib/execution/remote-sandbox/session-file-observer' -import { recordSessionFileInput } from '@/lib/execution/remote-sandbox/session-file-provenance' +import { + readSessionSecretProvenance, + recordSessionFileInput, +} from '@/lib/execution/remote-sandbox/session-file-provenance' import { withSandboxSessionLock } from '@/lib/execution/remote-sandbox/session-lock' import type { SandboxHandle } from '@/lib/execution/remote-sandbox/types' import { MAX_WORKSPACE_FILE_SIZE } from '@/lib/uploads/shared/types' @@ -39,7 +44,7 @@ export function resolveSessionPath(path: string): string { } export type SessionFileRead = - | { outcome: 'read'; content: string } + | { outcome: 'read'; content: string; secretProvenance?: DurableSecretProvenance } | { outcome: 'no-session' } | { outcome: 'no-file'; detail: string } | { outcome: 'error'; detail: string } @@ -62,15 +67,32 @@ export async function readSessionSandboxFile( if (!sandbox) return { outcome: 'no-session' } await sandbox.extendLifetime?.(SESSION_SANDBOX_IDLE_MS) signal.throwIfAborted() + let file: { content: string } try { - const file = await sandbox.readFileWithLimit(resolved, { + file = await sandbox.readFileWithLimit(resolved, { maxBytes: READ_LIMIT_BYTES, encoding, signal, }) - return { outcome: 'read', content: file.content } } catch (error) { - return { outcome: 'no-file', detail: getErrorMessage(error) } + signal.throwIfAborted() + if (isSandboxOutputLimitError(error) || isPayloadSizeLimitError(error)) { + return { + outcome: 'error', + detail: `Workbench file exceeds the maximum read size of ${READ_LIMIT_BYTES} bytes`, + } + } + return { outcome: 'no-file', detail: 'Workbench file is missing or unreadable' } + } + try { + const secretProvenance = await readSessionSecretProvenance(sessionKey, { + providerId: provider.id, + sandboxId: sandbox.sandboxId, + }) + return { outcome: 'read', content: file.content, secretProvenance } + } catch { + signal.throwIfAborted() + return { outcome: 'error', detail: 'Workbench file secret provenance is unavailable' } } }) } catch (error) { diff --git a/apps/sim/lib/execution/remote-sandbox/session-input-certification.integration.ts b/apps/sim/lib/execution/remote-sandbox/session-input-certification.integration.ts index 45066ff3fcd..703e0354186 100644 --- a/apps/sim/lib/execution/remote-sandbox/session-input-certification.integration.ts +++ b/apps/sim/lib/execution/remote-sandbox/session-input-certification.integration.ts @@ -100,7 +100,7 @@ describe.skipIf(!redisUrl)('workbench certification at the code boundary', () => const key = `certification-${generateShortId(12)}` const identity = { providerId: 'e2b', sandboxId } as const createdKeys.push( - `mothership:workbench-provenance:v1:${createHash('sha256') + `mothership:workbench-provenance:v2:${createHash('sha256') .update(JSON.stringify([key, identity.providerId, sandboxId])) .digest('hex')}` ) @@ -114,13 +114,15 @@ describe.skipIf(!redisUrl)('workbench certification at the code boundary', () => timeoutMs: 30_000, session: { key, ...(unprovenanced ? { unprovenancedInputs: true } : {}) }, } - await observeSandboxSessionInputs( + const execution = observeSandboxSessionInputs( () => true, () => kind === 'code' ? executeInSandbox(request) : executeShellInSandbox({ ...request, envs: {} }) ) + if (unprovenanced) await expect(execution).rejects.toThrow('Workbench output withheld') + else await expect(execution).resolves.toMatchObject({ sandboxId }) expect(await isSessionFileProvenanceClean(key, identity)).toBe(!unprovenanced) }) }) diff --git a/apps/sim/lib/execution/remote-sandbox/session-sandbox.test.ts b/apps/sim/lib/execution/remote-sandbox/session-sandbox.test.ts index 9b16a931e6d..0e18b1b07b6 100644 --- a/apps/sim/lib/execution/remote-sandbox/session-sandbox.test.ts +++ b/apps/sim/lib/execution/remote-sandbox/session-sandbox.test.ts @@ -964,6 +964,7 @@ describe('session sandbox lease', () => { vi.mock('@/lib/execution/remote-sandbox/session-file-provenance', () => ({ initializeSessionFileProvenance: vi.fn(), recordSessionFileInput: vi.fn(), + readSessionSecretProvenance: vi.fn().mockResolvedValue({ status: 'exact', entries: [] }), })) describe('scratch provenance at the actual code boundary', () => { @@ -977,7 +978,7 @@ describe('scratch provenance at the actual code boundary', () => { expect(recordSessionFileInput).toHaveBeenCalledWith( 'history', { providerId: 'e2b', sandboxId: 'provenance-physical' }, - safe + safe ? { status: 'exact', entries: [] } : { status: 'unknown' } ) return original(code, options) } diff --git a/apps/sim/lib/execution/remote-sandbox/types.ts b/apps/sim/lib/execution/remote-sandbox/types.ts index 473a4aa3732..d1b013ca3c5 100644 --- a/apps/sim/lib/execution/remote-sandbox/types.ts +++ b/apps/sim/lib/execution/remote-sandbox/types.ts @@ -1,4 +1,5 @@ import type { CodePlaceholderRuntimeBinding } from '@/lib/execution/code-placeholders/types' +import type { DurableSecretProvenance } from '@/lib/execution/durable-secret-provenance' import type { CodeLanguage } from '@/lib/execution/languages' import type { SandboxBuildError } from '@/lib/execution/remote-sandbox/build-errors' import type { SandboxSpec } from '@/lib/execution/remote-sandbox/sandbox-spec' @@ -96,6 +97,10 @@ export interface SandboxSessionRequest { * history must not stay certified clean even when the caller's own inputs are. */ unprovenancedInputs?: boolean + /** Host-only evidence; never populated from the Function wire contract. */ + inputProvenance?(): DurableSecretProvenance + /** Imports the full post-execution machine history before any result or export leaves the host. */ + acceptOutputProvenance?(provenance: DurableSecretProvenance): Promise } export interface SandboxShellExecutionRequest { diff --git a/apps/sim/lib/function-execution/execute-request.ts b/apps/sim/lib/function-execution/execute-request.ts index cdc1e7a7067..5a42e170d50 100644 --- a/apps/sim/lib/function-execution/execute-request.ts +++ b/apps/sim/lib/function-execution/execute-request.ts @@ -27,6 +27,12 @@ import { type CompiledCodePlaceholders, compileCodePlaceholders, } from '@/lib/execution/code-placeholders' +import { + type DurableSecretProvenance, + durableSecretProvenanceFromEnvelope, + importDurableSecretProvenance, + mergeDurableSecretProvenance, +} from '@/lib/execution/durable-secret-provenance' import { parseExecutionDeadlineHeader } from '@/lib/execution/execution-deadline-header' import { executeInIsolatedVM, type IsolatedVMBrokerHandler } from '@/lib/execution/isolated-vm' import { CodeLanguage, DEFAULT_CODE_LANGUAGE, isValidCodeLanguage } from '@/lib/execution/languages' @@ -72,6 +78,7 @@ import { executeShellInSandbox, SIM_RESULT_PREFIX, } from '@/lib/execution/remote-sandbox' +import { sandboxSessionInputProvenance } from '@/lib/execution/remote-sandbox/execution-observer' import { isSandboxOutputFileError, isSandboxOutputLimitError, @@ -132,8 +139,8 @@ import { scanResolvedSecretString, } from '@/executor/utils/resolved-secret-content-projection' import { isNonIdentifyingSecretLiteral } from '@/executor/utils/resolved-secret-match-policy' -import type { - ResolvedSecretTraceProvenanceV1, +import { + type ResolvedSecretTraceProvenanceV1, ResolvedSecretTraceRegistry, } from '@/executor/utils/resolved-secret-trace-registry' @@ -2716,10 +2723,41 @@ export async function executeFunctionRequest( ) } const { sandboxFiles: userFileMounts, manifest: mountManifest } = resolvedMounts - const mothershipSession = - admittedSession && resolvedMounts.unprovenancedMountCount > 0 - ? { ...admittedSession, unprovenancedInputs: true } - : admittedSession + const activeRouteContext = routeContext + const mothershipSession = admittedSession + ? { + ...admittedSession, + unprovenancedInputs: resolvedMounts.unprovenancedMountCount > 0, + inputProvenance: () => { + const runtime = activeRouteContext.runtimeFileSecretTraceRegistry?.exportProvenance() + return mergeDurableSecretProvenance( + sandboxSessionInputProvenance(), + runtime + ? durableSecretProvenanceFromEnvelope(runtime) + : { status: 'exact', entries: [] } + ) + }, + acceptOutputProvenance: async (provenance: DurableSecretProvenance) => { + const registry = activeRouteContext.resolvedSecretTraceRegistry + activeRouteContext.runtimeFileSecretTraceRegistry ??= new ResolvedSecretTraceRegistry( + [], + { + userId: auth.attributedUserId, + ...(workspaceId ? { workspaceId } : {}), + } + ) + const runtimeRegistry = activeRouteContext.runtimeFileSecretTraceRegistry + const runtimeImported = await importDurableSecretProvenance(runtimeRegistry, provenance) + const imported = registry && (await importDurableSecretProvenance(registry, provenance)) + activeRouteContext.runtimeFileSecretProvenanceScanner = undefined + if (!runtimeImported || !imported) { + throw new Error( + 'Workbench output withheld because its secret provenance is unavailable' + ) + } + }, + } + : undefined const sandboxFiles = mergeSandboxFileMounts(_sandboxFiles, userFileMounts) // Every `` marker becomes the path its file was mounted at, diff --git a/apps/sim/lib/mothership/agent-cli/run-cli-files.test.ts b/apps/sim/lib/mothership/agent-cli/run-cli-files.test.ts index e940ee745ba..78789d98dba 100644 --- a/apps/sim/lib/mothership/agent-cli/run-cli-files.test.ts +++ b/apps/sim/lib/mothership/agent-cli/run-cli-files.test.ts @@ -124,6 +124,7 @@ describe('the CLI owns workbench file semantics', () => { return { outcome: 'read', content: Buffer.from(JSON.stringify({ session })).toString('base64'), + secretProvenance: { status: 'exact', entries: [] }, } }) const transport = async (input: string | URL | Request, init?: RequestInit) => { @@ -153,6 +154,7 @@ describe('the CLI owns workbench file semantics', () => { read.mockResolvedValue({ outcome: 'read', content: Buffer.from('wf-one\nwf-two\n').toString('base64'), + secretProvenance: { status: 'exact', entries: [] }, }) const transport = async (input: string | URL | Request) => { requests.push(new URL(input instanceof Request ? input.url : input)) diff --git a/apps/sim/lib/mothership/agent-cli/run-cli.test.ts b/apps/sim/lib/mothership/agent-cli/run-cli.test.ts index d4ca4003970..585a506f457 100644 --- a/apps/sim/lib/mothership/agent-cli/run-cli.test.ts +++ b/apps/sim/lib/mothership/agent-cli/run-cli.test.ts @@ -23,7 +23,11 @@ describe('embedded CLI binary workbench bridge', () => { it('preserves arbitrary bytes in both directions and binds both to the same chat', async () => { const bytes = Uint8Array.from([0, 255, 137, 80, 78, 71, 13, 10, 128, 195, 0]) const stream = new Blob([bytes]).stream() - readFile.mockResolvedValue({ outcome: 'read', content: Buffer.from(bytes).toString('base64') }) + readFile.mockResolvedValue({ + outcome: 'read', + content: Buffer.from(bytes).toString('base64'), + secretProvenance: { status: 'exact', entries: [] }, + }) writeFile.mockResolvedValue({ outcome: 'written', path: '/home/user/result.png' }) embedded.mockImplementation(async (_args, _identity, options) => { expect(await options.readFile('image.png')).toEqual(Buffer.from(bytes)) @@ -51,7 +55,11 @@ describe('embedded CLI binary workbench bridge', () => { }) it('resolves equals-form file flags identically without reading escaped literals', async () => { - readFile.mockResolvedValue({ outcome: 'read', content: Buffer.from('{}').toString('base64') }) + readFile.mockResolvedValue({ + outcome: 'read', + content: Buffer.from('{}').toString('base64'), + secretProvenance: { status: 'exact', entries: [] }, + }) embedded.mockImplementation(async (_args, _identity, options) => { expect(await options.readFile('input.json')).toEqual(Buffer.from('{}')) return { exitCode: 0, stdout: '', stderr: '' } diff --git a/apps/sim/lib/mothership/agent-cli/run-cli.ts b/apps/sim/lib/mothership/agent-cli/run-cli.ts index f1a56dc2fb5..a3ed837a645 100644 --- a/apps/sim/lib/mothership/agent-cli/run-cli.ts +++ b/apps/sim/lib/mothership/agent-cli/run-cli.ts @@ -1,4 +1,6 @@ import { type EmbeddedCliIdentity, runEmbeddedCli } from 'sim/embed' +import { importDurableSecretProvenance } from '@/lib/execution/durable-secret-provenance' +import { isBinarySandboxPath } from '@/lib/execution/remote-sandbox/sandbox-encoding' import type { SessionFileObserver } from '@/lib/execution/remote-sandbox/session-file-observer' import { openSessionFileSnapshot } from '@/lib/execution/remote-sandbox/session-file-snapshot' import { @@ -7,6 +9,8 @@ import { writeSessionSandboxFile, } from '@/lib/execution/remote-sandbox/session-files' import type { AgentCliRawResult } from '@/lib/mothership/generated/agent-cli' +import { projectResolvedSecretModelContent } from '@/executor/utils/resolved-secret-content-projection' +import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' /** * Runs one real-CLI invocation in-process through the installed CLI's own command tree, @@ -64,7 +68,21 @@ export async function readCliInputFile( signal?.throwIfAborted() const read = await readSessionSandboxFile(sessionKey, path, 'base64', signal) signal?.throwIfAborted() - if (read.outcome === 'read') return Buffer.from(read.content, 'base64') + if (read.outcome === 'read') { + const provenance = read.secretProvenance + if (!provenance || provenance.status !== 'exact') + throw new Error('CLI input withheld because workbench secret provenance is unavailable') + const buffer = Buffer.from(read.content, 'base64') + if (provenance.entries.length === 0) return buffer + const registry = new ResolvedSecretTraceRegistry([]) + if (!(await importDurableSecretProvenance(registry, provenance))) + throw new Error('CLI input withheld because workbench secret provenance is unavailable') + const text = buffer.toString('utf8') + const projection = projectResolvedSecretModelContent(text, registry) + if (!projection.safe || isBinarySandboxPath(path) || projection.value !== text) + throw new Error('CLI input may contain protected workbench values') + return buffer + } throw new Error( read.outcome === 'no-session' ? `No workbench exists for this chat; write "${path}" first or pass the value inline.` diff --git a/apps/sim/lib/mothership/agent-cli/workbench-file-provenance.ts b/apps/sim/lib/mothership/agent-cli/workbench-file-provenance.ts index 573748cd1da..8bef5617c0e 100644 --- a/apps/sim/lib/mothership/agent-cli/workbench-file-provenance.ts +++ b/apps/sim/lib/mothership/agent-cli/workbench-file-provenance.ts @@ -87,7 +87,7 @@ export function createWorkbenchFileProvenance(scope: WorkbenchFileScope) { recordSessionFileInput( scope.sessionKey, machine, - provenance.status === 'exact' && provenance.entries.length === 0 + provenance.status === 'exact' ? provenance : { status: 'unknown' } ) ) const observeDownload: SessionFileObserver = (machine, stream) => diff --git a/apps/sim/lib/mothership/tools/handlers/function-execute-provenance.test.ts b/apps/sim/lib/mothership/tools/handlers/function-execute-provenance.test.ts index 9679e422a27..343e8e73374 100644 --- a/apps/sim/lib/mothership/tools/handlers/function-execute-provenance.test.ts +++ b/apps/sim/lib/mothership/tools/handlers/function-execute-provenance.test.ts @@ -15,7 +15,7 @@ vi.mock('@/lib/secrets/usage/record', () => ({ recordSecretUsage: vi.fn() })) vi.mock('@/lib/billing/core/subscription', () => billingSubscriptionMock) import { encryptSecret } from '@/lib/core/security/encryption' -import { sandboxSessionInputsSafe } from '@/lib/execution/remote-sandbox/execution-observer' +import { sandboxSessionInputProvenance } from '@/lib/execution/remote-sandbox/execution-observer' import { executeFunctionExecute } from '@/lib/mothership/tools/handlers/function-execute' import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' @@ -26,7 +26,7 @@ beforeEach(() => { mocks.execute.mockReset() mocks.execute.mockImplementation(async () => ({ success: true, - output: { sessionInputsSafe: sandboxSessionInputsSafe() }, + output: { sessionInputProvenance: sandboxSessionInputProvenance() }, })) }) @@ -51,8 +51,10 @@ describe('Function physical-session input certification', () => { ) expect(mocks.execute).toHaveBeenCalledOnce() const response = await mocks.execute.mock.results[0].value - expect(response.output.sessionInputsSafe).toBe(state === 'empty') - expect(sandboxSessionInputsSafe()).toBe(false) + expect(response.output.sessionInputProvenance).toEqual( + state === 'empty' ? { status: 'exact', entries: [] } : { status: 'unknown' } + ) + expect(sandboxSessionInputProvenance()).toEqual({ status: 'unknown' }) } ) }) diff --git a/apps/sim/lib/mothership/tools/handlers/function-execute-session.test.ts b/apps/sim/lib/mothership/tools/handlers/function-execute-session.test.ts index 8ba776a8015..b67f82bec14 100644 --- a/apps/sim/lib/mothership/tools/handlers/function-execute-session.test.ts +++ b/apps/sim/lib/mothership/tools/handlers/function-execute-session.test.ts @@ -7,9 +7,12 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { observeServiceCosts } from '@/lib/mothership/billing/service-observer' const { mockMaterializeSecrets } = vi.hoisted(() => ({ - mockMaterializeSecrets: vi - .fn() - .mockResolvedValue({ envVars: { API_KEY: 'test-value' }, catalogEntries: [] }), + mockMaterializeSecrets: vi.fn().mockResolvedValue({ + envVars: { API_KEY: 'test-value' }, + catalogEntries: [ + { name: 'API_KEY', plaintext: 'test-value', encryptedValue: 'mock-encrypted-test-value' }, + ], + }), })) vi.mock('@/tools', () => toolsMock) @@ -17,22 +20,6 @@ vi.mock('@/lib/mothership/tools/secret-mount-materializer.server', () => ({ materializeCopilotCodeSecrets: mockMaterializeSecrets, CopilotCodeSecretAccessError: class extends Error {}, })) -vi.mock('@/executor/utils/resolved-secret-trace-registry', () => ({ - ResolvedSecretTraceRegistry: class { - getUnredactedSecretNames() { - return [] - } - exportProvenance() { - return { complete: true } - } - exportProvenanceForValue() { - return { complete: true } - } - getResolvedSecretUsage() { - return [] - } - }, -})) vi.mock('@/lib/secrets/usage/record', () => ({ recordSecretUsage: vi.fn() })) vi.mock('@/lib/billing/core/subscription', () => billingSubscriptionMock) diff --git a/apps/sim/lib/mothership/tools/handlers/function-execute.ts b/apps/sim/lib/mothership/tools/handlers/function-execute.ts index 1f878ea48d0..50e6281582a 100644 --- a/apps/sim/lib/mothership/tools/handlers/function-execute.ts +++ b/apps/sim/lib/mothership/tools/handlers/function-execute.ts @@ -3,7 +3,11 @@ import { createLogger } from '@sim/logger' import { omit, toRecord } from '@sim/utils/object' import { hasWorkspaceSandboxAccess } from '@/lib/billing/core/subscription' import { OrchestrationError } from '@/lib/core/orchestration/types' -import { importDurableSecretProvenance } from '@/lib/execution/durable-secret-provenance' +import { + durableSecretProvenanceFromEnvelope, + importDurableSecretProvenance, + mergeDurableSecretProvenance, +} from '@/lib/execution/durable-secret-provenance' import type { PrivateSecretProvenanceBundleV1 } from '@/lib/execution/model-input-provenance' import { MOUNTED_WORKSPACE_FILES_PROVENANCE_KEY, @@ -509,6 +513,7 @@ export async function executeFunctionExecute( ): Promise { const enrichedParams = omit(params, [ 'secrets', + 'unredactedSecretNames', 'sandboxProfile', 'internalSandboxProfile', // Server-derived below — a model-supplied value must never select a session. @@ -590,6 +595,12 @@ export async function executeFunctionExecute( ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), }) + /** Receipt records every value placed in the runtime, including silent resolutions. */ + for (const [name, plaintext] of Object.entries(mounted.envVars)) { + if (plaintext.length > 0 && !mountedRegistry.recordResolved(name, plaintext)) { + throw new CopilotCodeSecretAccessError('Mounted secret provenance is unavailable') + } + } enrichedParams.envVars = mounted.envVars enrichedParams.secretScope = 'selected' enrichedParams.mountedSecrets = requestedNames @@ -666,15 +677,15 @@ export async function executeFunctionExecute( */ const result = await observeSandboxSessionInputs( () => { - const mounted = mountedRegistry?.exportProvenance() + const mounted = mountedRegistry?.exportCheckpointProvenance() const code = context.resolvedSecretTraceRegistry?.exportCommittedProvenanceForValue(params) - return ( - mounted?.complete === true && - mounted.entries.length === 0 && - code?.complete === true && - code.entries.length === 0 - ) + return mounted && code + ? mergeDurableSecretProvenance( + durableSecretProvenanceFromEnvelope(mounted), + durableSecretProvenanceFromEnvelope(code) + ) + : { status: 'unknown' as const } }, () => executeAppTool('function_execute', enrichedParams, { diff --git a/apps/sim/lib/mothership/tools/handlers/workbench-confidentiality.live.test.ts b/apps/sim/lib/mothership/tools/handlers/workbench-confidentiality.live.test.ts new file mode 100644 index 00000000000..79fbfb88d4b --- /dev/null +++ b/apps/sim/lib/mothership/tools/handlers/workbench-confidentiality.live.test.ts @@ -0,0 +1,416 @@ +import { execFile } from 'node:child_process' +import { mkdir, mkdtemp, readdir, readFile, rm, stat, writeFile } from 'node:fs/promises' +import { dirname, join } from 'node:path' +import { promisify } from 'node:util' +import { createDelegatedPrincipal } from '@sim/testing/factories/principal.factory' +import { createDeferred } from '@sim/testing/helpers/deferred' +import { setEnv } from '@sim/testing/mocks/env.mock' +import { envFlagsMock } from '@sim/testing/mocks/env-flags.mock' +import { redisConfigMockFns } from '@sim/testing/mocks/redis-config.mock' +import { + remoteSandboxProviderMock, + remoteSandboxProviderMockFns, +} from '@sim/testing/mocks/remote-sandbox-provider.mock' +import { toolsMock, toolsMockFns } from '@sim/testing/mocks/tools.mock' +import { generateShortId } from '@sim/utils/id' +import Redis from 'ioredis' +import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' + +const io = vi.hoisted(() => ({ mount: vi.fn(), find: vi.fn(), write: vi.fn() })) +vi.mock('@/tools', () => toolsMock) +vi.mock('@/lib/mothership/tools/secret-mount-materializer.server', () => ({ + materializeCopilotCodeSecrets: io.mount, + CopilotCodeSecretAccessError: class extends Error {}, +})) +vi.mock('@/lib/secrets/usage/record', () => ({ recordSecretUsage: vi.fn() })) +vi.mock('@/lib/execution/remote-sandbox/provider', () => remoteSandboxProviderMock) +vi.mock('@/lib/execution/remote-sandbox/resolve', () => ({ + resolveWorkspaceSandbox: async () => null, + provisionRuntimeDependencies: async () => {}, + repairMissingSandboxImage: async () => null, + RUNTIME_INSTALL_TIMEOUT_MS: 60_000, +})) +vi.mock('@/lib/mothership/tools/sandbox-session', () => ({ + buildMothershipSandboxSession: async (args: { sessionKey: string }) => ({ key: args.sessionKey }), +})) +vi.mock('@/lib/workspace-files/application/delegated-principal', () => ({ + rebindWorkspaceFileDelegatedPrincipal: ({ principal }: { principal: unknown }) => principal, +})) +vi.mock('@/lib/mothership/vfs/resource-writer', () => ({ + validateWorkspaceFileWriteTarget: async () => ({ vfsPath: 'files/review.txt' }), + writeWorkspaceFileByPath: io.write, +})) + +import { functionExecuteBodySchema } from '@/lib/api/contracts' +import { encryptSecret } from '@/lib/core/security/encryption' +import { + PRIVATE_TOOL_METADATA_REQUEST_HEADER, + RESOLVED_SECRET_NAMES_FIELD, + RESOLVED_SECRET_NAMES_METADATA_V1, +} from '@/lib/execution/private-tool-metadata' +import { + initializeSessionFileProvenance, + isSessionFileProvenanceClean, + readSessionSecretProvenance, + recordSessionFileInput, +} from '@/lib/execution/remote-sandbox/session-file-provenance' +import type { SandboxHandle } from '@/lib/execution/remote-sandbox/types' +import { executeFunctionRequest } from '@/lib/function-execution/execute-request' +import { readCliInputFile } from '@/lib/mothership/agent-cli/run-cli' +import { inspectToolResultForCopilot } from '@/lib/mothership/request/tools/resolved-secret-result' +import type { ToolExecutionContext } from '@/lib/mothership/tool-executor/types' +import { executeFunctionExecute } from '@/lib/mothership/tools/handlers/function-execute' +import { executeRunCode } from '@/lib/mothership/tools/handlers/run-code' +import { chatSandboxSessionKey } from '@/lib/mothership/tools/sandbox-session-key' +import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' +import { buildFunctionExecuteBody, functionExecuteTool } from '@/tools/function/execute' +import type { CodeExecutionInput } from '@/tools/function/types' + +const socket = process.env.MSHIP_TEST_REDIS_SOCKET +if (!socket) throw new Error('MSHIP_TEST_REDIS_SOCKET must identify a disposable local Redis') +const redis = new Redis({ path: socket, retryStrategy: () => null, maxRetriesPerRequest: 1 }) +const execute = promisify(execFile) +const canary = 'SYNTHETIC_REVIEW_SECRET_9b78e6dc' +const scope = { userId: 'review-actor', workspaceId: 'review-workspace' } +const roots: string[] = [] +let root: string +let chatId: string +let machine: SandboxHandle +let catalog: Array<{ name: string; plaintext: string; encryptedValue: string }> +let parent: ResolvedSecretTraceRegistry + +function workerPath(path: string) { + return path.startsWith('/') ? join(root, path.slice(1)) : join(root, 'home/user', path) +} + +function localWorker(): SandboxHandle { + return { + sandboxId: `local-${generateShortId(12)}`, + runCode: async () => { + throw new Error('This reproduction uses actual shell processes') + }, + async runCommand(command, options) { + const envs = Object.fromEntries( + Object.entries(options.envs ?? {}).map(([key, value]) => [ + key, + value + .replaceAll('/home/user', workerPath('/home/user')) + .replaceAll('/tmp/sim/', `${workerPath('/tmp/sim')}/`), + ]) + ) + try { + const output = await execute('/bin/bash', ['-c', command], { + cwd: workerPath('/home/user'), + env: { PATH: '/usr/bin:/bin:/opt/homebrew/bin', ...envs }, + timeout: options.timeoutMs, + maxBuffer: options.maxOutputBytes, + }) + return { ...output, exitCode: 0 } + } catch (error) { + const failure = error as { stdout: string; stderr: string; code: number } + return { stdout: failure.stdout, stderr: failure.stderr, exitCode: failure.code } + } + }, + extendLifetime: async () => {}, + getFileSize: async (path) => (await stat(workerPath(path))).size, + readFile: async (path) => readFile(workerPath(path), 'utf8'), + async readFileWithLimit(path, options) { + const bytes = await readFile(workerPath(path)) + if (bytes.length > options.maxBytes) throw new Error('Local worker byte cap') + return { content: bytes.toString(options.encoding), byteLength: bytes.length } + }, + async writeFile(path, content) { + await mkdir(dirname(workerPath(path)), { recursive: true }) + await writeFile( + workerPath(path), + typeof content === 'string' ? content : Buffer.from(content) + ) + }, + removeFile: async (path) => rm(workerPath(path), { force: true }), + async listFiles(path) { + const entries = await readdir(workerPath(path), { withFileTypes: true }) + return Promise.all( + entries.map(async (entry) => ({ + path: `${path}/${entry.name}`, + relativePath: entry.name, + kind: entry.isDirectory() ? ('directory' as const) : ('file' as const), + size: (await stat(workerPath(`${path}/${entry.name}`))).size, + })) + ) + }, + kill: async () => {}, + } +} + +beforeEach(async () => { + redisConfigMockFns.mockGetRedisClient.mockReturnValue(redis) + redisConfigMockFns.mockAcquireLock.mockImplementation( + async (key: string, owner: string, ttl: number) => + (await redis.set(key, owner, 'EX', ttl, 'NX')) === 'OK' + ) + redisConfigMockFns.mockExtendLock.mockImplementation( + async (key: string, owner: string, ttl: number) => + (await redis.eval( + "if redis.call('GET',KEYS[1]) == ARGV[1] then return redis.call('EXPIRE',KEYS[1],ARGV[2]) else return 0 end", + 1, + key, + owner, + ttl + )) === 1 + ) + redisConfigMockFns.mockReleaseLock.mockImplementation(async (key: string, owner: string) => { + await redis.eval( + "if redis.call('GET',KEYS[1]) == ARGV[1] then return redis.call('DEL',KEYS[1]) else return 0 end", + 1, + key, + owner + ) + }) + remoteSandboxProviderMockFns.mockResolveProvider.mockReturnValue({ + id: 'e2b', + dependencyStrategy: 'prebuilt', + resolveLifetimeMs: (ms: number) => ms, + findSessionSandbox: io.find, + create: async () => { + throw new Error('Only the existing disposable worker may be used') + }, + }) + setEnv({ ENCRYPTION_KEY: 'a'.repeat(64) }) + envFlagsMock.isMothershipSandboxEnabled = true + envFlagsMock.isRemoteSandboxEnabled = true + root = await mkdtemp('/private/tmp/sim-workbench-test-') + roots.push(root) + await mkdir(workerPath('/home/user'), { recursive: true }) + chatId = `review-${generateShortId(12)}` + machine = localWorker() + io.find.mockResolvedValue(machine) + const encryptedValue = (await encryptSecret(canary)).encrypted + catalog = [{ name: 'TOKEN', plaintext: canary, encryptedValue }] + parent = new ResolvedSecretTraceRegistry(catalog, scope) + io.mount.mockResolvedValue({ envVars: { TOKEN: canary }, catalogEntries: catalog }) + await initializeSessionFileProvenance(chatSandboxSessionKey(chatId), { + providerId: 'e2b', + sandboxId: machine.sandboxId, + }) + io.write.mockImplementation(async () => ({ + file: { id: 'review-file', name: 'review.txt', size: canary.length, type: 'text/plain' }, + vfsPath: 'files/review.txt', + })) + toolsMockFns.mockExecuteTool.mockImplementation( + async ( + _id, + params: CodeExecutionInput, + options: { resolvedSecretTraceRegistry: ResolvedSecretTraceRegistry } + ) => { + const finishActivation = options.resolvedSecretTraceRegistry.beginPendingActivation() + try { + const headers = new Headers({ + [PRIVATE_TOOL_METADATA_REQUEST_HEADER]: RESOLVED_SECRET_NAMES_METADATA_V1, + }) + const response = await executeFunctionRequest( + { headers, signal: AbortSignal.timeout(15_000) }, + functionExecuteBodySchema.parse(buildFunctionExecuteBody(params)), + { + attributedUserId: scope.userId, + principal: createDelegatedPrincipal({ + subjectUserId: scope.userId, + workspaceId: scope.workspaceId, + }), + sandboxProfile: 'mothership', + resolvedSecretTraceRegistry: options.resolvedSecretTraceRegistry, + } + ) + const payload = await response.json() + for (const name of payload[RESOLVED_SECRET_NAMES_FIELD] ?? []) { + expect( + options.resolvedSecretTraceRegistry.recordResolved(name, params.envVars![name], { + propagated: true, + }) + ).toBe(true) + } + return await functionExecuteTool.transformResponse!(Response.json(payload)) + } finally { + finishActivation() + } + } + ) +}) + +afterAll(async () => { + redis.disconnect() + for (const path of roots) await rm(path, { recursive: true, force: true }) +}) + +function context(): ToolExecutionContext { + return { + ...scope, + workflowId: '', + chatId, + resolvedSecretTraceRegistry: parent.forkForInputPaths([]), + } +} + +async function run(code: string, secrets: string[] = []) { + const current = context() + const raw = await executeRunCode({ code, language: 'shell', secrets }, current) + const projected = inspectToolResultForCopilot( + raw, + current.resolvedSecretTraceRegistry, + 'run_code' + ) + if (projected.safe) parent.mergeToolCallRegistry(current.resolvedSecretTraceRegistry!) + return { raw, projected } +} + +describe('persistent workbench output confidentiality', () => { + it('allows a mounted empty value without requiring a redaction receipt', async () => { + const emptyCatalog = [ + { name: 'TOKEN', plaintext: '', encryptedValue: (await encryptSecret('')).encrypted }, + ] + io.mount.mockResolvedValue({ envVars: { TOKEN: '' }, catalogEntries: emptyCatalog }) + const result = await run('printenv TOKEN >/dev/null && test -z "$TOKEN" && printf allowed', [ + 'TOKEN', + ]) + expect(result.raw.success).toBe(true) + expect(result.projected.safe).toBe(true) + expect(JSON.stringify(result.projected.result)).toContain('allowed') + }) + it('control: same-call secret output is redacted', async () => { + const result = await run('printf "%s" "$TOKEN"', ['TOKEN']) + expect(result.raw.success).toBe(true) + expect(result.projected.safe).toBe(true) + expect(JSON.stringify(result.projected.result)).not.toContain(canary) + expect(JSON.stringify(result.projected.result)).toContain('{{TOKEN}}') + }) + it.each([ + ['stdout', 'cat saved.txt'], + ['structured result', 'printf "__SIM_RESULT__=\\\"%s\\\"\\n" "$(cat saved.txt)"'], + ['error and stderr', 'cat saved.txt >&2; exit 1'], + ])('protects later-call output in %s', async (_name, code) => { + const first = await run('printf "%s" "$TOKEN" > saved.txt', ['TOKEN']) + expect(first.raw.success).toBe(true) + expect(JSON.stringify(first.projected.result)).not.toContain(canary) + const later = await run(code) + expect(later.projected.safe).toBe(true) + expect(JSON.stringify(later.projected.result)).not.toContain(canary) + expect(JSON.stringify(later.projected.result)).toContain('{{TOKEN}}') + }) + it('protects later-call output after an earlier result was redacted', async () => { + const first = await run('printf "%s" "$TOKEN" > saved.txt; cat saved.txt', ['TOKEN']) + expect(JSON.stringify(first.projected.result)).toContain('{{TOKEN}}') + expect(parent.getModelEgressSnapshot().matches?.length).toBeGreaterThan(0) + const later = await run('cat saved.txt') + expect(later.projected.safe).toBe(true) + expect(JSON.stringify(later.projected.result)).not.toContain(canary) + expect(JSON.stringify(later.projected.result)).toContain('{{TOKEN}}') + }) + it('refuses secret-bearing CLI input', async () => { + await run('printf "%s" "$TOKEN" > saved.txt', ['TOKEN']) + await expect(readCliInputFile(chatSandboxSessionKey(chatId), 'saved.txt')).rejects.toThrow() + }) + it('permits secret-free CLI input after a secret was received', async () => { + await run('printf clean > clean.txt', ['TOKEN']) + expect((await readCliInputFile(chatSandboxSessionKey(chatId), 'clean.txt')).toString()).toBe( + 'clean' + ) + }) + it('cannot disable redaction through model-supplied secret flags', async () => { + const current = context() + const raw = await executeRunCode( + { + code: 'printf \"%s\" \"$TOKEN\"', + language: 'shell', + secrets: ['TOKEN'], + unredactedSecretNames: ['TOKEN'], + }, + current + ) + const projected = inspectToolResultForCopilot( + raw, + current.resolvedSecretTraceRegistry, + 'run_code' + ) + expect(projected.safe).toBe(true) + expect(JSON.stringify(projected.result)).not.toContain(canary) + }) + it('withholds output when physical machine history is unknown', async () => { + await run('printf "%s" "$TOKEN" > saved.txt', ['TOKEN']) + await recordSessionFileInput( + chatSandboxSessionKey(chatId), + { providerId: 'e2b', sandboxId: machine.sandboxId }, + false + ) + expect( + await isSessionFileProvenanceClean(chatSandboxSessionKey(chatId), { + providerId: 'e2b', + sandboxId: machine.sandboxId, + }) + ).toBe(false) + const result = await run('cat saved.txt') + expect(result.projected.safe).toBe(false) + expect(JSON.stringify(result.projected.result)).not.toContain(canary) + expect(result.raw.success).toBe(false) + }) + it('retains both values of a rotated secret without storing plaintext', async () => { + await run('printf "%s" "$TOKEN" > first.txt', ['TOKEN']) + const rotated = 'SYNTHETIC_ROTATED_VALUE_8e13b77f' + const entry = { + name: 'TOKEN', + plaintext: rotated, + encryptedValue: (await encryptSecret(rotated)).encrypted, + } + io.mount.mockResolvedValue({ envVars: { TOKEN: rotated }, catalogEntries: [entry] }) + await run('printf "%s" "$TOKEN" > second.txt', ['TOKEN']) + const output = await run('cat first.txt second.txt') + expect(output.projected.safe).toBe(true) + expect(JSON.stringify(output.projected.result)).not.toContain(canary) + expect(JSON.stringify(output.projected.result)).not.toContain(rotated) + const history = await readSessionSecretProvenance(chatSandboxSessionKey(chatId), { + providerId: 'e2b', + sandboxId: machine.sandboxId, + }) + expect(history.status).toBe('exact') + expect(JSON.stringify(history)).not.toContain(canary) + expect(JSON.stringify(history)).not.toContain(rotated) + }) + it('reads history after overlapping code has received a new secret', async () => { + const started = createDeferred() + const finish = createDeferred() + const runCommand = machine.runCommand + machine.runCommand = async (command, options) => { + if (command === 'WAIT_FOR_LATER_INPUT') { + started.resolve() + await finish.promise + return { stdout: canary, stderr: '', exitCode: 0 } + } + return runCommand(command, options) + } + const earlier = run('WAIT_FOR_LATER_INPUT') + await started.promise + const later = await run('printf "%s" "$TOKEN" > overlap.txt', ['TOKEN']) + expect(later.raw.success).toBe(true) + finish.resolve() + const output = await earlier + expect(output.projected.safe).toBe(true) + expect(JSON.stringify(output.projected.result)).not.toContain(canary) + expect(JSON.stringify(output.projected.result)).toContain('{{TOKEN}}') + }) + it('retains historical secret provenance on a text export', async () => { + await run('printf "%s" "$TOKEN" > saved.txt', ['TOKEN']) + const current = context() + const result = await executeFunctionExecute( + { + code: 'true', + language: 'shell', + outputs: { files: [{ path: 'files/review.txt', sandboxPath: 'saved.txt' }] }, + }, + current + ) + expect(result.success).toBe(true) + expect(io.write).toHaveBeenCalled() + const saved = io.write.mock.calls.at(-1)![0] + expect(saved.buffer.toString()).toBe(canary) + expect(saved.secretProvenance.status).toBe('exact') + expect(saved.secretProvenance.entries.length).toBeGreaterThan(0) + }) +}) diff --git a/apps/sim/lib/mothership/tools/sandbox-resource-transport.test.ts b/apps/sim/lib/mothership/tools/sandbox-resource-transport.test.ts index ad410d0c797..952b40b5f3b 100644 --- a/apps/sim/lib/mothership/tools/sandbox-resource-transport.test.ts +++ b/apps/sim/lib/mothership/tools/sandbox-resource-transport.test.ts @@ -303,7 +303,7 @@ it.each([ expect(await response.text()).toBe('filebytes') expect(recordInput).toHaveBeenCalledWith( 'mothership-chat:chat', - provenance.status === 'exact' && provenance.entries.length === 0 + provenance.status === 'exact' ? provenance : false ) expect(fetcher).toHaveBeenCalledOnce() } diff --git a/apps/sim/lib/mothership/tools/sandbox-resource-transport.ts b/apps/sim/lib/mothership/tools/sandbox-resource-transport.ts index 745358f5cce..635dbf15591 100644 --- a/apps/sim/lib/mothership/tools/sandbox-resource-transport.ts +++ b/apps/sim/lib/mothership/tools/sandbox-resource-transport.ts @@ -14,6 +14,7 @@ import { matchV2Route } from '@/lib/api/server/routes/in-process-transport' import { withWorkspaceInvocationScope } from '@/lib/core/application/workspace-invocation-scope' import { asOrchestrationError, statusForOrchestrationError } from '@/lib/core/orchestration/types' import { getInternalApiBaseUrl } from '@/lib/core/utils/urls' +import type { DurableSecretProvenance } from '@/lib/execution/durable-secret-provenance' import { recordExistingSessionFileInput } from '@/lib/execution/remote-sandbox/session-file-provenance' import { createResourceEffectTransport } from '@/lib/mothership/agent-cli/resource-effects' import { resolveInvocationWorkspace } from '@/lib/mothership/application/workspace-target' @@ -138,8 +139,8 @@ async function proxyAuthorizedSandboxRequest( encodeURIComponent(matched.params[key] ?? '') ) === path ) - const recordInput = (safe: boolean) => - recordExistingSessionFileInput(chatSandboxSessionKey(scope.chatId), safe) + const recordInput = (provenance: boolean | DurableSecretProvenance) => + recordExistingSessionFileInput(chatSandboxSessionKey(scope.chatId), provenance) const fileRead = method === 'GET' && [v2DownloadFileContract, v2ReadFileTextContract].some( @@ -153,7 +154,7 @@ async function proxyAuthorizedSandboxRequest( if (!publicCatalog && !fileRead && !blockCatalog) await recordInput(false) let observed = false const result = await observeWorkspaceFileDelivery(async (provenance) => { - await recordInput(provenance?.status === 'exact' && provenance.entries.length === 0) + await recordInput(provenance?.status === 'exact' ? provenance : false) observed = true }, dispatch) try { diff --git a/apps/sim/lib/secrets/application/use-cases.test.ts b/apps/sim/lib/secrets/application/use-cases.test.ts index ada9837e200..e932a46ccd0 100644 --- a/apps/sim/lib/secrets/application/use-cases.test.ts +++ b/apps/sim/lib/secrets/application/use-cases.test.ts @@ -8,6 +8,7 @@ import { credentialsEnvironmentMock, credentialsEnvironmentMockFns, } from '@sim/testing/mocks/credentials-environment.mock' +import { environmentUtilsMockFns } from '@sim/testing/mocks/environment-utils.mock' import { permissionsMock, permissionsMockFns } from '@sim/testing/mocks/permissions.mock' import { workspaceAuthzMock, workspaceAuthzMockFns } from '@sim/testing/mocks/workspace-authz.mock' import { @@ -28,7 +29,6 @@ const { mocks: hoisted } = vi.hoisted(() => ({ setPersonal: vi.fn(), deletePersonal: vi.fn(), listCredentials: vi.fn(), - readWorkspaceValues: vi.fn(), secretUsage: vi.fn(), scanReferences: vi.fn(), }, @@ -51,7 +51,6 @@ vi.mock('@/lib/secrets/usage/queries', () => ({ vi.mock('@/lib/credentials/secret-values', () => ({ deletePersonalSecret: hoisted.deletePersonal, deleteWorkspaceSecret: vi.fn(), - readWorkspaceSecretValues: hoisted.readWorkspaceValues, setPersonalSecret: hoisted.setPersonal, setWorkspaceSecret: hoisted.setWorkspace, updateWorkspaceSecretMetadata: hoisted.updateWorkspaceMetadata, @@ -136,7 +135,6 @@ describe('secret application use cases', () => { mocks.personalMetadata.mockResolvedValue(null) mocks.deletePersonal.mockResolvedValue(true) mocks.listCredentials.mockResolvedValue({ data: [secret], nextCursorKeys: null }) - mocks.readWorkspaceValues.mockResolvedValue({}) mocks.secretUsage.mockResolvedValue({ entries: [] }) }) @@ -145,8 +143,15 @@ describe('secret application use cases', () => { data: [secret, visibleSecret, personalSecret], nextCursorKeys: null, }) - mocks.readWorkspaceValues.mockResolvedValue({ - [visibleSecret.envKey]: 'https://staging.example.com', + environmentUtilsMockFns.mockGetEffectiveEnvironmentSnapshot.mockResolvedValueOnce({ + personalEncrypted: {}, + personalDecrypted: {}, + personalOwners: {}, + conflicts: [], + decryptionFailures: [], + workspaceEncrypted: { [visibleSecret.envKey]: 'encrypted-visible' }, + workspaceDecrypted: { [visibleSecret.envKey]: 'https://staging.example.com' }, + workspaceUnredactedKeys: [visibleSecret.envKey], }) const result = await listSecretsUseCase.execute({ @@ -159,13 +164,39 @@ describe('secret application use cases', () => { }, }) - expect(mocks.readWorkspaceValues).toHaveBeenCalledWith({ - workspaceId: workspace.workspaceId, - names: [visibleSecret.envKey], - }) expect(result.values).toEqual({ [visibleSecret.envKey]: 'https://staging.example.com' }) }) + it('withholds a visible value shared with a protected secret', async () => { + mocks.listCredentials.mockResolvedValue({ data: [visibleSecret], nextCursorKeys: null }) + environmentUtilsMockFns.mockGetEffectiveEnvironmentSnapshot.mockResolvedValueOnce({ + personalEncrypted: {}, + personalDecrypted: {}, + personalOwners: {}, + conflicts: [], + decryptionFailures: [], + workspaceEncrypted: { + [visibleSecret.envKey]: 'encrypted-visible', + HIDDEN: 'encrypted-hidden', + }, + workspaceDecrypted: { + [visibleSecret.envKey]: 'shared-protected-value', + HIDDEN: 'shared-protected-value', + }, + workspaceUnredactedKeys: [visibleSecret.envKey], + }) + const result = await listSecretsUseCase.execute({ + principal: session, + input: { + workspaceId: workspace.workspaceId, + sortBy: 'name', + sortOrder: 'asc', + limit: 50, + }, + }) + expect(result.values).toEqual({}) + }) + it('rejects workspace keys before resolving or reading secret state', async () => { const execute = setSecretUseCase.execute as (args: { principal: Principal diff --git a/apps/sim/lib/secrets/application/use-cases.ts b/apps/sim/lib/secrets/application/use-cases.ts index c031411e6e4..fc25760776a 100644 --- a/apps/sim/lib/secrets/application/use-cases.ts +++ b/apps/sim/lib/secrets/application/use-cases.ts @@ -5,6 +5,7 @@ import type { CursorKey, ListSortOrder } from '@/lib/api/list-query' import { defineAuthorizedWorkspaceUseCase } from '@/lib/core/application' import { ForbiddenOperationError } from '@/lib/core/application/forbidden' import { OrchestrationError } from '@/lib/core/orchestration/types' +import { setRecordValue } from '@/lib/core/utils/records' import { getPersonalEnvCredentialMetadata, getWorkspaceEnvKeyAdminAccess, @@ -17,16 +18,17 @@ import { import { deletePersonalSecret, deleteWorkspaceSecret, - readWorkspaceSecretValues, setPersonalSecret, setWorkspaceSecret, updateWorkspaceSecretMetadata, } from '@/lib/credentials/secret-values' +import { getEffectiveEnvironmentSnapshot } from '@/lib/environment/utils' import { secretOperations } from '@/lib/secrets/application/operations' import { scanSecretReferences } from '@/lib/secrets/references/scan' import { getSecretUsage } from '@/lib/secrets/usage/queries' import { loadActiveWorkspaceContext } from '@/lib/uploads/contexts/workspace' import { checkWorkspaceAccess } from '@/lib/workspaces/permissions/utils' +import { createResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' export type SecretScope = 'workspace' | 'personal' export type SecretSortBy = 'name' | 'createdAt' | 'updatedAt' @@ -242,17 +244,26 @@ export const listSecretsUseCase = defineAuthorizedWorkspaceUseCase({ }) /** * The one place a secret value rides a read response: rows the workspace marked - * visible (unredacted) — whose values already print into every run log this - * caller can open — so external agents don't have to scrape logs for them. - * Bounded by the page, and read from one environment row. + * visible (unredacted), provided the shared registry also permits that value. + * A visible alias must not expose the literal of a protected secret. Returned + * values remain bounded by the metadata page. */ const visibleNames = page.data.flatMap((row) => row.type === 'env_workspace' && row.unredacted && row.envKey ? [row.envKey] : [] ) - const values = await readWorkspaceSecretValues({ - workspaceId: context.workspaceId, - names: visibleNames, - }) + const values: Record = {} + if (visibleNames.length > 0) { + const snapshot = await getEffectiveEnvironmentSnapshot(userId, context.workspaceId) + const registry = await createResolvedSecretTraceRegistry({ + ...snapshot, + scope: { userId, workspaceId: context.workspaceId }, + }) + const visible = new Set(registry.getUnredactedSecretNames()) + for (const name of visibleNames) { + if (visible.has(name) && Object.hasOwn(snapshot.workspaceDecrypted, name)) + setRecordValue(values, name, snapshot.workspaceDecrypted[name]) + } + } return { secrets: page.data, values, diff --git a/apps/sim/lib/webhooks/provider-subscriptions.test.ts b/apps/sim/lib/webhooks/provider-subscriptions.test.ts index 380c72afa2f..45cb9f976a8 100644 --- a/apps/sim/lib/webhooks/provider-subscriptions.test.ts +++ b/apps/sim/lib/webhooks/provider-subscriptions.test.ts @@ -33,6 +33,30 @@ describe('createExternalWebhookSubscription', () => { mockGetEffectiveDecryptedEnv.mockResolvedValue({ ASHBY_API_KEY: 'real-secret-key' }) }) + it.each([ + ['{{ASHBY_API_KEY}}', '{{ASHBY_API_KEY}}'], + ['real-secret-key', '[REDACTED_SECRET]'], + ])( + 'projects configured credential %s out of provider failures while preserving status', + async (apiKey, replacement) => { + const failure = Object.assign(new Error('Provider refused real-secret-key'), { status: 429 }) + mockGetProviderHandler.mockReturnValue({ + createSubscription: async () => { + throw failure + }, + }) + await expect( + createExternalWebhookSubscription( + {} as NextRequest, + { provider: 'ashby', providerConfig: { apiKey } }, + { workspaceId: 'ws-1' }, + 'user-1', + 'req-1' + ) + ).rejects.toMatchObject({ message: `Provider refused ${replacement}`, status: 429 }) + } + ) + it('resolves {{ENV_VAR}} references in providerConfig before calling the provider', async () => { const createSubscription = vi.fn().mockResolvedValue({ providerConfigUpdates: { externalId: 'ext-1' }, @@ -100,6 +124,28 @@ describe('cleanupExternalWebhook', () => { * non-admin owner without a credential grant leave `{{VAR}}` unresolved, and * the provider was handed the literal reference as its credential. */ + it.each([ + ['{{CALENDLY_API_KEY}}', '{{CALENDLY_API_KEY}}'], + ['real-secret-key', '[REDACTED_SECRET]'], + ])( + 'keeps configured cleanup credential %s out of retryable deployment failures', + async (apiKey, replacement) => { + mockGetProviderHandler.mockReturnValue({ + deleteSubscription: async () => { + throw new Error('Provider refused real-secret-key') + }, + }) + await expect( + cleanupExternalWebhook( + { provider: 'calendly', providerConfig: { apiKey } }, + { userId: 'user-1', workspaceId: 'workspace-1' }, + 'req-1', + { throwOnError: true } + ) + ).rejects.toThrow(`Provider refused ${replacement}`) + } + ) + it('resolves {{ENV_VAR}} references before deleting the provider subscription', async () => { const deleteSubscription = vi.fn().mockResolvedValue(undefined) mockGetProviderHandler.mockReturnValue({ deleteSubscription }) diff --git a/apps/sim/lib/webhooks/provider-subscriptions.ts b/apps/sim/lib/webhooks/provider-subscriptions.ts index ddd7b11a0ee..ca4c0e6e5e4 100644 --- a/apps/sim/lib/webhooks/provider-subscriptions.ts +++ b/apps/sim/lib/webhooks/provider-subscriptions.ts @@ -1,17 +1,59 @@ import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' +import { getErrorMessage } from '@sim/utils/errors' import { omit } from '@sim/utils/object' import type { NextRequest } from 'next/server' import { withResourceOutboundScope } from '@/lib/core/network/resource-scope.server' +import { isSensitiveKey } from '@/lib/core/security/redaction' import { resolveBackgroundWebhookEnv, resolveWebhookProviderConfig, resolveWebhookRecordProviderConfig, } from '@/lib/webhooks/env-resolver' import { getProviderHandler } from '@/lib/webhooks/providers' +import { WebhookDeploymentConfigurationError } from '@/lib/webhooks/providers/errors' +import { + createResolvedSecretMatcher, + projectResolvedSecretContent, +} from '@/executor/utils/resolved-secret-content-projection' +import { OPAQUE_RESOLVED_SECRET_REPLACEMENT } from '@/executor/utils/resolved-secret-matcher' const logger = createLogger('WebhookProviderSubscriptions') +/** Resolving credentials must not make a provider's exception a durable plaintext export. */ +function projectProviderFailure( + error: unknown, + secrets: ReadonlyMap, + providerConfig: Record +): Error { + let message = 'Webhook provider request failed' + try { + const matches = [...secrets].map(([name, plaintext]) => ({ + plaintext, + replacement: `{{${name}}}`, + })) + const resolvedValues = new Set(secrets.values()) + for (const [field, value] of Object.entries(providerConfig)) { + if (isSensitiveKey(field) && typeof value === 'string' && !resolvedValues.has(value)) { + matches.push({ plaintext: value, replacement: OPAQUE_RESOLVED_SECRET_REPLACEMENT }) + } + } + const matcher = createResolvedSecretMatcher(matches) + const projection = matcher + ? projectResolvedSecretContent(getErrorMessage(error), matcher) + : { safe: true, value: getErrorMessage(error) } + if (projection.safe && typeof projection.value === 'string') message = projection.value + } catch { + /** Uninspectable diagnostics retain no provider-controlled content. */ + } + const projected = + error instanceof WebhookDeploymentConfigurationError + ? new WebhookDeploymentConfigurationError(message) + : new Error(message) + if (error instanceof Error && 'status' in error && typeof error.status === 'number') + Object.assign(projected, { status: error.status }) + return projected +} + type ExternalSubscriptionResult = { updatedProviderConfig: Record externalSubscriptionCreated: boolean @@ -141,10 +183,12 @@ export async function createExternalWebhookSubscription( const workspaceId = typeof workflow.workspaceId === 'string' ? workflow.workspaceId : undefined + const secrets = new Map() const resolvedProviderConfig = await resolveWebhookProviderConfig( providerConfig, userId, - workspaceId + workspaceId, + { onResolved: (name, value) => secrets.set(name, value) } ) /** @@ -161,6 +205,8 @@ export async function createExternalWebhookSubscription( requestId, request, }) + }).catch((error: unknown) => { + throw projectProviderFailure(error, secrets, resolvedProviderConfig) }) if (!result) { @@ -201,6 +247,8 @@ export async function cleanupExternalWebhook( return } + const secrets = new Map() + let resolvedProviderConfig: Record = {} try { if (typeof workflow.userId !== 'string') { throw new Error('Cannot resolve webhook credentials without a workflow owner') @@ -212,8 +260,9 @@ export async function cleanupExternalWebhook( webhook, workflow.userId, workspaceId, - { envVars } + { envVars, onResolved: (name, value) => secrets.set(name, value) } ) + resolvedProviderConfig = resolvedWebhook.providerConfig /** Workspace archival precedes provider cleanup; routing still uses its canonical owner. */ await withResourceOutboundScope( @@ -228,13 +277,14 @@ export async function cleanupExternalWebhook( { includeArchived: true } ) } catch (error) { + const projected = projectProviderFailure(error, secrets, resolvedProviderConfig) logger.warn(`[${requestId}] Error cleaning up external webhook (non-fatal)`, { provider, webhookId: webhook.id, - error: toError(error).message, + error: projected.message, }) if (options.throwOnError) { - throw error + throw projected } } }