fix(mothership): bound run_workflow block-log outputs before the secret projection

A run_workflow result echoes every block's output in `logs`. A run with many
row-shaped block outputs can exceed the projection's 100k-value traversal cap
while staying under its byte cap. Whenever the call had an active secret, the
whole result was then withheld, including the final output and error, and the
model saw a bare success.

Block-log outputs are now bounded to a quarter of each projection cap
(`MAX_CONTENT_NODES` values and the default byte cap). Past the budget, the
bulkiest outputs are replaced, largest first, with a
`logs get <executionId> --trace` pointer, in the same form the oversized-input
compaction already uses. The executionId, status, final output, error, and
`select` values are untouched, and the other three quarters of each cap stay
free for them.

Author: waleedlatif1

apps/sim/executor/utils/resolved-secret-content-projection.ts

@@ -23,7 +23,8 @@ export {
   scanResolvedSecretString,
 } from '@/executor/utils/resolved-secret-matcher'
 
-const MAX_CONTENT_NODES = 100_000
+/** Values one model-content projection will walk before refusing the whole payload. */
+export const MAX_CONTENT_NODES = 100_000
 const MAX_CONTENT_DEPTH = 100
 const INTERNAL_DIAGNOSTIC_IDENTIFIER_PATTERN =
   /__var_[A-Za-z0-9_]+|__sim_code_\d+_(?:binding|input|runtime)_\d+[A-Za-z0-9_]*|__sim_placeholder_[a-f0-9]{64}__|__sim_runtime_[A-Za-z0-9_]+_\d+[A-Za-z0-9_]*|__SIM_RUNTIME_PAYLOAD_PATH/g

apps/sim/lib/mothership/tools/handlers/workflow/mutations.ts

@@ -2,6 +2,7 @@ import { createLogger } from '@sim/logger'
 import { filterUndefined, isPlainRecord, isRecordLike } from '@sim/utils/object'
 import { createCopilotWorkspaceApiKey } from '@/lib/api-key/application/create-api-key'
 import { PlatformEvents } from '@/lib/core/telemetry'
+import { MAX_INLINE_MATERIALIZATION_BYTES } from '@/lib/execution/payloads/limits'
 import { messageForCopilotApplicationError } from '@/lib/mothership/application/error'
 import { executeCopilotApiKeyUseCase } from '@/lib/mothership/application/execute-api-key-use-case'
 import {
@@ -47,6 +48,7 @@ import {
 } from '@/lib/workflows/application/update-workflow-content'
 import { sanitizeForCopilot } from '@/lib/workflows/sanitization/json-sanitizer'
 import { hasExecutionResult, readAttemptedExecutionId } from '@/executor/utils/errors'
+import { MAX_CONTENT_NODES } from '@/executor/utils/resolved-secret-content-projection'
 import type { WorkflowState } from '@/stores/workflows/workflow/types'
 
 const logger = createLogger('WorkflowMutations')
@@ -60,7 +62,7 @@ const LOG_INPUT_KEEP_CHARS = 200
 /**
  * Compacts the block inputs echoed back in `logs`. A Function block's `input.code` embeds the
  * fully serialized upstream rows, so a seven-block run repeated the same rows several times
- * across ~14k chars of tool result. Outputs are never touched — they are what the run was for —
+ * across ~14k chars of tool result. Outputs are bounded separately by {@link compactBlockLogOutputs},
  * and the full input stays one `logs get <executionId> --trace` away.
  */
 function compactBlockLogInputs(logs: unknown, executionId: string | undefined): unknown {
@@ -80,6 +82,67 @@ function compactBlockLogInputs(logs: unknown, executionId: string | undefined):
   })
 }
 
+/**
+ * Budgets for the block outputs echoed back in `logs`, a quarter of each cap the model-facing
+ * projection enforces on the whole result. Reaching either cap withholds everything, including
+ * the final output and error the run was for; those stay intact here and share the remaining
+ * three quarters with the rest of the envelope. The value budget matters first: row-shaped
+ * outputs reach the projection's traversal cap long before its byte cap.
+ */
+const LOG_OUTPUT_VALUE_BUDGET = Math.floor(MAX_CONTENT_NODES / 4)
+const LOG_OUTPUT_BYTE_BUDGET = Math.floor(MAX_INLINE_MATERIALIZATION_BYTES / 4)
+
+/** Counts values the way the projection walks them, and the bytes they encode to. */
+function measureLogOutput(value: unknown): { values: number; bytes: number } {
+  let values = 0
+  const encoded = JSON.stringify(value, (_key, item) => {
+    values += 1
+    return item
+  })
+  return { values, bytes: encoded === undefined ? 0 : Buffer.byteLength(encoded, 'utf8') }
+}
+
+/**
+ * Replaces the bulkiest block outputs in `logs` with a pointer once they exceed the budgets
+ * above, largest first, so the rest of the run still reaches the model. The full outputs stay in
+ * the run's archived trace, one `logs get <executionId> --trace` away, matching how
+ * {@link compactBlockLogInputs} treats oversized inputs.
+ */
+function compactBlockLogOutputs(logs: unknown, executionId: string | undefined): unknown {
+  if (!Array.isArray(logs)) return logs
+  const reference = executionId ?? '<executionId>'
+  const sizes = logs.map((entry) =>
+    isPlainRecord(entry) && entry.output !== undefined ? measureLogOutput(entry.output) : undefined
+  )
+  let values = 0
+  let bytes = 0
+  for (const size of sizes) {
+    values += size?.values ?? 0
+    bytes += size?.bytes ?? 0
+  }
+  if (values <= LOG_OUTPUT_VALUE_BUDGET && bytes <= LOG_OUTPUT_BYTE_BUDGET) return logs
+
+  const compacted = [...logs]
+  const bulkiestFirst = sizes
+    .map((size, index) => ({ size, index }))
+    .filter((entry): entry is { size: { values: number; bytes: number }; index: number } =>
+      Boolean(entry.size)
+    )
+    .sort(
+      (left, right) => right.size.values - left.size.values || right.size.bytes - left.size.bytes
+    )
+  for (const { size, index } of bulkiestFirst) {
+    if (values <= LOG_OUTPUT_VALUE_BUDGET && bytes <= LOG_OUTPUT_BYTE_BUDGET) break
+    compacted[index] = {
+      ...(logs[index] as Record<string, unknown>),
+      output: `…[output omitted: ${size.values} values, ${size.bytes} bytes; see logs get ${reference} --trace]`,
+    }
+    values -= size.values
+    bytes -= size.bytes
+  }
+  return compacted
+}
+
 function stripBinaryFields(value: unknown): unknown {
   if (value === null || value === undefined) return value
   if (typeof value !== 'object') return value
@@ -182,7 +245,11 @@ function buildExecutionOutput(
       ...extra,
       output: lifted ? lifted.output : output,
       ...(lifted ? { outputFrom: lifted.outputFrom } : {}),
-      ...presentWorkflowLogs(logs, select),
+      // `select` reads full values from the run's own logs, so only the echoed logs are bounded.
+      ...presentWorkflowLogs(
+        select?.length ? logs : compactBlockLogOutputs(logs, executionId),
+        select
+      ),
     },
     error: result.success
       ? undefined

apps/sim/lib/mothership/tools/handlers/workflow/run-workflow-result-budget.test.ts

@@ -0,0 +1,234 @@
+/**
+ * A run_workflow result sized like the production refusals: thirteen table-query blocks,
+ * ~11.5k rows, ~9.7 MiB. That is under the 16 MiB byte cap, yet with one active secret the
+ * projection walked ~138k values against its 100k traversal cap and withheld the whole result,
+ * leaving the model a bare success. The model-facing result is now bounded before it reaches
+ * the projection, and the omitted block outputs stay reachable through the run's archived trace.
+ */
+
+import { executeWorkflowMock } from '@sim/testing/mocks/execute-workflow.mock'
+import {
+  largeValueMetadataMock,
+  largeValueMetadataMockFns,
+} from '@sim/testing/mocks/large-value-metadata.mock'
+import { storageServiceMockFns } from '@sim/testing/mocks/storage-service.mock'
+import { telemetryMock } from '@sim/testing/mocks/telemetry.mock'
+import { uploadsMock } from '@sim/testing/mocks/uploads.mock'
+import { workflowsOrchestrationMock } from '@sim/testing/mocks/workflows-orchestration.mock'
+import { getErrorMessage } from '@sim/utils/errors'
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+import { clearLargeValueCacheForTests } from '@/lib/execution/payloads/cache'
+import {
+  externalizeExecutionData,
+  materializeExecutionData,
+} from '@/lib/logs/execution/trace-store'
+import { inspectToolResultForCopilot } from '@/lib/mothership/request/tools/resolved-secret-result'
+import type { ExecutionContext } from '@/lib/mothership/request/types'
+import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry'
+
+const { mocks } = vi.hoisted(() => ({ mocks: { executeWorkflowUseCase: vi.fn() } }))
+
+vi.mock('@/lib/mothership/application/execute-workflow-use-case', () => ({
+  executeCopilotWorkflowUseCase: mocks.executeWorkflowUseCase,
+  messageForCopilotWorkflowError: (error: unknown, fallback = 'Workflow operation failed') =>
+    getErrorMessage(error, fallback),
+}))
+vi.mock('@/lib/workflows/sanitization/json-sanitizer', () => ({
+  sanitizeForCopilot: vi.fn((state) => state),
+}))
+vi.mock('@/lib/workflows/executor/execute-workflow', () => executeWorkflowMock)
+vi.mock('@/lib/execution/cancel-workflow-execution', () => ({
+  cancelWorkflowExecution: vi.fn(),
+  WorkflowExecutionNotFoundError: class WorkflowExecutionNotFoundError extends Error {},
+}))
+vi.mock('@/lib/workflows/orchestration', () => workflowsOrchestrationMock)
+vi.mock('@/lib/core/telemetry', () => telemetryMock)
+vi.mock('@/lib/uploads', () => uploadsMock)
+vi.mock('@/lib/execution/payloads/large-value-metadata', () => largeValueMetadataMock)
+
+import { executeRunWorkflow } from '@/lib/mothership/tools/handlers/workflow/mutations'
+
+const EXECUTION_ID = '0f4d5a4c-6a1e-4c2f-9b7d-2c8f1a3e5d90'
+const SECRET = 'fake-secret-for-test-only'
+const context = {
+  userId: 'user-1',
+  workspaceId: 'workspace-1',
+  toolCallId: 'tool-call-1',
+} as ExecutionContext
+
+/** Row counts and stored bytes of the table queries in one production refusal. */
+const TABLE_QUERIES: ReadonlyArray<readonly [rows: number, bytes: number]> = [
+  [32, 9_513],
+  [2, 2_590],
+  [1_848, 2_079_072],
+  [2_533, 1_698_688],
+  [4_833, 3_179_855],
+  [1_294, 1_517_841],
+  [32, 5_563],
+  [11, 3_246],
+  [835, 721_833],
+  [33, 52_482],
+  [28, 8_274],
+  [13, 9_669],
+  [33, 293_266],
+]
+
+function tableRows(count: number, bytes: number) {
+  const columns = 8
+  const width = Math.max(1, Math.floor(bytes / count / columns) - 12)
+  return Array.from({ length: count }, (_, index) => ({
+    id: `row_${index}`,
+    data: Object.fromEntries(
+      Array.from({ length: columns }, (_, column) => [`col_${column}`, 'x'.repeat(width)])
+    ),
+    createdAt: '2026-09-24T00:00:00.000Z',
+  }))
+}
+
+function traceShapedLogs() {
+  return TABLE_QUERIES.map(([rows, bytes], index) => {
+    const result = tableRows(rows, bytes)
+    return {
+      blockId: `query-${index}`,
+      blockName: `Query ${index}`,
+      success: true,
+      output: { rows: result, rowCount: result.length },
+    }
+  })
+}
+
+function secretRegistry() {
+  const registry = new ResolvedSecretTraceRegistry([
+    { name: 'API_KEY', plaintext: SECRET, encryptedValue: 'ciphertext' },
+  ])
+  registry.recordResolved('API_KEY', SECRET, { propagated: true })
+  return registry
+}
+
+describe('run_workflow model-facing result budget', () => {
+  beforeEach(() => {
+    mocks.executeWorkflowUseCase.mockReset()
+  })
+
+  it('projects a trace-shaped result with an active secret instead of withholding it', async () => {
+    const logs = traceShapedLogs()
+    const finalOutput = { summary: `report for ${SECRET}`, rowCount: 11_529 }
+    mocks.executeWorkflowUseCase.mockResolvedValue({
+      success: false,
+      error: `Report block failed after reading ${SECRET}`,
+      output: finalOutput,
+      logs,
+      metadata: { executionId: EXECUTION_ID },
+    })
+
+    const settled = await executeRunWorkflow({ workflowId: 'wf-1' }, context)
+    const projection = inspectToolResultForCopilot(settled, secretRegistry(), 'run_workflow')
+
+    expect(projection.safe).toBe(true)
+    const output = projection.result.output as Record<string, unknown>
+    expect(output.executionId).toBe(EXECUTION_ID)
+    expect(output.success).toBe(false)
+    expect(output.output).toEqual({ summary: 'report for {{API_KEY}}', rowCount: 11_529 })
+    expect(projection.result.error).toBe('Report block failed after reading {{API_KEY}}')
+    const presented = output.logs as Array<Record<string, unknown>>
+    expect(presented.map((log) => log.blockName)).toEqual(logs.map((log) => log.blockName))
+    const omitted = presented.filter((log) => typeof log.output === 'string')
+    expect(omitted.length).toBeGreaterThan(0)
+    for (const log of omitted) {
+      expect(log.output).toContain(`logs get ${EXECUTION_ID} --trace`)
+    }
+    // Small outputs still arrive in full; only the bulky ones are replaced.
+    expect(presented.find((log) => log.blockName === 'Query 1')?.output).toEqual(logs[1].output)
+  })
+
+  /** The final output is what the run was for, so it is never compacted and needs the headroom. */
+  it('leaves room for a large final output beside the bounded logs', async () => {
+    const finalOutput = { rows: tableRows(4_833, 3_179_855) }
+    mocks.executeWorkflowUseCase.mockResolvedValue({
+      success: true,
+      output: finalOutput,
+      logs: traceShapedLogs(),
+      metadata: { executionId: EXECUTION_ID },
+    })
+
+    const settled = await executeRunWorkflow({ workflowId: 'wf-1' }, context)
+    const projection = inspectToolResultForCopilot(settled, secretRegistry(), 'run_workflow')
+
+    expect(projection.safe).toBe(true)
+    expect((projection.result.output as Record<string, unknown>).output).toEqual(finalOutput)
+  })
+
+  /** Narrow rows reach the traversal cap while staying far under every byte budget. */
+  it('bounds narrow row outputs by value count alone', async () => {
+    const logs = Array.from({ length: 6 }, (_, index) => ({
+      blockId: `narrow-${index}`,
+      blockName: `Narrow ${index}`,
+      success: true,
+      output: { rows: tableRows(2_500, 2_500 * 8 * 13) },
+    }))
+    const finalOutput = { rows: tableRows(2_000, 2_000 * 8 * 13) }
+    mocks.executeWorkflowUseCase.mockResolvedValue({
+      success: true,
+      output: finalOutput,
+      logs,
+      metadata: { executionId: EXECUTION_ID },
+    })
+
+    const settled = await executeRunWorkflow({ workflowId: 'wf-1' }, context)
+    expect(Buffer.byteLength(JSON.stringify(settled.output))).toBeLessThan(4 * 1024 * 1024)
+    const projection = inspectToolResultForCopilot(settled, secretRegistry(), 'run_workflow')
+
+    expect(projection.safe).toBe(true)
+    expect((projection.result.output as Record<string, unknown>).output).toEqual(finalOutput)
+  })
+
+  it('keeps every omitted block output reachable through the pointer', async () => {
+    const logs = traceShapedLogs()
+    mocks.executeWorkflowUseCase.mockResolvedValue({
+      success: true,
+      output: {},
+      logs,
+      metadata: { executionId: EXECUTION_ID },
+    })
+    const settled = await executeRunWorkflow({ workflowId: 'wf-1' }, context)
+    const presented = (settled.output as { logs: Array<Record<string, unknown>> }).logs
+    const omitted = presented.filter((log) => typeof log.output === 'string')
+    expect(omitted.length).toBeGreaterThan(0)
+
+    // `logs get <id> --trace` reads the run's archived execution data; archive and read it back.
+    storageServiceMockFns.mockUploadFile.mockImplementation(async ({ customKey, file }) => {
+      storageServiceMockFns.mockDownloadFile.mockResolvedValue(file)
+      return { key: customKey }
+    })
+    largeValueMetadataMockFns.mockRegisterLargeValueOwner.mockResolvedValue(true)
+    clearLargeValueCacheForTests()
+    const archiveContext = {
+      workspaceId: 'workspace-1',
+      workflowId: 'wf-1',
+      executionId: EXECUTION_ID,
+      userId: 'user-1',
+    }
+    const slim = await externalizeExecutionData(
+      {
+        traceSpans: logs.map((log) => ({
+          id: log.blockId,
+          name: log.blockName,
+          output: log.output,
+        })),
+      },
+      archiveContext,
+      { throwOnError: true }
+    )
+    clearLargeValueCacheForTests()
+    const archived = (await materializeExecutionData(slim, archiveContext)) as {
+      traceSpans: Array<{ name: string; output: unknown }>
+    }
+
+    for (const log of omitted) {
+      const original = logs.find((entry) => entry.blockName === log.blockName)
+      expect(archived.traceSpans.find((span) => span.name === log.blockName)?.output).toEqual(
+        original?.output
+      )
+    }
+  })
+})