From 87277152bec2745a60d016ae96a06b68647987e7 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 28 Sep 2026 21:01:12 -0700 Subject: [PATCH 1/7] fix(logs): keep compacted child span trees shaped as trees Block output compaction spilled oversized child span lists to large-value references, so a span's children could stop being an array and trace span building threw while finalizing the run. That left pauses unpersisted and runs unfinalized. - Compact child span trees structurally, spilling only each span's payload fields - Drop non-list child spans with a warning when building trace spans - Finalize without spans if building them fails, so logs, pauses, and billing settle --- apps/sim/executor/execution/block-executor.ts | 4 +- .../lib/execution/payloads/serializer.test.ts | 126 +++++++++++++++++- apps/sim/lib/execution/payloads/serializer.ts | 67 +++++++++- .../execution/trace-spans/span-factory.ts | 32 +++-- .../execution/trace-spans/trace-spans.test.ts | 62 +++++++++ .../workflows/executor/execution-core.test.ts | 54 ++++++++ .../lib/workflows/executor/execution-core.ts | 33 ++++- 7 files changed, 360 insertions(+), 18 deletions(-) diff --git a/apps/sim/executor/execution/block-executor.ts b/apps/sim/executor/execution/block-executor.ts index 34ddc144e50..38f0f0b253e 100644 --- a/apps/sim/executor/execution/block-executor.ts +++ b/apps/sim/executor/execution/block-executor.ts @@ -7,7 +7,7 @@ import { isTimeoutAbortReason } from '@/lib/core/execution-limits/types' import { redactApiKeys } from '@/lib/core/security/redaction' import { normalizeStringArray } from '@/lib/core/utils/arrays' import { getBaseUrl } from '@/lib/core/utils/urls' -import { compactExecutionPayload } from '@/lib/execution/payloads/serializer' +import { compactBlockOutput } from '@/lib/execution/payloads/serializer' import { redactLargeValueRefsInValue } from '@/lib/logs/execution/pii-large-values' import { redactObjectStrings } from '@/lib/logs/execution/pii-redaction' import { @@ -379,7 +379,7 @@ export class BlockExecutor { normalizedOutput = await redactObjectStrings(normalizedOutput, redactionOptions) } - normalizedOutput = (await compactExecutionPayload(normalizedOutput, { + normalizedOutput = (await compactBlockOutput(normalizedOutput, { workspaceId: blockCtx.workspaceId, workflowId: blockCtx.workflowId, executionId: blockCtx.executionId, diff --git a/apps/sim/lib/execution/payloads/serializer.test.ts b/apps/sim/lib/execution/payloads/serializer.test.ts index 938a53c902e..3ecded1c619 100644 --- a/apps/sim/lib/execution/payloads/serializer.test.ts +++ b/apps/sim/lib/execution/payloads/serializer.test.ts @@ -15,8 +15,14 @@ import { getLargeValueMaterializationError, isLargeValueRef, } from '@/lib/execution/payloads/large-value-ref' -import { compactExecutionPayload, compactSubflowResults } from '@/lib/execution/payloads/serializer' -import type { UserFile } from '@/executor/types' +import { + compactBlockLogs, + compactBlockOutput, + compactExecutionPayload, + compactSubflowResults, +} from '@/lib/execution/payloads/serializer' +import type { TraceSpan } from '@/lib/logs/types' +import type { BlockLog, UserFile } from '@/executor/types' const { mockDownloadFile, mockUploadFile } = storageServiceMockFns @@ -295,3 +301,119 @@ describe('compactExecutionPayload', () => { expect(error.message).not.toContain('lv_CQcekP8gSJI5') }) }) + +/** + * A child workflow's spans as the workflow block reports them: a loop whose one + * iteration holds two block spans. With a 1 KiB threshold each block span stays + * inline, but the iteration's `children` array together is over it — the shape + * generic compaction turned into a manifest. + */ +function childWorkflowSpans(): TraceSpan[] { + const blockSpan = (id: string): TraceSpan => ({ + id, + name: id, + type: 'function', + duration: 1, + startTime: '2026-09-29T00:00:00.000Z', + endTime: '2026-09-29T00:00:00.001Z', + output: { result: 'x'.repeat(600) }, + }) + return [ + { + id: 'loop', + name: 'Loop', + type: 'loop', + duration: 2, + startTime: '2026-09-29T00:00:00.000Z', + endTime: '2026-09-29T00:00:00.002Z', + children: [ + { + id: 'iteration-0', + name: 'Iteration 0', + type: 'loop-iteration', + duration: 2, + startTime: '2026-09-29T00:00:00.000Z', + endTime: '2026-09-29T00:00:00.002Z', + children: [blockSpan('span-a'), blockSpan('span-b')], + }, + ], + }, + ] +} + +/** Asserts the loop → iteration → block span nesting survived with every `children` an array. */ +function expectSpanTree(spans: unknown): void { + expect(Array.isArray(spans)).toBe(true) + const [loop] = spans as TraceSpan[] + expect(Array.isArray(loop.children)).toBe(true) + const [iteration] = loop.children ?? [] + expect(Array.isArray(iteration.children)).toBe(true) + expect(iteration.children?.map((span) => span.id)).toEqual(['span-a', 'span-b']) +} + +describe('compacting span trees', () => { + const options = { thresholdBytes: 1024, requireDurable: true, ...TEST_EXECUTION_CONTEXT } + + beforeEach(() => { + clearLargeValueCacheForTests() + mockUploadFile.mockImplementation(async ({ customKey }) => ({ key: customKey })) + mockRegisterLargeValueOwner.mockResolvedValue(true) + }) + + it('keeps a block output child span tree shaped as a tree', async () => { + const compacted = await compactBlockOutput( + { result: 'done', childTraceSpans: childWorkflowSpans() }, + options + ) + + expectSpanTree(compacted.childTraceSpans) + }) + + it('still spills an oversized span payload', async () => { + const spans = childWorkflowSpans() + const iteration = spans[0].children?.[0] + if (iteration?.children) iteration.children[0].output = { result: 'y'.repeat(4096) } + + const compacted = await compactBlockOutput({ childTraceSpans: spans }, options) + + const spilled = (compacted.childTraceSpans as TraceSpan[])[0].children?.[0].children?.[0] + expect(isLargeValueRef(spilled?.output?.result)).toBe(true) + }) + + it('keeps block log child span trees shaped as trees', async () => { + const log = { + blockId: 'child-workflow', + blockType: 'workflow', + startedAt: '2026-09-29T00:00:00.000Z', + endedAt: '2026-09-29T00:00:00.002Z', + durationMs: 2, + success: true, + childTraceSpans: childWorkflowSpans(), + } as BlockLog + + const [compacted] = (await compactBlockLogs([log], options)) ?? [] + + expectSpanTree(compacted?.childTraceSpans) + }) + + it('keeps a nested child workflow span tree shaped as a tree', async () => { + const nestedWorkflowSpan: TraceSpan = { + id: 'nested-workflow', + name: 'Nested Workflow', + type: 'workflow', + duration: 2, + startTime: '2026-09-29T00:00:00.000Z', + endTime: '2026-09-29T00:00:00.002Z', + output: { result: 'done', childTraceSpans: childWorkflowSpans() }, + } + + const compacted = await compactBlockOutput( + { result: 'done', childTraceSpans: [nestedWorkflowSpan] }, + options + ) + + const [nested] = compacted.childTraceSpans as TraceSpan[] + expect(nested.output?.result).toBe('done') + expectSpanTree(nested.output?.childTraceSpans) + }) +}) diff --git a/apps/sim/lib/execution/payloads/serializer.ts b/apps/sim/lib/execution/payloads/serializer.ts index e8239f9b141..14acff28cf9 100644 --- a/apps/sim/lib/execution/payloads/serializer.ts +++ b/apps/sim/lib/execution/payloads/serializer.ts @@ -1,3 +1,4 @@ +import { isRecordLike } from '@sim/utils/object' import { PayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { isUserFileWithMetadata } from '@/lib/core/utils/user-file' import { @@ -254,6 +255,66 @@ export async function compactSubflowResults( return compactedResults } +/** + * Compacts a trace span tree without collapsing its structure. Readers walk + * `children` and `output.childTraceSpans` as arrays, so those stay arrays and + * only each span's payload fields are spilled when oversized. Size of the tree + * as a whole is bounded where the log is stored, not here. + */ +async function compactTraceSpanTree( + spans: unknown, + options: CompactExecutionPayloadOptions +): Promise { + if (!Array.isArray(spans)) { + return compactExecutionPayload(spans, options) + } + return Promise.all(spans.map((span) => compactTraceSpan(span, options))) +} + +async function compactTraceSpan( + span: unknown, + options: CompactExecutionPayloadOptions +): Promise { + if (!isRecordLike(span)) { + return compactExecutionPayload(span, options) + } + return Object.fromEntries( + await Promise.all( + Object.entries(span).map(async ([key, value]) => [ + key, + key === 'children' + ? await compactTraceSpanTree(value, options) + : key === 'output' + ? await compactBlockOutput(value, options) + : await compactExecutionPayload(value, options), + ]) + ) + ) +} + +/** + * Compacts a block output, keeping any `childTraceSpans` shaped as a span tree + * (see {@link compactTraceSpanTree}). Everything else compacts as a normal + * execution payload. + */ +export async function compactBlockOutput( + output: T, + options: CompactExecutionPayloadOptions = {} +): Promise { + if (!isRecordLike(output) || !('childTraceSpans' in output)) { + return compactExecutionPayload(output, options) + } + const { childTraceSpans, ...rest } = output + const [compactedRest, compactedSpans] = await Promise.all([ + compactExecutionPayload(rest, options), + compactTraceSpanTree(childTraceSpans, options), + ]) + if (!isRecordLike(compactedRest)) { + return compactedRest as T + } + return { ...compactedRest, childTraceSpans: compactedSpans } as T +} + export async function compactBlockLogs( logs: BlockLog[] | undefined, options: CompactExecutionPayloadOptions = {} @@ -276,13 +337,13 @@ export async function compactBlockLogs( compactedLog.input = await compactExecutionPayload(compactedLog.input, options) } if ('output' in compactedLog) { - compactedLog.output = await compactExecutionPayload(compactedLog.output, options) + compactedLog.output = await compactBlockOutput(compactedLog.output, options) } if ('childTraceSpans' in compactedLog) { - compactedLog.childTraceSpans = await compactExecutionPayload( + compactedLog.childTraceSpans = (await compactTraceSpanTree( compactedLog.childTraceSpans, options - ) + )) as BlockLog['childTraceSpans'] } compactedLogs[index] = compactedLog } diff --git a/apps/sim/lib/logs/execution/trace-spans/span-factory.ts b/apps/sim/lib/logs/execution/trace-spans/span-factory.ts index 7abe0bcbc71..335336a8e99 100644 --- a/apps/sim/lib/logs/execution/trace-spans/span-factory.ts +++ b/apps/sim/lib/logs/execution/trace-spans/span-factory.ts @@ -392,8 +392,11 @@ function resolveToolCallsList(output: NormalizedBlockOutput | undefined): BlockT /** Extracts and flattens child workflow trace spans into the parent span's children. */ function attachChildWorkflowSpans(span: TraceSpan, log: ValidBlockLog): void { - const childTraceSpans = log.childTraceSpans ?? log.output?.childTraceSpans - if (!childTraceSpans?.length) return + const childTraceSpans = readChildSpans( + log.childTraceSpans ?? log.output?.childTraceSpans, + 'childTraceSpans' + ) + if (childTraceSpans.length === 0) return span.children = flattenWorkflowChildren(childTraceSpans) span.output = stripChildTraceSpansFromOutput(span.output) @@ -404,10 +407,21 @@ function isSyntheticWorkflowWrapper(span: TraceSpan): boolean { return span.type === 'workflow' && !span.blockId } +/** + * Reads a list of child spans, or `[]` when absent. Anything other than an + * array (for example a span list that was spilled to a large-value reference) + * is dropped with a warning so one malformed subtree cannot fail the trace. + */ +function readChildSpans(value: unknown, source: 'children' | 'childTraceSpans'): TraceSpan[] { + if (value === undefined || value === null) return [] + if (Array.isArray(value)) return value + logger.warn('Dropping child spans that are not a list', { source, shape: typeof value }) + return [] +} + /** Reads nested `childTraceSpans` off a span's output, or `[]` if absent. */ function extractOutputChildren(output: TraceSpan['output']): TraceSpan[] { - const nested = (output as { childTraceSpans?: TraceSpan[] } | undefined)?.childTraceSpans - return Array.isArray(nested) ? nested : [] + return readChildSpans(output?.childTraceSpans, 'childTraceSpans') } /** Returns a copy of `output` with `childTraceSpans` removed, or undefined unchanged. */ @@ -430,21 +444,21 @@ export function flattenWorkflowChildren(spans: TraceSpan[]): TraceSpan[] { for (const span of spans) { if (isSyntheticWorkflowWrapper(span)) { - if (span.children?.length) { - flattened.push(...flattenWorkflowChildren(span.children)) - } + flattened.push(...flattenWorkflowChildren(readChildSpans(span.children, 'children'))) continue } - const directChildren = span.children ?? [] + const directChildren = readChildSpans(span.children, 'children') const outputChildren = extractOutputChildren(span.output) const allChildren = [...directChildren, ...outputChildren] const nextSpan: TraceSpan = { ...span } if (allChildren.length > 0) { nextSpan.children = flattenWorkflowChildren(allChildren) + } else if (!Array.isArray(span.children)) { + nextSpan.children = undefined } - if (outputChildren.length > 0) { + if (span.output && 'childTraceSpans' in span.output) { nextSpan.output = stripChildTraceSpansFromOutput(nextSpan.output) } diff --git a/apps/sim/lib/logs/execution/trace-spans/trace-spans.test.ts b/apps/sim/lib/logs/execution/trace-spans/trace-spans.test.ts index 495e8ddb28d..4c4a8211d40 100644 --- a/apps/sim/lib/logs/execution/trace-spans/trace-spans.test.ts +++ b/apps/sim/lib/logs/execution/trace-spans/trace-spans.test.ts @@ -1419,3 +1419,65 @@ describe('custom block invoked as an Agent tool', () => { expect(toolSpan?.output).not.toHaveProperty('_childTraceDisabled') }) }) + +describe('child span trees that were compacted into references', () => { + const spilledChildren = { + __simLargeValueRef: true, + version: 1, + id: 'lv_ABCDEFGHIJKLMNOPQRSTUV', + kind: 'array', + size: 9_000_000, + } as unknown as TraceSpan[] + + function childWorkflowResult(childTraceSpans: TraceSpan[]): ExecutionResult { + return { + success: true, + output: {}, + logs: [ + { + blockId: 'workflow-1', + blockName: 'Child Workflow', + blockType: 'workflow', + startedAt: '2024-01-01T10:00:00.000Z', + endedAt: '2024-01-01T10:00:05.000Z', + durationMs: 5000, + success: true, + output: { success: true }, + childTraceSpans, + }, + ], + } + } + + const loopSpan = (overrides: Partial): TraceSpan => ({ + id: 'loop-1', + name: 'Loop', + type: 'loop', + blockId: 'loop-1', + duration: 1000, + startTime: '2024-01-01T10:00:01.000Z', + endTime: '2024-01-01T10:00:02.000Z', + status: 'success', + ...overrides, + }) + + it.concurrent('keeps the span when its children are a reference', () => { + const { traceSpans } = buildTraceSpans( + childWorkflowResult([loopSpan({ children: spilledChildren })]) + ) + + expect(traceSpans[0].children?.map((span) => span.id)).toEqual(['loop-1']) + expect(traceSpans[0].children?.[0].children).toBeUndefined() + }) + + it.concurrent('keeps the span when its output child spans are a reference', () => { + const { traceSpans } = buildTraceSpans( + childWorkflowResult([ + loopSpan({ output: { result: 'done', childTraceSpans: spilledChildren } }), + ]) + ) + + expect(traceSpans[0].children?.map((span) => span.id)).toEqual(['loop-1']) + expect(traceSpans[0].children?.[0].output).toEqual({ result: 'done' }) + }) +}) diff --git a/apps/sim/lib/workflows/executor/execution-core.test.ts b/apps/sim/lib/workflows/executor/execution-core.test.ts index 613dcab195f..37ce5f47360 100644 --- a/apps/sim/lib/workflows/executor/execution-core.test.ts +++ b/apps/sim/lib/workflows/executor/execution-core.test.ts @@ -1570,6 +1570,60 @@ describe('executeWorkflowCore terminal finalization sequencing', () => { expect(clearExecutionCancellationMock).not.toHaveBeenCalled() }) + it('still persists a pause when its trace spans cannot be built', async () => { + buildTraceSpansMock.mockImplementation(() => { + throw new TypeError('directChildren is not iterable') + }) + executorExecuteMock.mockResolvedValue({ + success: true, + status: 'paused', + output: {}, + logs: [], + metadata: { duration: 123, startTime: 'start', endTime: 'end' }, + executionState: { blockStates: {} }, + }) + + await executeWorkflowCore({ + snapshot: createSnapshot() as any, + callbacks: {}, + loggingSession: loggingSession as any, + }) + await loggingSession.setPostExecutionPromise.mock.calls[0][0] + + expect(safeCompleteWithPauseMock).toHaveBeenCalledWith( + expect.objectContaining({ totalDurationMs: 123, traceSpans: [] }) + ) + }) + + it('still finalizes a failed execution when its trace spans cannot be built', async () => { + buildTraceSpansMock.mockImplementation(() => { + throw new TypeError('directChildren is not iterable') + }) + const error = Object.assign(new Error('block threw'), { + executionResult: { + success: false, + output: {}, + logs: [], + metadata: { duration: 55, startTime: 'start', endTime: 'end' }, + }, + }) + executorExecuteMock.mockRejectedValue(error) + + await expect( + executeWorkflowCore({ + snapshot: createSnapshot() as any, + callbacks: {}, + loggingSession: loggingSession as any, + }) + ).rejects.toBe(error) + await loggingSession.setPostExecutionPromise.mock.calls[0][0] + + expect(safeCompleteWithErrorMock).toHaveBeenCalledWith( + expect.objectContaining({ traceSpans: [] }) + ) + expect(wasExecutionFinalizedByCore(error, 'execution-1')).toBe(true) + }) + it('clears cancellation intent when pause finalization observes a persisted cancellation', async () => { executorExecuteMock.mockResolvedValue({ success: true, diff --git a/apps/sim/lib/workflows/executor/execution-core.ts b/apps/sim/lib/workflows/executor/execution-core.ts index 420be743ab9..fa337c3cef1 100644 --- a/apps/sim/lib/workflows/executor/execution-core.ts +++ b/apps/sim/lib/workflows/executor/execution-core.ts @@ -290,6 +290,28 @@ async function recordSettledRun(workflowId: string, requestId: string): Promise< } } +/** + * Builds a run's trace spans for its log. Spans are diagnostics: a failure to + * build them is logged and the run is finalized without them, so the log, + * pause, billing, and run counts still settle. + */ +function buildTraceSpansForLog( + result: ExecutionResult, + loggingSession: LoggingSession, + requestId: string, + executionId: string +): ReturnType { + try { + return buildTraceSpans(result) + } catch (error) { + logger.error( + `[${requestId}] Failed to build trace spans; finalizing without them`, + loggingSession.projectDiagnosticError(error, { executionId }) + ) + return { traceSpans: [], totalDuration: result.metadata?.duration ?? 0 } + } +} + async function finalizeExecutionOutcome(params: { result: ExecutionResult loggingSession: LoggingSession @@ -301,7 +323,12 @@ async function finalizeExecutionOutcome(params: { }): Promise { const { result, loggingSession, workflowId, executionId, requestId, workflowInput, abortSignal } = params - const { traceSpans, totalDuration } = buildTraceSpans(result) + const { traceSpans, totalDuration } = buildTraceSpansForLog( + result, + loggingSession, + requestId, + executionId + ) const endedAt = new Date().toISOString() try { @@ -372,7 +399,9 @@ async function finalizeExecutionError(params: { }): Promise { const { error, loggingSession, workflowId, executionId, requestId } = params const executionResult = hasExecutionResult(error) ? error.executionResult : undefined - const { traceSpans } = executionResult ? buildTraceSpans(executionResult) : { traceSpans: [] } + const { traceSpans } = executionResult + ? buildTraceSpansForLog(executionResult, loggingSession, requestId, executionId) + : { traceSpans: [] } let finalized = false try { From 44a75760c9c95d2ce45ca33017afed141b21f3c5 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 28 Sep 2026 21:10:56 -0700 Subject: [PATCH 2/7] fix(logs): keep an output carrying child spans a record when compacting it --- apps/sim/lib/execution/payloads/serializer.test.ts | 12 ++++++++++++ apps/sim/lib/execution/payloads/serializer.ts | 8 +++----- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/apps/sim/lib/execution/payloads/serializer.test.ts b/apps/sim/lib/execution/payloads/serializer.test.ts index 3ecded1c619..01c3457c3d3 100644 --- a/apps/sim/lib/execution/payloads/serializer.test.ts +++ b/apps/sim/lib/execution/payloads/serializer.test.ts @@ -396,6 +396,18 @@ describe('compacting span trees', () => { expectSpanTree(compacted?.childTraceSpans) }) + it('keeps the output a record when its other fields together exceed the threshold', async () => { + const compacted = await compactBlockOutput( + { first: 'a'.repeat(600), second: 'b'.repeat(600), childTraceSpans: childWorkflowSpans() }, + options + ) + + expect(isLargeValueRef(compacted)).toBe(false) + expect(compacted.first).toBe('a'.repeat(600)) + expect(compacted.second).toBe('b'.repeat(600)) + expectSpanTree(compacted.childTraceSpans) + }) + it('keeps a nested child workflow span tree shaped as a tree', async () => { const nestedWorkflowSpan: TraceSpan = { id: 'nested-workflow', diff --git a/apps/sim/lib/execution/payloads/serializer.ts b/apps/sim/lib/execution/payloads/serializer.ts index 14acff28cf9..6a5a30b6a02 100644 --- a/apps/sim/lib/execution/payloads/serializer.ts +++ b/apps/sim/lib/execution/payloads/serializer.ts @@ -295,7 +295,8 @@ async function compactTraceSpan( /** * Compacts a block output, keeping any `childTraceSpans` shaped as a span tree * (see {@link compactTraceSpanTree}). Everything else compacts as a normal - * execution payload. + * execution payload, except that an output carrying spans keeps its root so + * the spans stay attached to it; its fields still spill individually. */ export async function compactBlockOutput( output: T, @@ -306,12 +307,9 @@ export async function compactBlockOutput( } const { childTraceSpans, ...rest } = output const [compactedRest, compactedSpans] = await Promise.all([ - compactExecutionPayload(rest, options), + compactExecutionPayload(rest, { ...options, preserveRoot: true }), compactTraceSpanTree(childTraceSpans, options), ]) - if (!isRecordLike(compactedRest)) { - return compactedRest as T - } return { ...compactedRest, childTraceSpans: compactedSpans } as T } From 405f9bfb6b80a9f5aea0bfe4643a1153fe3e3774 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 28 Sep 2026 21:27:03 -0700 Subject: [PATCH 3/7] fix(logs): split child spans off block output so oversized state still spills --- apps/sim/executor/execution/block-executor.ts | 10 +- .../lib/execution/payloads/serializer.test.ts | 94 +++++++++++++------ apps/sim/lib/execution/payloads/serializer.ts | 80 ++++++++++++---- 3 files changed, 133 insertions(+), 51 deletions(-) diff --git a/apps/sim/executor/execution/block-executor.ts b/apps/sim/executor/execution/block-executor.ts index 38f0f0b253e..680a2b6f73d 100644 --- a/apps/sim/executor/execution/block-executor.ts +++ b/apps/sim/executor/execution/block-executor.ts @@ -379,14 +379,15 @@ export class BlockExecutor { normalizedOutput = await redactObjectStrings(normalizedOutput, redactionOptions) } - normalizedOutput = (await compactBlockOutput(normalizedOutput, { + const compacted = await compactBlockOutput(normalizedOutput, { workspaceId: blockCtx.workspaceId, workflowId: blockCtx.workflowId, executionId: blockCtx.executionId, userId: blockCtx.userId, preserveUserFileBase64: blockCtx.includeFileBase64 === true, requireDurable: true, - })) as NormalizedBlockOutput + }) + normalizedOutput = compacted.output const endedAt = new Date().toISOString() const duration = performance.now() - startTime @@ -396,8 +397,8 @@ export class BlockExecutor { blockLog.durationMs = duration blockLog.success = true blockLog.output = filterOutputForLog(block.metadata?.id || '', normalizedOutput, { block }) - if (normalizedOutput.childTraceSpans && Array.isArray(normalizedOutput.childTraceSpans)) { - blockLog.childTraceSpans = normalizedOutput.childTraceSpans + if (Array.isArray(compacted.childTraceSpans)) { + blockLog.childTraceSpans = compacted.childTraceSpans } const childExecutionId = normalizedOutput[CHILD_EXECUTION_ID_OUTPUT_KEY] if (typeof childExecutionId === 'string' && childExecutionId) { @@ -409,7 +410,6 @@ export class BlockExecutor { } const { - childTraceSpans: _traces, [CHILD_EXECUTION_ID_OUTPUT_KEY]: _childExecutionId, [CHILD_TRACE_DISABLED_OUTPUT_KEY]: _childTraceDisabled, ...outputForState diff --git a/apps/sim/lib/execution/payloads/serializer.test.ts b/apps/sim/lib/execution/payloads/serializer.test.ts index 01c3457c3d3..e3b268542ef 100644 --- a/apps/sim/lib/execution/payloads/serializer.test.ts +++ b/apps/sim/lib/execution/payloads/serializer.test.ts @@ -360,12 +360,23 @@ describe('compacting span trees', () => { mockRegisterLargeValueOwner.mockResolvedValue(true) }) - it('keeps a block output child span tree shaped as a tree', async () => { + const childWorkflowLog = (overrides: Partial): BlockLog => ({ + blockId: 'child-workflow', + blockType: 'workflow', + startedAt: '2026-09-29T00:00:00.000Z', + endedAt: '2026-09-29T00:00:00.002Z', + durationMs: 2, + success: true, + ...overrides, + }) + + it('splits a block output child span tree off shaped as a tree', async () => { const compacted = await compactBlockOutput( { result: 'done', childTraceSpans: childWorkflowSpans() }, options ) + expect(compacted.output).toEqual({ result: 'done' }) expectSpanTree(compacted.childTraceSpans) }) @@ -380,34 +391,46 @@ describe('compacting span trees', () => { expect(isLargeValueRef(spilled?.output?.result)).toBe(true) }) - it('keeps block log child span trees shaped as trees', async () => { - const log = { - blockId: 'child-workflow', - blockType: 'workflow', - startedAt: '2026-09-29T00:00:00.000Z', - endedAt: '2026-09-29T00:00:00.002Z', - durationMs: 2, - success: true, - childTraceSpans: childWorkflowSpans(), - } as BlockLog - - const [compacted] = (await compactBlockLogs([log], options)) ?? [] - - expectSpanTree(compacted?.childTraceSpans) - }) - - it('keeps the output a record when its other fields together exceed the threshold', async () => { + it('still spills a block output whose fields together exceed the threshold', async () => { const compacted = await compactBlockOutput( { first: 'a'.repeat(600), second: 'b'.repeat(600), childTraceSpans: childWorkflowSpans() }, options ) - expect(isLargeValueRef(compacted)).toBe(false) - expect(compacted.first).toBe('a'.repeat(600)) - expect(compacted.second).toBe('b'.repeat(600)) + expect(isLargeValueRef(compacted.output)).toBe(true) expectSpanTree(compacted.childTraceSpans) }) + it('keeps block log child span trees shaped as trees', async () => { + const [compacted] = + (await compactBlockLogs( + [childWorkflowLog({ childTraceSpans: childWorkflowSpans() })], + options + )) ?? [] + + expectSpanTree(compacted?.childTraceSpans) + }) + + it('keeps a block log output carrying child spans a record', async () => { + const [compacted] = + (await compactBlockLogs( + [ + childWorkflowLog({ + output: { + first: 'a'.repeat(600), + second: 'b'.repeat(600), + childTraceSpans: childWorkflowSpans(), + }, + }), + ], + options + )) ?? [] + + expect(isLargeValueRef(compacted?.output)).toBe(false) + expect(compacted?.output?.first).toBe('a'.repeat(600)) + expectSpanTree(compacted?.output?.childTraceSpans) + }) + it('keeps a nested child workflow span tree shaped as a tree', async () => { const nestedWorkflowSpan: TraceSpan = { id: 'nested-workflow', @@ -416,16 +439,33 @@ describe('compacting span trees', () => { duration: 2, startTime: '2026-09-29T00:00:00.000Z', endTime: '2026-09-29T00:00:00.002Z', - output: { result: 'done', childTraceSpans: childWorkflowSpans() }, + output: { + first: 'a'.repeat(600), + second: 'b'.repeat(600), + childTraceSpans: childWorkflowSpans(), + }, } - const compacted = await compactBlockOutput( - { result: 'done', childTraceSpans: [nestedWorkflowSpan] }, - options - ) + const compacted = await compactBlockOutput({ childTraceSpans: [nestedWorkflowSpan] }, options) const [nested] = compacted.childTraceSpans as TraceSpan[] - expect(nested.output?.result).toBe('done') + expect(nested.output?.first).toBe('a'.repeat(600)) expectSpanTree(nested.output?.childTraceSpans) }) + + it('terminates on a cyclic span tree', async () => { + const span: TraceSpan = { + id: 'cyclic', + name: 'Cyclic', + type: 'function', + duration: 1, + startTime: '2026-09-29T00:00:00.000Z', + endTime: '2026-09-29T00:00:00.001Z', + } + span.children = [span] + + const compacted = await compactBlockOutput({ childTraceSpans: [span] }, options) + + expect((compacted.childTraceSpans as TraceSpan[])[0].id).toBe('cyclic') + }) }) diff --git a/apps/sim/lib/execution/payloads/serializer.ts b/apps/sim/lib/execution/payloads/serializer.ts index 6a5a30b6a02..745ea64e82f 100644 --- a/apps/sim/lib/execution/payloads/serializer.ts +++ b/apps/sim/lib/execution/payloads/serializer.ts @@ -258,34 +258,40 @@ export async function compactSubflowResults( /** * Compacts a trace span tree without collapsing its structure. Readers walk * `children` and `output.childTraceSpans` as arrays, so those stay arrays and - * only each span's payload fields are spilled when oversized. Size of the tree - * as a whole is bounded where the log is stored, not here. + * only each span's payload fields are spilled when oversized. Spans are log + * data: the tree as a whole is bounded where the log is stored, not here. */ async function compactTraceSpanTree( spans: unknown, - options: CompactExecutionPayloadOptions + options: CompactExecutionPayloadOptions, + seen: WeakSet ): Promise { if (!Array.isArray(spans)) { return compactExecutionPayload(spans, options) } - return Promise.all(spans.map((span) => compactTraceSpan(span, options))) + return Promise.all(spans.map((span) => compactTraceSpan(span, options, seen))) } async function compactTraceSpan( span: unknown, - options: CompactExecutionPayloadOptions + options: CompactExecutionPayloadOptions, + seen: WeakSet ): Promise { if (!isRecordLike(span)) { return compactExecutionPayload(span, options) } + if (seen.has(span)) { + return span + } + seen.add(span) return Object.fromEntries( await Promise.all( Object.entries(span).map(async ([key, value]) => [ key, key === 'children' - ? await compactTraceSpanTree(value, options) + ? await compactTraceSpanTree(value, options, seen) : key === 'output' - ? await compactBlockOutput(value, options) + ? await compactLoggedOutput(value, options, seen) : await compactExecutionPayload(value, options), ]) ) @@ -293,24 +299,55 @@ async function compactTraceSpan( } /** - * Compacts a block output, keeping any `childTraceSpans` shaped as a span tree - * (see {@link compactTraceSpanTree}). Everything else compacts as a normal - * execution payload, except that an output carrying spans keeps its root so - * the spans stay attached to it; its fields still spill individually. + * Compacts a span or block log output. One carrying `childTraceSpans` keeps its + * root so the spans stay attached; its other fields spill individually. + */ +async function compactLoggedOutput( + output: unknown, + options: CompactExecutionPayloadOptions, + seen: WeakSet +): Promise { + if (!isRecordLike(output) || !('childTraceSpans' in output)) { + return compactExecutionPayload(output, options) + } + return Object.fromEntries( + await Promise.all( + Object.entries(output).map(async ([key, value]) => [ + key, + key === 'childTraceSpans' + ? await compactTraceSpanTree(value, options, seen) + : await compactExecutionPayload(value, options), + ]) + ) + ) +} + +export interface CompactedBlockOutput { + /** The output without `childTraceSpans`, compacted as execution state. */ + output: T + /** The output's child span tree, compacted as log data. */ + childTraceSpans?: unknown +} + +/** + * Compacts a block output for execution state and splits off its + * `childTraceSpans`, which belong to the block log rather than state. The + * output compacts as any execution payload, so an oversized one still spills + * whole; the spans compact as a tree (see {@link compactTraceSpanTree}). */ export async function compactBlockOutput( output: T, options: CompactExecutionPayloadOptions = {} -): Promise { +): Promise> { if (!isRecordLike(output) || !('childTraceSpans' in output)) { - return compactExecutionPayload(output, options) + return { output: await compactExecutionPayload(output, options) } } const { childTraceSpans, ...rest } = output - const [compactedRest, compactedSpans] = await Promise.all([ - compactExecutionPayload(rest, { ...options, preserveRoot: true }), - compactTraceSpanTree(childTraceSpans, options), + const [compactedOutput, compactedSpans] = await Promise.all([ + compactExecutionPayload(rest, options), + compactTraceSpanTree(childTraceSpans, options, new WeakSet()), ]) - return { ...compactedRest, childTraceSpans: compactedSpans } as T + return { output: compactedOutput as T, childTraceSpans: compactedSpans } } export async function compactBlockLogs( @@ -335,12 +372,17 @@ export async function compactBlockLogs( compactedLog.input = await compactExecutionPayload(compactedLog.input, options) } if ('output' in compactedLog) { - compactedLog.output = await compactBlockOutput(compactedLog.output, options) + compactedLog.output = (await compactLoggedOutput( + compactedLog.output, + options, + new WeakSet() + )) as BlockLog['output'] } if ('childTraceSpans' in compactedLog) { compactedLog.childTraceSpans = (await compactTraceSpanTree( compactedLog.childTraceSpans, - options + options, + new WeakSet() )) as BlockLog['childTraceSpans'] } compactedLogs[index] = compactedLog From 3f074f62a7a2aef40036d8b8d4831a8203adfc24 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 28 Sep 2026 21:59:21 -0700 Subject: [PATCH 4/7] fix(logs): keep a child span tree whole or drop it, bounded as before A structurally compacted tree had no whole-tree bound, so a large one stayed inline in block logs and pause snapshots. A tree still over the threshold after its payloads spill is now dropped (or rejected), as generic compaction bounded it. Block log outputs compact generically again; new logs never carry child spans there. --- apps/sim/executor/execution/block-executor.ts | 2 +- .../lib/execution/payloads/serializer.test.ts | 98 +++++++-------- apps/sim/lib/execution/payloads/serializer.ts | 113 +++++++++++------- .../execution/trace-spans/span-factory.ts | 4 +- 4 files changed, 124 insertions(+), 93 deletions(-) diff --git a/apps/sim/executor/execution/block-executor.ts b/apps/sim/executor/execution/block-executor.ts index 680a2b6f73d..e893f5dde92 100644 --- a/apps/sim/executor/execution/block-executor.ts +++ b/apps/sim/executor/execution/block-executor.ts @@ -397,7 +397,7 @@ export class BlockExecutor { blockLog.durationMs = duration blockLog.success = true blockLog.output = filterOutputForLog(block.metadata?.id || '', normalizedOutput, { block }) - if (Array.isArray(compacted.childTraceSpans)) { + if (compacted.childTraceSpans) { blockLog.childTraceSpans = compacted.childTraceSpans } const childExecutionId = normalizedOutput[CHILD_EXECUTION_ID_OUTPUT_KEY] diff --git a/apps/sim/lib/execution/payloads/serializer.test.ts b/apps/sim/lib/execution/payloads/serializer.test.ts index e3b268542ef..34149fe5b9a 100644 --- a/apps/sim/lib/execution/payloads/serializer.test.ts +++ b/apps/sim/lib/execution/payloads/serializer.test.ts @@ -2,7 +2,7 @@ import { largeValueMetadataMock, largeValueMetadataMockFns, } from '@sim/testing/mocks/large-value-metadata.mock' -import { storageServiceMockFns } from '@sim/testing/mocks/storage-service.mock' +import { storageServiceMock, storageServiceMockFns } from '@sim/testing/mocks/storage-service.mock' import { uploadsMock } from '@sim/testing/mocks/uploads.mock' import { beforeEach, describe, expect, it, vi } from 'vitest' import { clearLargeValueCacheForTests } from '@/lib/execution/payloads/cache' @@ -27,6 +27,7 @@ import type { BlockLog, UserFile } from '@/executor/types' const { mockDownloadFile, mockUploadFile } = storageServiceMockFns vi.mock('@/lib/uploads', () => uploadsMock) +vi.mock('@/lib/uploads/core/storage-service', () => storageServiceMock) vi.mock('@/lib/execution/payloads/large-value-metadata', () => largeValueMetadataMock) @@ -304,11 +305,9 @@ describe('compactExecutionPayload', () => { /** * A child workflow's spans as the workflow block reports them: a loop whose one - * iteration holds two block spans. With a 1 KiB threshold each block span stays - * inline, but the iteration's `children` array together is over it — the shape - * generic compaction turned into a manifest. + * iteration holds two block spans, each with a `resultBytes` payload. */ -function childWorkflowSpans(): TraceSpan[] { +function childWorkflowSpans(resultBytes: number): TraceSpan[] { const blockSpan = (id: string): TraceSpan => ({ id, name: id, @@ -316,7 +315,7 @@ function childWorkflowSpans(): TraceSpan[] { duration: 1, startTime: '2026-09-29T00:00:00.000Z', endTime: '2026-09-29T00:00:00.001Z', - output: { result: 'x'.repeat(600) }, + output: { result: 'x'.repeat(resultBytes) }, }) return [ { @@ -341,6 +340,16 @@ function childWorkflowSpans(): TraceSpan[] { ] } +/** Spans whose payloads each exceed the 4 KiB test threshold, so each spills on its own. */ +const spansWithLargePayloads = () => childWorkflowSpans(8192) + +/** + * Spans whose payloads each stay under the 4 KiB test threshold but whose + * iteration `children` together exceed it — the shape generic compaction + * turned into a manifest nested inside the tree. + */ +const spansTooLargeAsAWhole = () => childWorkflowSpans(2500) + /** Asserts the loop → iteration → block span nesting survived with every `children` an array. */ function expectSpanTree(spans: unknown): void { expect(Array.isArray(spans)).toBe(true) @@ -352,7 +361,7 @@ function expectSpanTree(spans: unknown): void { } describe('compacting span trees', () => { - const options = { thresholdBytes: 1024, requireDurable: true, ...TEST_EXECUTION_CONTEXT } + const options = { thresholdBytes: 4096, requireDurable: true, ...TEST_EXECUTION_CONTEXT } beforeEach(() => { clearLargeValueCacheForTests() @@ -370,30 +379,45 @@ describe('compacting span trees', () => { ...overrides, }) - it('splits a block output child span tree off shaped as a tree', async () => { + it('splits a block output child span tree off, spilling each oversized payload', async () => { const compacted = await compactBlockOutput( - { result: 'done', childTraceSpans: childWorkflowSpans() }, + { result: 'done', childTraceSpans: spansWithLargePayloads() }, options ) expect(compacted.output).toEqual({ result: 'done' }) expectSpanTree(compacted.childTraceSpans) + const [loop] = compacted.childTraceSpans as TraceSpan[] + const spilled = loop.children?.[0].children?.[0] + expect(isLargeValueRef(spilled?.output?.result)).toBe(true) }) - it('still spills an oversized span payload', async () => { - const spans = childWorkflowSpans() - const iteration = spans[0].children?.[0] - if (iteration?.children) iteration.children[0].output = { result: 'y'.repeat(4096) } + it('drops a block output child span tree too large as a whole', async () => { + const compacted = await compactBlockOutput( + { result: 'done', childTraceSpans: spansTooLargeAsAWhole() }, + options + ) - const compacted = await compactBlockOutput({ childTraceSpans: spans }, options) + expect(compacted.output).toEqual({ result: 'done' }) + expect(compacted.childTraceSpans).toBeUndefined() + }) - const spilled = (compacted.childTraceSpans as TraceSpan[])[0].children?.[0].children?.[0] - expect(isLargeValueRef(spilled?.output?.result)).toBe(true) + it('rejects a child span tree too large as a whole when large values are rejected', async () => { + await expect( + compactBlockOutput( + { childTraceSpans: spansTooLargeAsAWhole() }, + { ...options, rejectLargeValues: true } + ) + ).rejects.toThrow() }) it('still spills a block output whose fields together exceed the threshold', async () => { const compacted = await compactBlockOutput( - { first: 'a'.repeat(600), second: 'b'.repeat(600), childTraceSpans: childWorkflowSpans() }, + { + first: 'a'.repeat(2500), + second: 'b'.repeat(2500), + childTraceSpans: spansWithLargePayloads(), + }, options ) @@ -401,34 +425,18 @@ describe('compacting span trees', () => { expectSpanTree(compacted.childTraceSpans) }) - it('keeps block log child span trees shaped as trees', async () => { - const [compacted] = - (await compactBlockLogs( - [childWorkflowLog({ childTraceSpans: childWorkflowSpans() })], - options - )) ?? [] - - expectSpanTree(compacted?.childTraceSpans) - }) - - it('keeps a block log output carrying child spans a record', async () => { - const [compacted] = + it('keeps block log child span trees whole or drops them', async () => { + const compacted = (await compactBlockLogs( [ - childWorkflowLog({ - output: { - first: 'a'.repeat(600), - second: 'b'.repeat(600), - childTraceSpans: childWorkflowSpans(), - }, - }), + childWorkflowLog({ childTraceSpans: spansWithLargePayloads() }), + childWorkflowLog({ childTraceSpans: spansTooLargeAsAWhole() }), ], options )) ?? [] - expect(isLargeValueRef(compacted?.output)).toBe(false) - expect(compacted?.output?.first).toBe('a'.repeat(600)) - expectSpanTree(compacted?.output?.childTraceSpans) + expectSpanTree(compacted[0]?.childTraceSpans) + expect(compacted[1]?.childTraceSpans).toBeUndefined() }) it('keeps a nested child workflow span tree shaped as a tree', async () => { @@ -439,17 +447,13 @@ describe('compacting span trees', () => { duration: 2, startTime: '2026-09-29T00:00:00.000Z', endTime: '2026-09-29T00:00:00.002Z', - output: { - first: 'a'.repeat(600), - second: 'b'.repeat(600), - childTraceSpans: childWorkflowSpans(), - }, + output: { result: 'done', childTraceSpans: spansWithLargePayloads() }, } const compacted = await compactBlockOutput({ childTraceSpans: [nestedWorkflowSpan] }, options) const [nested] = compacted.childTraceSpans as TraceSpan[] - expect(nested.output?.first).toBe('a'.repeat(600)) + expect(nested.output?.result).toBe('done') expectSpanTree(nested.output?.childTraceSpans) }) @@ -464,8 +468,6 @@ describe('compacting span trees', () => { } span.children = [span] - const compacted = await compactBlockOutput({ childTraceSpans: [span] }, options) - - expect((compacted.childTraceSpans as TraceSpan[])[0].id).toBe('cyclic') + await expect(compactBlockOutput({ childTraceSpans: [span] }, options)).resolves.toBeDefined() }) }) diff --git a/apps/sim/lib/execution/payloads/serializer.ts b/apps/sim/lib/execution/payloads/serializer.ts index 745ea64e82f..56c4f80b40c 100644 --- a/apps/sim/lib/execution/payloads/serializer.ts +++ b/apps/sim/lib/execution/payloads/serializer.ts @@ -1,3 +1,4 @@ +import { createLogger } from '@sim/logger' import { isRecordLike } from '@sim/utils/object' import { PayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { isUserFileWithMetadata } from '@/lib/core/utils/user-file' @@ -10,8 +11,11 @@ import { LARGE_VALUE_THRESHOLD_BYTES, } from '@/lib/execution/payloads/large-value-ref' import { type LargeValueStoreContext, storeLargeValue } from '@/lib/execution/payloads/store' +import type { TraceSpan } from '@/lib/logs/types' import type { BlockLog } from '@/executor/types' +const logger = createLogger('ExecutionPayloadSerializer') + export interface CompactExecutionPayloadOptions extends LargeValueStoreContext { thresholdBytes?: number preserveUserFileBase64?: boolean @@ -255,11 +259,22 @@ export async function compactSubflowResults( return compactedResults } +/** Maps each entry of a record concurrently, keeping its keys. */ +async function mapEntriesAsync( + record: Record, + mapValue: (key: string, value: unknown) => Promise +): Promise> { + return Object.fromEntries( + await Promise.all( + Object.entries(record).map(async ([key, value]) => [key, await mapValue(key, value)]) + ) + ) +} + /** - * Compacts a trace span tree without collapsing its structure. Readers walk - * `children` and `output.childTraceSpans` as arrays, so those stay arrays and - * only each span's payload fields are spilled when oversized. Spans are log - * data: the tree as a whole is bounded where the log is stored, not here. + * Compacts a trace span tree without collapsing its structure: `children` and + * `output.childTraceSpans` stay arrays and only each span's payload fields + * spill when oversized. See {@link compactChildTraceSpans} for the size bound. */ async function compactTraceSpanTree( spans: unknown, @@ -284,25 +299,19 @@ async function compactTraceSpan( return span } seen.add(span) - return Object.fromEntries( - await Promise.all( - Object.entries(span).map(async ([key, value]) => [ - key, - key === 'children' - ? await compactTraceSpanTree(value, options, seen) - : key === 'output' - ? await compactLoggedOutput(value, options, seen) - : await compactExecutionPayload(value, options), - ]) - ) - ) + return mapEntriesAsync(span, (key, value) => { + if (key === 'children') return compactTraceSpanTree(value, options, seen) + if (key === 'output') return compactSpanOutput(value, options, seen) + return compactExecutionPayload(value, options) + }) } /** - * Compacts a span or block log output. One carrying `childTraceSpans` keeps its - * root so the spans stay attached; its other fields spill individually. + * Compacts a span's output. One carrying a nested child workflow's + * `childTraceSpans` keeps its root so the spans stay attached; its other + * fields spill individually. */ -async function compactLoggedOutput( +async function compactSpanOutput( output: unknown, options: CompactExecutionPayloadOptions, seen: WeakSet @@ -310,30 +319,57 @@ async function compactLoggedOutput( if (!isRecordLike(output) || !('childTraceSpans' in output)) { return compactExecutionPayload(output, options) } - return Object.fromEntries( - await Promise.all( - Object.entries(output).map(async ([key, value]) => [ - key, - key === 'childTraceSpans' - ? await compactTraceSpanTree(value, options, seen) - : await compactExecutionPayload(value, options), - ]) - ) + return mapEntriesAsync(output, (key, value) => + key === 'childTraceSpans' + ? compactTraceSpanTree(value, options, seen) + : compactExecutionPayload(value, options) ) } +/** + * Compacts a block's child span tree for its log. Readers walk span trees as + * arrays, so a tree is kept whole or not at all: payload fields spill + * individually (see {@link compactTraceSpanTree}), and a tree still over the + * threshold as a whole is dropped, bounding it as generic compaction did. + */ +async function compactChildTraceSpans( + spans: unknown, + options: CompactExecutionPayloadOptions +): Promise { + if (!Array.isArray(spans)) { + if (spans !== undefined) { + logger.warn('Dropping child trace spans that are not a list', { shape: typeof spans }) + } + return undefined + } + const compacted = await compactTraceSpanTree(spans, options, new WeakSet()) + const measured = getJsonAndSize(compacted) + const maxBytes = options.thresholdBytes ?? LARGE_VALUE_THRESHOLD_BYTES + if (measured && measured.size <= maxBytes) { + return compacted as TraceSpan[] + } + if (measured && options.rejectLargeValues) { + throw largeValueLimitError(options, measured.size) + } + logger.warn('Dropping child trace spans too large to keep', { + observedBytes: measured?.size, + maxBytes, + }) + return undefined +} + export interface CompactedBlockOutput { /** The output without `childTraceSpans`, compacted as execution state. */ output: T - /** The output's child span tree, compacted as log data. */ - childTraceSpans?: unknown + /** The output's child span tree for the block log (see {@link compactChildTraceSpans}). */ + childTraceSpans?: TraceSpan[] } /** * Compacts a block output for execution state and splits off its * `childTraceSpans`, which belong to the block log rather than state. The * output compacts as any execution payload, so an oversized one still spills - * whole; the spans compact as a tree (see {@link compactTraceSpanTree}). + * whole. */ export async function compactBlockOutput( output: T, @@ -345,7 +381,7 @@ export async function compactBlockOutput( const { childTraceSpans, ...rest } = output const [compactedOutput, compactedSpans] = await Promise.all([ compactExecutionPayload(rest, options), - compactTraceSpanTree(childTraceSpans, options, new WeakSet()), + compactChildTraceSpans(childTraceSpans, options), ]) return { output: compactedOutput as T, childTraceSpans: compactedSpans } } @@ -372,18 +408,13 @@ export async function compactBlockLogs( compactedLog.input = await compactExecutionPayload(compactedLog.input, options) } if ('output' in compactedLog) { - compactedLog.output = (await compactLoggedOutput( - compactedLog.output, - options, - new WeakSet() - )) as BlockLog['output'] + compactedLog.output = await compactExecutionPayload(compactedLog.output, options) } if ('childTraceSpans' in compactedLog) { - compactedLog.childTraceSpans = (await compactTraceSpanTree( + compactedLog.childTraceSpans = await compactChildTraceSpans( compactedLog.childTraceSpans, - options, - new WeakSet() - )) as BlockLog['childTraceSpans'] + options + ) } compactedLogs[index] = compactedLog } diff --git a/apps/sim/lib/logs/execution/trace-spans/span-factory.ts b/apps/sim/lib/logs/execution/trace-spans/span-factory.ts index 335336a8e99..af9b981aed5 100644 --- a/apps/sim/lib/logs/execution/trace-spans/span-factory.ts +++ b/apps/sim/lib/logs/execution/trace-spans/span-factory.ts @@ -458,9 +458,7 @@ export function flattenWorkflowChildren(spans: TraceSpan[]): TraceSpan[] { } else if (!Array.isArray(span.children)) { nextSpan.children = undefined } - if (span.output && 'childTraceSpans' in span.output) { - nextSpan.output = stripChildTraceSpansFromOutput(nextSpan.output) - } + nextSpan.output = stripChildTraceSpansFromOutput(nextSpan.output) flattened.push(nextSpan) } From 5ce7f18fcbff62c86b8041eb7018f17620ecc590 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 28 Sep 2026 22:16:22 -0700 Subject: [PATCH 5/7] fix(logs): keep the skeleton of a child span tree too large to keep whole A tree over the threshold as a whole now keeps its shape, names, timing, status, and cost instead of disappearing, using the same content stripping the execution log applies to oversized traces (moved to a shared module). Only a tree whose skeleton is still over the threshold is dropped. --- .../lib/execution/payloads/serializer.test.ts | 21 +++++- apps/sim/lib/execution/payloads/serializer.ts | 37 ++++++++--- apps/sim/lib/logs/execution/logger.ts | 64 ++---------------- .../logs/execution/trace-spans/summarize.ts | 65 +++++++++++++++++++ 4 files changed, 116 insertions(+), 71 deletions(-) create mode 100644 apps/sim/lib/logs/execution/trace-spans/summarize.ts diff --git a/apps/sim/lib/execution/payloads/serializer.test.ts b/apps/sim/lib/execution/payloads/serializer.test.ts index 34149fe5b9a..0d5ebcd34f7 100644 --- a/apps/sim/lib/execution/payloads/serializer.test.ts +++ b/apps/sim/lib/execution/payloads/serializer.test.ts @@ -392,13 +392,26 @@ describe('compacting span trees', () => { expect(isLargeValueRef(spilled?.output?.result)).toBe(true) }) - it('drops a block output child span tree too large as a whole', async () => { + it('keeps the skeleton of a block output child span tree too large as a whole', async () => { const compacted = await compactBlockOutput( { result: 'done', childTraceSpans: spansTooLargeAsAWhole() }, options ) expect(compacted.output).toEqual({ result: 'done' }) + expectSpanTree(compacted.childTraceSpans) + const [loop] = compacted.childTraceSpans as TraceSpan[] + expect(loop.children?.[0].children?.[0].output).toBeUndefined() + }) + + it('drops a child span tree whose skeleton alone exceeds the threshold', async () => { + const spans = Array.from({ length: 64 }, (_, index) => ({ + ...spansTooLargeAsAWhole()[0], + id: `loop-${index}`, + })) + + const compacted = await compactBlockOutput({ childTraceSpans: spans }, options) + expect(compacted.childTraceSpans).toBeUndefined() }) @@ -425,7 +438,7 @@ describe('compacting span trees', () => { expectSpanTree(compacted.childTraceSpans) }) - it('keeps block log child span trees whole or drops them', async () => { + it('keeps block log child span trees whole or as a skeleton', async () => { const compacted = (await compactBlockLogs( [ @@ -436,7 +449,9 @@ describe('compacting span trees', () => { )) ?? [] expectSpanTree(compacted[0]?.childTraceSpans) - expect(compacted[1]?.childTraceSpans).toBeUndefined() + expectSpanTree(compacted[1]?.childTraceSpans) + const [loop] = compacted[1]?.childTraceSpans ?? [] + expect(loop.children?.[0].children?.[0].output).toBeUndefined() }) it('keeps a nested child workflow span tree shaped as a tree', async () => { diff --git a/apps/sim/lib/execution/payloads/serializer.ts b/apps/sim/lib/execution/payloads/serializer.ts index 56c4f80b40c..38bc4ee67ae 100644 --- a/apps/sim/lib/execution/payloads/serializer.ts +++ b/apps/sim/lib/execution/payloads/serializer.ts @@ -11,6 +11,7 @@ import { LARGE_VALUE_THRESHOLD_BYTES, } from '@/lib/execution/payloads/large-value-ref' import { type LargeValueStoreContext, storeLargeValue } from '@/lib/execution/payloads/store' +import { summarizeTraceSpansWithoutIo } from '@/lib/logs/execution/trace-spans/summarize' import type { TraceSpan } from '@/lib/logs/types' import type { BlockLog } from '@/executor/types' @@ -328,9 +329,10 @@ async function compactSpanOutput( /** * Compacts a block's child span tree for its log. Readers walk span trees as - * arrays, so a tree is kept whole or not at all: payload fields spill - * individually (see {@link compactTraceSpanTree}), and a tree still over the - * threshold as a whole is dropped, bounding it as generic compaction did. + * arrays, so a tree is never collapsed: payload fields spill individually (see + * {@link compactTraceSpanTree}). A tree still over the threshold as a whole + * keeps only its skeleton (shape, names, timing, status, cost), and one whose + * skeleton is still over it is dropped, bounding it as generic compaction did. */ async function compactChildTraceSpans( spans: unknown, @@ -342,17 +344,34 @@ async function compactChildTraceSpans( } return undefined } - const compacted = await compactTraceSpanTree(spans, options, new WeakSet()) - const measured = getJsonAndSize(compacted) + const compacted = (await compactTraceSpanTree( + spans, + options, + new WeakSet() + )) as TraceSpan[] const maxBytes = options.thresholdBytes ?? LARGE_VALUE_THRESHOLD_BYTES - if (measured && measured.size <= maxBytes) { - return compacted as TraceSpan[] + const measured = getJsonAndSize(compacted) + if (!measured) { + logger.warn('Dropping child trace spans that cannot be serialized') + return undefined } - if (measured && options.rejectLargeValues) { + if (measured.size <= maxBytes) { + return compacted + } + if (options.rejectLargeValues) { throw largeValueLimitError(options, measured.size) } + const skeleton = summarizeTraceSpansWithoutIo(compacted) + const skeletonSize = getJsonAndSize(skeleton)?.size + if (skeletonSize !== undefined && skeletonSize <= maxBytes) { + logger.warn('Kept only the skeleton of child trace spans too large to keep whole', { + observedBytes: measured.size, + maxBytes, + }) + return skeleton + } logger.warn('Dropping child trace spans too large to keep', { - observedBytes: measured?.size, + observedBytes: skeletonSize ?? measured.size, maxBytes, }) return undefined diff --git a/apps/sim/lib/logs/execution/logger.ts b/apps/sim/lib/logs/execution/logger.ts index 533e9629604..b6ca886ba74 100644 --- a/apps/sim/lib/logs/execution/logger.ts +++ b/apps/sim/lib/logs/execution/logger.ts @@ -50,6 +50,11 @@ import { } from '@/lib/logs/execution/progress-markers' import { snapshotService } from '@/lib/logs/execution/snapshot/service' import { traceSpansHaveHandledErrors } from '@/lib/logs/execution/trace-spans/handled-errors' +import { + stripLegacyToolCallContent, + stripModelToolCallArguments, + summarizeTraceSpansWithoutIo, +} from '@/lib/logs/execution/trace-spans/summarize' import { traceSpansIndicateFailure } from '@/lib/logs/execution/trace-spans/trace-spans' import { copyTraceSpansWithoutCosts, @@ -195,12 +200,6 @@ function retainBoundedTraceContent(value: T, maxBytes = MAX_TRACE_IO_BYTES): return size !== undefined && size <= maxBytes ? value : undefined } -function stripModelToolCallArguments( - calls: NonNullable -): NonNullable { - return calls.map(({ arguments: _arguments, ...call }) => call as (typeof calls)[number]) -} - function compactModelToolCalls( calls: NonNullable ): NonNullable | undefined { @@ -226,12 +225,6 @@ function compactLegacyToolCalls( return retainBoundedTraceContent(compacted) } -function stripLegacyToolCallContent( - calls: NonNullable -): NonNullable { - return calls.map(({ input: _input, output: _output, error: _error, ...call }) => call) -} - function compactProviderTiming( providerTiming: NonNullable ): NonNullable { @@ -253,26 +246,6 @@ function compactProviderTiming( } } -function stripProviderTimingContent( - providerTiming: NonNullable -): NonNullable { - return { - ...providerTiming, - segments: providerTiming.segments.map( - ({ - assistantContent: _assistantContent, - thinkingContent: _thinkingContent, - errorMessage: _errorMessage, - toolCalls, - ...segment - }) => ({ - ...segment, - ...(toolCalls ? { toolCalls: stripModelToolCallArguments(toolCalls) } : {}), - }) - ), - } -} - function summarizeTraceSpansForExecutionData(traceSpans?: TraceSpan[]): TraceSpan[] | undefined { if (!traceSpans) { return traceSpans @@ -317,33 +290,6 @@ function summarizeTraceSpansForExecutionData(traceSpans?: TraceSpan[]): TraceSpa }) } -function summarizeTraceSpansWithoutIo(traceSpans?: TraceSpan[]): TraceSpan[] | undefined { - if (!traceSpans) { - return traceSpans - } - - return traceSpans.map((span) => { - const { - input: _input, - output: _output, - children, - thinking: _thinking, - errorMessage: _errorMessage, - modelToolCalls, - toolCalls, - providerTiming, - ...rest - } = span - return { - ...rest, - ...(modelToolCalls ? { modelToolCalls: stripModelToolCallArguments(modelToolCalls) } : {}), - ...(toolCalls ? { toolCalls: stripLegacyToolCallContent(toolCalls) } : {}), - ...(providerTiming ? { providerTiming: stripProviderTimingContent(providerTiming) } : {}), - ...(children?.length ? { children: summarizeTraceSpansWithoutIo(children) } : {}), - } - }) -} - function summarizeExecutionState(executionState?: SerializableExecutionState) { if (!executionState) { return undefined diff --git a/apps/sim/lib/logs/execution/trace-spans/summarize.ts b/apps/sim/lib/logs/execution/trace-spans/summarize.ts new file mode 100644 index 00000000000..43d6c9982ad --- /dev/null +++ b/apps/sim/lib/logs/execution/trace-spans/summarize.ts @@ -0,0 +1,65 @@ +import type { TraceSpan } from '@/lib/logs/types' + +export function stripModelToolCallArguments( + calls: NonNullable +): NonNullable { + return calls.map(({ arguments: _arguments, ...call }) => call as (typeof calls)[number]) +} + +export function stripLegacyToolCallContent( + calls: NonNullable +): NonNullable { + return calls.map(({ input: _input, output: _output, error: _error, ...call }) => call) +} + +export function stripProviderTimingContent( + providerTiming: NonNullable +): NonNullable { + return { + ...providerTiming, + segments: providerTiming.segments.map( + ({ + assistantContent: _assistantContent, + thinkingContent: _thinkingContent, + errorMessage: _errorMessage, + toolCalls, + ...segment + }) => ({ + ...segment, + ...(toolCalls ? { toolCalls: stripModelToolCallArguments(toolCalls) } : {}), + }) + ), + } +} + +/** + * A trace span tree with every span's content removed: inputs, outputs, + * thinking, error text, tool-call arguments, and provider content. Keeps the + * tree's shape, names, timing, status, and cost. + */ +export function summarizeTraceSpansWithoutIo(traceSpans?: TraceSpan[]): TraceSpan[] | undefined { + if (!traceSpans) { + return traceSpans + } + + return traceSpans.map((span) => { + const { + input: _input, + output: _output, + children, + thinking: _thinking, + errorMessage: _errorMessage, + modelToolCalls, + toolCalls, + providerTiming, + ...rest + } = span + return { + ...rest, + ...(modelToolCalls ? { modelToolCalls: stripModelToolCallArguments(modelToolCalls) } : {}), + ...(toolCalls ? { toolCalls: stripLegacyToolCallContent(toolCalls) } : {}), + ...(providerTiming ? { providerTiming: stripProviderTimingContent(providerTiming) } : {}), + ...(children?.length ? { children: summarizeTraceSpansWithoutIo(children) } : {}), + } + }) +} From 2e2206dcd11a3a4cf5b2b2325fd266251d54287b Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 28 Sep 2026 22:33:41 -0700 Subject: [PATCH 6/7] fix(logs): keep span trees well formed and keep nested child workflows in the skeleton The structural walk now keeps only span objects and drops a child list that is not an array, so building a skeleton can never throw on a malformed entry. The skeleton keeps a nested child workflow's output.childTraceSpans. --- .../lib/execution/payloads/serializer.test.ts | 32 +++++++++++++++++++ apps/sim/lib/execution/payloads/serializer.ts | 20 ++++++------ .../logs/execution/trace-spans/summarize.ts | 10 ++++-- 3 files changed, 50 insertions(+), 12 deletions(-) diff --git a/apps/sim/lib/execution/payloads/serializer.test.ts b/apps/sim/lib/execution/payloads/serializer.test.ts index 0d5ebcd34f7..c66f4666bec 100644 --- a/apps/sim/lib/execution/payloads/serializer.test.ts +++ b/apps/sim/lib/execution/payloads/serializer.test.ts @@ -404,6 +404,38 @@ describe('compacting span trees', () => { expect(loop.children?.[0].children?.[0].output).toBeUndefined() }) + it('drops malformed span entries so an oversized tree still keeps its skeleton', async () => { + const spans = spansTooLargeAsAWhole() + const iteration = spans[0].children?.[0] + iteration?.children?.push(null as unknown as TraceSpan) + + const compacted = await compactBlockOutput( + { childTraceSpans: [...spans, undefined as unknown as TraceSpan] }, + options + ) + + expectSpanTree(compacted.childTraceSpans) + expect(compacted.childTraceSpans).toHaveLength(1) + }) + + it('keeps nested child workflow trees in the skeleton', async () => { + const nestedWorkflowSpan: TraceSpan = { + id: 'nested-workflow', + name: 'Nested Workflow', + type: 'workflow', + duration: 2, + startTime: '2026-09-29T00:00:00.000Z', + endTime: '2026-09-29T00:00:00.002Z', + output: { result: 'done', childTraceSpans: spansTooLargeAsAWhole() }, + } + + const compacted = await compactBlockOutput({ childTraceSpans: [nestedWorkflowSpan] }, options) + + const [nested] = compacted.childTraceSpans as TraceSpan[] + expect(nested.output?.result).toBeUndefined() + expectSpanTree(nested.output?.childTraceSpans) + }) + it('drops a child span tree whose skeleton alone exceeds the threshold', async () => { const spans = Array.from({ length: 64 }, (_, index) => ({ ...spansTooLargeAsAWhole()[0], diff --git a/apps/sim/lib/execution/payloads/serializer.ts b/apps/sim/lib/execution/payloads/serializer.ts index 38bc4ee67ae..7d7680a3429 100644 --- a/apps/sim/lib/execution/payloads/serializer.ts +++ b/apps/sim/lib/execution/payloads/serializer.ts @@ -274,28 +274,28 @@ async function mapEntriesAsync( /** * Compacts a trace span tree without collapsing its structure: `children` and - * `output.childTraceSpans` stay arrays and only each span's payload fields - * spill when oversized. See {@link compactChildTraceSpans} for the size bound. + * `output.childTraceSpans` stay arrays of spans and only each span's payload + * fields spill when oversized. A malformed list or entry is dropped, so every + * reader can walk the tree. See {@link compactChildTraceSpans} for the size bound. */ async function compactTraceSpanTree( spans: unknown, options: CompactExecutionPayloadOptions, seen: WeakSet -): Promise { +): Promise { if (!Array.isArray(spans)) { - return compactExecutionPayload(spans, options) + return undefined } - return Promise.all(spans.map((span) => compactTraceSpan(span, options, seen))) + return Promise.all( + spans.filter(isRecordLike).map((span) => compactTraceSpan(span, options, seen)) + ) } async function compactTraceSpan( - span: unknown, + span: Record, options: CompactExecutionPayloadOptions, seen: WeakSet -): Promise { - if (!isRecordLike(span)) { - return compactExecutionPayload(span, options) - } +): Promise> { if (seen.has(span)) { return span } diff --git a/apps/sim/lib/logs/execution/trace-spans/summarize.ts b/apps/sim/lib/logs/execution/trace-spans/summarize.ts index 43d6c9982ad..8b47086f013 100644 --- a/apps/sim/lib/logs/execution/trace-spans/summarize.ts +++ b/apps/sim/lib/logs/execution/trace-spans/summarize.ts @@ -1,3 +1,4 @@ +import { isRecordLike } from '@sim/utils/object' import type { TraceSpan } from '@/lib/logs/types' export function stripModelToolCallArguments( @@ -35,7 +36,8 @@ export function stripProviderTimingContent( /** * A trace span tree with every span's content removed: inputs, outputs, * thinking, error text, tool-call arguments, and provider content. Keeps the - * tree's shape, names, timing, status, and cost. + * tree's shape, names, timing, status, and cost, including a nested child + * workflow's spans carried on `output.childTraceSpans`. */ export function summarizeTraceSpansWithoutIo(traceSpans?: TraceSpan[]): TraceSpan[] | undefined { if (!traceSpans) { @@ -45,7 +47,7 @@ export function summarizeTraceSpansWithoutIo(traceSpans?: TraceSpan[]): TraceSpa return traceSpans.map((span) => { const { input: _input, - output: _output, + output, children, thinking: _thinking, errorMessage: _errorMessage, @@ -54,8 +56,12 @@ export function summarizeTraceSpansWithoutIo(traceSpans?: TraceSpan[]): TraceSpa providerTiming, ...rest } = span + const nestedSpans = isRecordLike(output) ? output.childTraceSpans : undefined return { ...rest, + ...(Array.isArray(nestedSpans) && nestedSpans.length + ? { output: { childTraceSpans: summarizeTraceSpansWithoutIo(nestedSpans) } } + : {}), ...(modelToolCalls ? { modelToolCalls: stripModelToolCallArguments(modelToolCalls) } : {}), ...(toolCalls ? { toolCalls: stripLegacyToolCallContent(toolCalls) } : {}), ...(providerTiming ? { providerTiming: stripProviderTimingContent(providerTiming) } : {}), From c92129269cb0253bcf1ae58b6218faf38ed8fb41 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 28 Sep 2026 22:49:09 -0700 Subject: [PATCH 7/7] fix(logs): skip span fields not in their expected shape when summarizing Per-field compaction can spill an oversized modelToolCalls, toolCalls, or providerTiming to a large-value reference; the skeleton now drops such a field instead of reading it. --- .../lib/execution/payloads/serializer.test.ts | 20 +++++++++++++++++++ .../logs/execution/trace-spans/summarize.ts | 20 +++++++++++++------ 2 files changed, 34 insertions(+), 6 deletions(-) diff --git a/apps/sim/lib/execution/payloads/serializer.test.ts b/apps/sim/lib/execution/payloads/serializer.test.ts index c66f4666bec..c22a4bda001 100644 --- a/apps/sim/lib/execution/payloads/serializer.test.ts +++ b/apps/sim/lib/execution/payloads/serializer.test.ts @@ -418,6 +418,26 @@ describe('compacting span trees', () => { expect(compacted.childTraceSpans).toHaveLength(1) }) + it('keeps the skeleton when span metadata itself was spilled', async () => { + const spans = spansTooLargeAsAWhole() + const blockSpan = spans[0].children?.[0].children?.[0] + Object.assign(blockSpan ?? {}, { + modelToolCalls: Array.from({ length: 8 }, (_, index) => ({ + name: `tool-${index}`, + arguments: { query: 'q'.repeat(1024) }, + })), + toolCalls: Array.from({ length: 8 }, (_, index) => ({ + name: `tool-${index}`, + input: 'i'.repeat(1024), + })), + providerTiming: { segments: [{ assistantContent: 'a'.repeat(8192) }] }, + }) + + const compacted = await compactBlockOutput({ childTraceSpans: spans }, options) + + expectSpanTree(compacted.childTraceSpans) + }) + it('keeps nested child workflow trees in the skeleton', async () => { const nestedWorkflowSpan: TraceSpan = { id: 'nested-workflow', diff --git a/apps/sim/lib/logs/execution/trace-spans/summarize.ts b/apps/sim/lib/logs/execution/trace-spans/summarize.ts index 8b47086f013..77ddc4e9e65 100644 --- a/apps/sim/lib/logs/execution/trace-spans/summarize.ts +++ b/apps/sim/lib/logs/execution/trace-spans/summarize.ts @@ -27,7 +27,7 @@ export function stripProviderTimingContent( ...segment }) => ({ ...segment, - ...(toolCalls ? { toolCalls: stripModelToolCallArguments(toolCalls) } : {}), + ...(Array.isArray(toolCalls) ? { toolCalls: stripModelToolCallArguments(toolCalls) } : {}), }) ), } @@ -37,7 +37,9 @@ export function stripProviderTimingContent( * A trace span tree with every span's content removed: inputs, outputs, * thinking, error text, tool-call arguments, and provider content. Keeps the * tree's shape, names, timing, status, and cost, including a nested child - * workflow's spans carried on `output.childTraceSpans`. + * workflow's spans carried on `output.childTraceSpans`. A field that is not in + * its expected shape (for example one spilled to a large-value reference) is + * dropped rather than read. */ export function summarizeTraceSpansWithoutIo(traceSpans?: TraceSpan[]): TraceSpan[] | undefined { if (!traceSpans) { @@ -62,10 +64,16 @@ export function summarizeTraceSpansWithoutIo(traceSpans?: TraceSpan[]): TraceSpa ...(Array.isArray(nestedSpans) && nestedSpans.length ? { output: { childTraceSpans: summarizeTraceSpansWithoutIo(nestedSpans) } } : {}), - ...(modelToolCalls ? { modelToolCalls: stripModelToolCallArguments(modelToolCalls) } : {}), - ...(toolCalls ? { toolCalls: stripLegacyToolCallContent(toolCalls) } : {}), - ...(providerTiming ? { providerTiming: stripProviderTimingContent(providerTiming) } : {}), - ...(children?.length ? { children: summarizeTraceSpansWithoutIo(children) } : {}), + ...(Array.isArray(modelToolCalls) + ? { modelToolCalls: stripModelToolCallArguments(modelToolCalls) } + : {}), + ...(Array.isArray(toolCalls) ? { toolCalls: stripLegacyToolCallContent(toolCalls) } : {}), + ...(Array.isArray(providerTiming?.segments) + ? { providerTiming: stripProviderTimingContent(providerTiming) } + : {}), + ...(Array.isArray(children) && children.length + ? { children: summarizeTraceSpansWithoutIo(children) } + : {}), } }) }