fix(jev): harden evaluation transport and output discovery

Author: waleedlatif1

apps/sim/blocks/agent-evaluation.test.ts

@@ -1,4 +1,9 @@
 import { describe, expect, it, vi } from 'vitest'
+import {
+  getEffectiveBlockOutputPaths,
+  getEffectiveBlockOutputs,
+  getEffectiveBlockOutputType,
+} from '@/lib/workflows/blocks/block-outputs'
 import { evaluateSubBlockCondition } from '@/lib/workflows/subblocks/visibility'
 import { AgentBlock } from '@/blocks/blocks/agent'
 import { getAgentModelOptions, getModelOptions } from '@/blocks/utils'
@@ -29,6 +34,37 @@ describe('Agent evaluation configuration', () => {
     }
   })
 
+  it.each(['jev-1.13.0', '<start.model>', '{{MODEL_ID}}'])(
+    'exposes answers for %s in downstream selectors',
+    (model) => {
+      const values = { model: { value: model } }
+      expect(getEffectiveBlockOutputs('agent', values)).toHaveProperty('answers')
+      expect(getEffectiveBlockOutputPaths('agent', values)).toContain('answers')
+      expect(getEffectiveBlockOutputType('agent', 'answers', values)).toBe('json')
+    }
+  )
+
+  it('does not expose evaluation answers for a known chat model', () => {
+    expect(getEffectiveBlockOutputs('agent', { model: { value: 'gpt-4o' } })).not.toHaveProperty(
+      'answers'
+    )
+  })
+
+  it.each(['jev-1.13.0', '<start.model>'])(
+    'keeps answers accessible with a saved chat schema for %s',
+    (model) => {
+      const outputs = getEffectiveBlockOutputs('agent', {
+        model: { value: model },
+        responseFormat: {
+          value: { schema: { type: 'object', properties: { title: { type: 'string' } } } },
+        },
+      })
+      expect(outputs).toHaveProperty('answers')
+      if (model === 'jev-1.13.0') expect(outputs).not.toHaveProperty('title')
+      else expect(outputs).toHaveProperty('title')
+    }
+  )
+
   it('shows Jev only in the model picker that supports evaluation inputs', () => {
     useProvidersStore.getState().setProviderModels('base', Object.keys(getBaseModelProviders()))
     expect(getAgentModelOptions().map((option) => option.id)).toContain('jev-1.13.0')

apps/sim/blocks/blocks/agent.ts

@@ -650,7 +650,7 @@ Return ONLY the JSON array.`,
       type: 'json',
       description:
         'Evaluation answers keyed by question ID: choice, score, or noul, with probabilities and confidence where applicable',
-      condition: { field: 'model', value: EVALUATION_MODELS },
+      condition: { field: 'model', value: EVALUATION_MODELS, allowReference: true },
     },
     content: { type: 'string', description: 'Generated response content' },
     model: { type: 'string', description: 'Model used for generation' },

apps/sim/ee/organization-usage/components/usage-consumers.tsx

@@ -23,6 +23,7 @@ import {
   OpenRouterIcon,
   SakanaIcon,
   TogetherIcon,
+  TypeSafeIcon,
   VertexIcon,
   VllmIcon,
   xAIIcon,
@@ -74,6 +75,7 @@ const PROVIDER_ICONS: Readonly<Record<string, ComponentType<{ className?: string
   openrouter: OpenRouterIcon,
   sakana: SakanaIcon,
   together: TogetherIcon,
+  typesafe: TypeSafeIcon,
   vertex: VertexIcon,
   vllm: VllmIcon,
   xai: xAIIcon,

apps/sim/lib/workflows/blocks/block-outputs.ts

@@ -4,6 +4,7 @@ import {
   parseResponseFormatSafely,
 } from '@/lib/core/utils/response-format'
 import { normalizeInputFormatValue } from '@/lib/workflows/input-format'
+import { containsReference } from '@/lib/workflows/sanitization/references'
 import {
   classifyStartBlockType,
   StartBlockPath,
@@ -23,6 +24,7 @@ import {
   type OutputFieldDefinition,
 } from '@/blocks/types'
 import { isHumanInTheLoopBlock } from '@/executor/constants'
+import { isEvaluationModel } from '@/providers/models'
 import { getToolOutputsMetadata } from '@/tools/metadata-outputs'
 import { getTrigger, isTriggerValid } from '@/triggers'
 
@@ -61,15 +63,19 @@ function evaluateOutputCondition(
 
   const fieldValue = subBlocks[condition.field]?.value
 
+  const deferred =
+    condition.allowReference && typeof fieldValue === 'string' && containsReference(fieldValue)
   let matches: boolean
-  if (Array.isArray(condition.value)) {
+  if (deferred) {
+    matches = true
+  } else if (Array.isArray(condition.value)) {
     // For array conditions, check if fieldValue is a valid primitive and included
     matches = isConditionPrimitive(fieldValue) && condition.value.includes(fieldValue)
   } else {
     matches = fieldValue === condition.value
   }
 
-  if (condition.not) {
+  if (condition.not && !deferred) {
     matches = !matches
   }
 
@@ -428,8 +434,19 @@ export function getEffectiveBlockOutputs(
   const includeHidden = options?.includeHidden ?? false
 
   if (blockType === 'agent') {
-    const responseFormatOutputs = getResponseFormatOutputs(subBlocks, 'agent')
-    if (responseFormatOutputs) return responseFormatOutputs
+    const model = subBlocks?.model?.value
+    if (typeof model !== 'string' || !isEvaluationModel(model)) {
+      const responseFormatOutputs = getResponseFormatOutputs(subBlocks, 'agent')
+      if (responseFormatOutputs) {
+        /** A referenced model may select either evaluation or chat at execution time. */
+        return typeof model === 'string' && containsReference(model)
+          ? {
+              ...getBlockOutputs('agent', subBlocks, false, { includeHidden }),
+              ...responseFormatOutputs,
+            }
+          : responseFormatOutputs
+      }
+    }
   }
 
   let baseOutputs: OutputDefinition

apps/sim/providers/typesafe/index.ts

@@ -1,15 +1,12 @@
-import { consumeOrCancelBody, readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits'
 import type { StreamingExecution } from '@/executor/types'
 import { getProviderDefaultModel, getProviderModels } from '@/providers/models'
 import { createSettledAgentEventStream } from '@/providers/stream-events'
 import { createStreamingExecution } from '@/providers/streaming-execution'
-import { PROVIDER_HEADERS_TIMEOUT_MS } from '@/providers/transport'
 import type { ProviderConfig, ProviderRequest, ProviderResponse } from '@/providers/types'
 import { buildJevBody, parseJevResponse } from '@/providers/typesafe/schema'
+import { requestJevEvaluation } from '@/providers/typesafe/transport'
 import { calculateCost } from '@/providers/utils'
 
-const MAX_EVALUATION_RESPONSE_BYTES = 10 * 1024 * 1024
-
 export const typesafeProvider: ProviderConfig = {
   id: 'typesafe',
   name: 'TypeSafe',
@@ -40,26 +37,8 @@ export const typesafeProvider: ProviderConfig = {
     )
     const start = Date.now()
     const startTime = new Date(start).toISOString()
-    const timeout = AbortSignal.timeout(PROVIDER_HEADERS_TIMEOUT_MS)
-    const signal = request.abortSignal ? AbortSignal.any([request.abortSignal, timeout]) : timeout
-    signal.throwIfAborted()
-    const response = await fetch('https://api.typesafe.ai/v1/systemone', {
-      method: 'POST',
-      headers: { Authorization: `Bearer ${request.apiKey}`, 'Content-Type': 'application/json' },
-      body: JSON.stringify(body),
-      signal,
-      redirect: 'error',
-    })
-    if (!response.ok) {
-      await consumeOrCancelBody(response)
-      throw new Error(`TypeSafe evaluation failed (HTTP ${response.status})`)
-    }
     const result = parseJevResponse(
-      await readResponseJsonWithLimit(response, {
-        maxBytes: MAX_EVALUATION_RESPONSE_BYTES,
-        label: 'TypeSafe evaluation response',
-        signal,
-      }),
+      await requestJevEvaluation(body, request.apiKey, request.abortSignal),
       body.questions
     )
     const content = JSON.stringify(result.answers)

apps/sim/providers/typesafe/schema.ts

@@ -113,5 +113,15 @@ export function parseJevResponse(
       'TypeSafe returned Jev answers that do not match the requested question IDs and types'
     )
   }
+  for (const [id, question] of Object.entries(questions)) {
+    const answer = answers[id]
+    if (
+      question.type === 'choice' &&
+      answer.type === 'choice' &&
+      !Object.hasOwn(question.criteria, answer.choice)
+    ) {
+      throw new Error('TypeSafe returned a Jev Choice answer outside the requested options')
+    }
+  }
   return result.data
 }

apps/sim/providers/typesafe/transport.ts

@@ -0,0 +1,62 @@
+import { interruptibleSleep } from '@sim/utils/helpers'
+import { backoffWithJitter, parseRetryAfter } from '@sim/utils/retry'
+import { consumeOrCancelBody, readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits'
+import { PROVIDER_HEADERS_TIMEOUT_MS, PROVIDER_MAX_RETRIES } from '@/providers/transport'
+import type { buildJevBody } from '@/providers/typesafe/schema'
+
+const MAX_EVALUATION_RESPONSE_BYTES = 10 * 1024 * 1024
+
+class TypeSafeHttpError extends Error {
+  constructor(
+    readonly status: number,
+    readonly retryAfterMs: number | null
+  ) {
+    super(`TypeSafe evaluation failed (HTTP ${status})`)
+    this.name = 'TypeSafeHttpError'
+  }
+}
+
+export async function requestJevEvaluation(
+  body: ReturnType<typeof buildJevBody>,
+  apiKey: string,
+  abortSignal?: AbortSignal
+): Promise<unknown> {
+  const payload = JSON.stringify(body)
+  for (let attempt = 0; ; attempt++) {
+    abortSignal?.throwIfAborted()
+    const timeout = AbortSignal.timeout(PROVIDER_HEADERS_TIMEOUT_MS)
+    const signal = abortSignal ? AbortSignal.any([abortSignal, timeout]) : timeout
+    let response: Response | undefined
+    try {
+      signal.throwIfAborted()
+      response = await fetch('https://api.typesafe.ai/v1/systemone', {
+        method: 'POST',
+        headers: { Authorization: `Bearer ${apiKey}`, 'Content-Type': 'application/json' },
+        body: payload,
+        signal,
+        redirect: 'error',
+      })
+      if (!response.ok) {
+        await consumeOrCancelBody(response)
+        throw new TypeSafeHttpError(
+          response.status,
+          parseRetryAfter(response.headers.get('retry-after'))
+        )
+      }
+      return await readResponseJsonWithLimit(response, {
+        maxBytes: MAX_EVALUATION_RESPONSE_BYTES,
+        label: 'TypeSafe evaluation response',
+        signal,
+      })
+    } catch (error) {
+      abortSignal?.throwIfAborted()
+      const retryable =
+        error instanceof TypeSafeHttpError
+          ? error.status === 408 || error.status === 429 || error.status >= 500
+          : !response || timeout.aborted || error instanceof TypeError
+      if (!retryable || attempt >= PROVIDER_MAX_RETRIES) throw error
+      const retryAfterMs = error instanceof TypeSafeHttpError ? error.retryAfterMs : null
+      await interruptibleSleep(backoffWithJitter(attempt + 1, retryAfterMs), abortSignal)
+    }
+  }
+}

apps/sim/providers/typesafe/typesafe.test.ts

@@ -1,6 +1,7 @@
 /** @vitest-environment node */
 import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
 import { getHostedModels, getModelCapabilities, getProviderIcon } from '@/providers/models'
+import { PROVIDER_MAX_RETRIES } from '@/providers/transport'
 import type { ProviderRequest } from '@/providers/types'
 import { typesafeProvider } from '@/providers/typesafe'
 import { buildJevBody, parseJevResponse } from '@/providers/typesafe/schema'
@@ -52,7 +53,11 @@ describe('TypeSafe provider', () => {
     fetchMock.mockReset().mockResolvedValue(Response.json(RESULT))
     vi.stubGlobal('fetch', fetchMock)
   })
-  afterEach(() => vi.unstubAllGlobals())
+  afterEach(() => {
+    vi.unstubAllGlobals()
+    vi.restoreAllMocks()
+    vi.useRealTimers()
+  })
 
   it.each(['jev-1.13.0', 'jev-latest', 'jev-preview'])(
     'routes %s through native BYOK evaluation',
@@ -131,6 +136,79 @@ describe('TypeSafe provider', () => {
     await expect(typesafeProvider.executeRequest(REQUEST)).rejects.toThrow(
       'TypeSafe evaluation failed (HTTP 401)'
     )
+    expect(fetchMock).toHaveBeenCalledTimes(1)
+  })
+
+  it.each([408, 429, 500, 503])('retries HTTP %s and honors Retry-After', async (status) => {
+    vi.useFakeTimers()
+    fetchMock.mockResolvedValueOnce(new Response(null, { status, headers: { 'retry-after': '2' } }))
+    const result = typesafeProvider.executeRequest(REQUEST)
+    await vi.advanceTimersByTimeAsync(1999)
+    expect(fetchMock).toHaveBeenCalledTimes(1)
+    await vi.advanceTimersByTimeAsync(1)
+    expect(await result).toMatchObject({ answers: RESULT.answers })
+    expect(fetchMock).toHaveBeenCalledTimes(2)
+  })
+
+  it.each([400, 403, 422])('does not retry HTTP %s', async (status) => {
+    fetchMock.mockResolvedValueOnce(new Response(null, { status }))
+    await expect(typesafeProvider.executeRequest(REQUEST)).rejects.toThrow(`HTTP ${status}`)
+    expect(fetchMock).toHaveBeenCalledTimes(1)
+  })
+
+  it('retries connection failures within the shared provider retry budget', async () => {
+    vi.useFakeTimers()
+    fetchMock.mockRejectedValue(new TypeError('fetch failed'))
+    const result = expect(typesafeProvider.executeRequest(REQUEST)).rejects.toThrow('fetch failed')
+    await vi.runAllTimersAsync()
+    await result
+    expect(fetchMock).toHaveBeenCalledTimes(PROVIDER_MAX_RETRIES + 1)
+  })
+
+  it('stops retrying repeated server failures', async () => {
+    vi.useFakeTimers()
+    fetchMock.mockImplementation(async () => new Response(null, { status: 503 }))
+    const result = expect(typesafeProvider.executeRequest(REQUEST)).rejects.toThrow('HTTP 503')
+    await vi.runAllTimersAsync()
+    await result
+    expect(fetchMock).toHaveBeenCalledTimes(PROVIDER_MAX_RETRIES + 1)
+  })
+
+  it('gives a timed-out attempt a fresh deadline', async () => {
+    vi.useFakeTimers()
+    const deadline = new AbortController()
+    vi.spyOn(AbortSignal, 'timeout').mockReturnValueOnce(deadline.signal)
+    fetchMock.mockImplementationOnce(async () => {
+      deadline.abort(new DOMException('Timed out', 'TimeoutError'))
+      throw deadline.signal.reason
+    })
+    const result = typesafeProvider.executeRequest(REQUEST)
+    await vi.runAllTimersAsync()
+    expect(await result).toMatchObject({ answers: RESULT.answers })
+    expect(fetchMock).toHaveBeenCalledTimes(2)
+    expect(fetchMock.mock.calls[1][1]?.signal?.aborted).toBe(false)
+  })
+
+  it('cancels immediately during Retry-After without sending another attempt', async () => {
+    vi.useFakeTimers()
+    const controller = new AbortController()
+    fetchMock.mockResolvedValueOnce(
+      new Response(null, { status: 429, headers: { 'retry-after': '30' } })
+    )
+    const result = expect(
+      typesafeProvider.executeRequest({ ...REQUEST, abortSignal: controller.signal })
+    ).rejects.toThrow('Cancelled')
+    await vi.advanceTimersByTimeAsync(1)
+    controller.abort(new Error('Cancelled'))
+    await result
+    expect(fetchMock).toHaveBeenCalledTimes(1)
+    expect(vi.getTimerCount()).toBe(0)
+  })
+
+  it('does not retry a malformed successful response', async () => {
+    fetchMock.mockResolvedValueOnce(new Response('{'))
+    await expect(typesafeProvider.executeRequest(REQUEST)).rejects.toThrow()
+    expect(fetchMock).toHaveBeenCalledTimes(1)
   })
 
   it('honors cancellation before network access', async () => {
@@ -217,4 +295,14 @@ describe('Jev native schema', () => {
   ])('rejects invalid provider responses', (value) => {
     expect(() => parseJevResponse(value, QUESTIONS)).toThrow('invalid Jev evaluation response')
   })
+
+  it.each(['unknown', 'toString'])('rejects an unrequested Choice option %s', (choice) => {
+    const answers = {
+      ...RESULT.answers,
+      department: { type: 'choice', choice, probabilities: { [choice]: 1 }, confidence: 1 },
+    }
+    expect(() => parseJevResponse({ ...RESULT, answers }, QUESTIONS)).toThrow(
+      'outside the requested options'
+    )
+  })
 })

packages/workflow-types/src/blocks.ts

@@ -59,6 +59,8 @@ export type SubBlockType =
   | 'modal'
 
 export interface OutputCondition {
+  /** Keep the output selectable when the compared value is resolved at execution time. */
+  allowReference?: boolean
   field: string
   value: string | number | boolean | Array<string | number | boolean>
   not?: boolean