improvement(workflows): share one reference scanner and make tokenizing linear

Move the reference-aware list splitter into @sim/utils next to the tokenizer so both
consume a single <workflow.reference> candidate scan instead of a copied loop, and keep
ENV_REFERENCE_PATTERN private again.

findWorkflowReferenceTokens checked each workflow span against every prior token, which
is quadratic on reference-dense values (8s at 160k tokens). Environment tokens are
disjoint and ordered and spans arrive in order, so a forward cursor gives identical
output in linear time (21ms); a 300k-input differential fuzz against the previous
implementation found no mismatches.

With classification linear, the 10k-character selector entry cap is no longer needed,
so oversized literal ids are validated again instead of silently skipped.

Author: waleedlatif1

apps/sim/lib/workflows/editing/validation.test.ts

@@ -1267,8 +1267,7 @@ describe('collectUnresolvedReferences', () => {
   })
 
   it('validates the literal entries when a multi-select opens AND closes with a template', async () => {
-    // `isReference` is unanchored (startsWith '<' && endsWith '>'), so this value reads as one
-    // whole reference. Filtering per entry is what keeps `kb_real` validated.
+    // The whole string contains references, so only filtering per entry keeps `kb_real` checked.
     mockValidateSelectorIds.mockResolvedValue({ valid: [], invalid: ['kb_real'] })
     const state = {
       blocks: {
@@ -1304,63 +1303,36 @@ describe('collectUnresolvedReferences', () => {
     expect(refs).toHaveLength(0)
   })
 
-  it('skips a single oversized entry, which cannot be classified cheaply', async () => {
-    mockValidateSelectorIds.mockResolvedValue({ valid: [], invalid: ['x'] })
-    const state = {
-      blocks: {
-        kb1: {
-          type: 'knowledge',
-          name: 'KB',
-          subBlocks: { knowledgeBaseId: { value: `kb_${'a'.repeat(10_000)}` } },
-        },
-      },
-    }
-    const startedAt = performance.now()
-    const refs = await collectUnresolvedReferences(state, CTX)
-    expect(performance.now() - startedAt).toBeLessThan(1000)
-    expect(mockValidateSelectorIds).not.toHaveBeenCalled()
-    expect(refs).toHaveLength(0)
-  })
-
-  it('keeps a comma-bearing reference intact even in an oversized list', async () => {
-    // Splitting scans reference regions directly and stays linear, so size does not force a
-    // fallback that would tear `<start.pick(a,b)>` into fragments validated as ids.
-    mockValidateSelectorIds.mockResolvedValue({ valid: [], invalid: ['kb_missing'] })
-    const padding = Array.from({ length: 400 }, (_, index) => `kb_${'x'.repeat(30)}${index}`)
-    const value = [...padding, '<start.pick(a,b)>', 'kb_missing'].join(',')
-    expect(value.length).toBeGreaterThan(10_000)
+  it('still validates a literal id however long it is', async () => {
+    const value = `kb_${'a'.repeat(10_000)}`
+    mockValidateSelectorIds.mockResolvedValue({ valid: [], invalid: [value] })
     const state = {
       blocks: {
         kb1: { type: 'knowledge', name: 'KB', subBlocks: { knowledgeBaseId: { value } } },
       },
     }
-    await collectUnresolvedReferences(state, CTX)
-
-    const [, ids] = mockValidateSelectorIds.mock.calls[0]
-    expect(ids).toContain('kb_missing')
-    expect(ids).not.toContain('<start.pick(a')
-    expect(ids).not.toContain('b)>')
+    const refs = await collectUnresolvedReferences(state, CTX)
+    expect(mockValidateSelectorIds).toHaveBeenCalledWith('knowledge-base-selector', value, CTX)
+    expect(refs).toHaveLength(1)
   })
 
-  it('still validates the literal entries of an oversized list', async () => {
-    // The cap gives up reference-aware splitting, not validation: the entries are still short,
-    // so each is classified and the literals are still checked.
+  it('checks the literal ids around a reference that contains a comma', async () => {
     mockValidateSelectorIds.mockResolvedValue({ valid: [], invalid: ['kb_missing'] })
-    const padding = Array.from({ length: 400 }, (_, index) => `kb_${'x'.repeat(30)}${index}`)
-    const value = [...padding, '<start.kbId>', 'kb_missing'].join(',')
-    expect(value.length).toBeGreaterThan(10_000)
     const state = {
       blocks: {
-        kb1: { type: 'knowledge', name: 'KB', subBlocks: { knowledgeBaseId: { value } } },
+        kb1: {
+          type: 'knowledge',
+          name: 'KB',
+          subBlocks: { knowledgeBaseId: { value: 'kb_a,<start.pick(a,b)>,kb_missing' } },
+        },
       },
     }
-    const startedAt = performance.now()
     const refs = await collectUnresolvedReferences(state, CTX)
-
-    expect(performance.now() - startedAt).toBeLessThan(1000)
-    const [, ids] = mockValidateSelectorIds.mock.calls[0]
-    expect(ids).toContain('kb_missing')
-    expect(ids).not.toContain('<start.kbId>')
+    expect(mockValidateSelectorIds).toHaveBeenCalledWith(
+      'knowledge-base-selector',
+      ['kb_a', 'kb_missing'],
+      CTX
+    )
     expect(refs).toHaveLength(1)
   })
 
@@ -1384,10 +1356,7 @@ describe('collectUnresolvedReferences', () => {
     expect(refs).toHaveLength(1)
   })
 
-  // `splitOutsideReferences` is what keeps a comma INSIDE a reference from becoming a separator.
-  // Every other reference test above would still pass with a naive `.split(',')` (no comma ->
-  // never split at all), so these two are the only ones that pin the reference-aware split from
-  // the consumer's side: a torn reference reads as plain literals and gets validated as ids.
+  // A torn reference reads as plain literals, so these pin the reference-aware split end to end.
   it('does not split a <block.output> reference that contains a comma', async () => {
     const state = {
       blocks: {
@@ -1433,8 +1402,7 @@ describe('collectUnresolvedReferences', () => {
     expect(refs).toHaveLength(0)
   })
 
-  // A multi-select that already stores a native array never reaches the comma split, so the
-  // array filter is entered by a second, independent route.
+  // A native array value never reaches the comma split, so it enters the filter independently.
   it('filters templates out of a value that is already an array', async () => {
     mockValidateSelectorIds.mockResolvedValue({ valid: [], invalid: ['kb_missing'] })
     const state = {
@@ -1470,9 +1438,7 @@ describe('collectUnresolvedReferences', () => {
     expect(refs).toHaveLength(0)
   })
 
-  // A separator-only string is truthy, so it survives the `!subBlockValue` bail and reaches the
-  // split, which returns nothing. Reusing the all-references bail is what stops an empty list
-  // from being sent to the database as if it were a set of ids.
+  // A separator-only string is truthy, so it passes the `!subBlockValue` bail and splits to nothing.
   it('skips a value that is nothing but separators', async () => {
     const state = {
       blocks: {
@@ -1495,10 +1461,7 @@ describe('collectUnresolvedReferences', () => {
     expect(refs).toHaveLength(0)
   })
 
-  // The per-entry filter calls `containsReference` on whatever the array holds, so its
-  // non-string bail is load-bearing here - without it a numeric entry throws. A non-string can
-  // never be a reference, so it must survive untouched, `null` included: the `filter(Boolean)`
-  // that would have dropped it lives inside the comma split, which a native array never reaches.
+  // A non-string can never be a reference, so it passes through the filter untouched.
   it('does not throw on a non-string entry inside an array value', async () => {
     mockValidateSelectorIds.mockResolvedValue({ valid: [], invalid: [] })
     const state = {
@@ -1519,8 +1482,7 @@ describe('collectUnresolvedReferences', () => {
     expect(refs).toHaveLength(0)
   })
 
-  // Both delimiters are required. A lone `<` (or a lone `{{`) is a malformed literal, not a
-  // template, and must keep being reported rather than silently waved through.
+  // A lone `<` or `{{` is a malformed literal, not a reference, and must still be reported.
   it.each([
     ['an unclosed < delimiter', 'kb_<start'],
     ['an unopened > delimiter', 'start.kbId>'],
@@ -1557,8 +1519,7 @@ describe('collectUnresolvedReferences', () => {
     expect(refs).toHaveLength(1)
   })
 
-  // The guard runs AFTER the canonical active-member check, so a template in the active member
-  // must be skipped by the guard - and must not push mode resolution onto the empty twin.
+  // The guard runs after the canonical active-member check, so it must not flip the active member.
   it('skips a template held by the ACTIVE canonical member', async () => {
     mockValidateSelectorIds.mockResolvedValue({ valid: [], invalid: ['<start.cred>'] })
     const state = {
@@ -1576,9 +1537,7 @@ describe('collectUnresolvedReferences', () => {
   })
 })
 
-// The lint path (collectUnresolvedReferences) and the agent edit path share collectSelectorFields,
-// but only the edit path can REJECT an operation. A dynamically-bound selector must not block an
-// edit - that rejection is the user-visible failure this guard exists to prevent.
+// validateWorkflowSelectorIds shares collectSelectorFields with the lint, so it skips the same values.
 describe('validateWorkflowSelectorIds (reference guard)', () => {
   beforeEach(() => {
     vi.clearAllMocks()
@@ -1589,7 +1548,7 @@ describe('validateWorkflowSelectorIds (reference guard)', () => {
     ['a block-output reference', '<start.kbId>'],
     ['an env-var reference', '{{KB_ID}}'],
     ['a partially templated value', 'kb_<start.suffix>'],
-  ])('does not reject an edit whose selector holds %s', async (_label, value) => {
+  ])('reports no error for a selector holding %s', async (_label, value) => {
     mockValidateSelectorIds.mockResolvedValue({ valid: [], invalid: [value] })
     const state = {
       blocks: {
@@ -1601,7 +1560,7 @@ describe('validateWorkflowSelectorIds (reference guard)', () => {
     expect(errors).toHaveLength(0)
   })
 
-  it('still rejects an edit whose selector holds a literal id that does not resolve', async () => {
+  it('still reports a literal id that does not resolve', async () => {
     mockValidateSelectorIds.mockResolvedValue({ valid: [], invalid: ['kb_gone'] })
     const state = {
       blocks: {

apps/sim/lib/workflows/editing/validation.ts

@@ -1082,17 +1082,6 @@ interface SelectorFieldToValidate {
   value: string | string[]
 }
 
-/**
- * Longest single selector entry `containsReference` will classify.
- *
- * Classifying one entry tokenizes it, and `findWorkflowReferenceTokens` is superlinear in
- * candidate count, so an entry of unbounded length is expensive on a write path that admits
- * megabytes. Splitting is unaffected - it scans reference regions directly and stays linear - so
- * only an individual oversized ENTRY is skipped, where there is no cheap way to tell a literal
- * from a dynamic binding. A long LIST of ordinary ids still splits and validates normally.
- */
-const MAX_SELECTOR_ENTRY_LENGTH = 10_000
-
 /**
  * Walk a workflow state and collect selector/credential fields to validate.
  * For canonical pairs only the ACTIVE member is collected (an intentionally-empty
@@ -1147,26 +1136,18 @@ function collectSelectorFields(
       const subBlockValue = blockData.subBlocks?.[subBlockConfig.id]?.value
       if (!subBlockValue) continue
 
-      const isOversized = (entry: unknown) =>
-        typeof entry === 'string' && entry.length > MAX_SELECTOR_ENTRY_LENGTH
-
       // Handle comma-separated values for multi-select
       let values: string | string[] = subBlockValue
       if (typeof subBlockValue === 'string' && subBlockValue.includes(',')) {
         values = splitOutsideReferences(subBlockValue)
       }
 
-      // A dynamically bound value only acquires its id at execution time, so a static
-      // id-existence check cannot evaluate it. Filtered per entry rather than on the whole
-      // string, because a multi-select can mix literal ids with dynamic ones: testing
-      // `<a.b>,kb_real,<c.d>` as a whole would drop `kb_real` along with the references.
+      // A reference or env var only resolves to an id at execution time, so it cannot be checked
+      // here. Filtered per entry so the literal ids of a mixed multi-select are still checked.
       if (Array.isArray(values)) {
-        const literalValues = values.filter(
-          (entry) => !isOversized(entry) && !containsReference(entry)
-        )
-        if (literalValues.length === 0) continue
-        values = literalValues
-      } else if (isOversized(values) || containsReference(values)) {
+        values = values.filter((entry) => !containsReference(entry))
+        if (values.length === 0) continue
+      } else if (containsReference(values)) {
         continue
       }
 

apps/sim/lib/workflows/sanitization/references.test.ts

@@ -2,7 +2,6 @@ import { describe, expect, it } from 'vitest'
 import {
   containsReference,
   isLikelyReferenceSegment,
-  splitOutsideReferences,
   splitReferenceSegment,
 } from '@/lib/workflows/sanitization/references'
 
@@ -91,63 +90,3 @@ describe('containsReference', () => {
     expect(containsReference('a<b<c>d')).toBe(false)
   })
 })
-
-describe('splitOutsideReferences', () => {
-  it('splits on separator commas', () => {
-    expect(splitOutsideReferences('kb_a,kb_b')).toEqual(['kb_a', 'kb_b'])
-  })
-
-  it('trims entries and drops empties', () => {
-    expect(splitOutsideReferences(' kb_a , , kb_b ')).toEqual(['kb_a', 'kb_b'])
-  })
-
-  it('keeps a comma that sits inside a workflow reference', () => {
-    expect(splitOutsideReferences('<start.pick(a,b)>')).toEqual(['<start.pick(a,b)>'])
-  })
-
-  it('keeps a comma inside a reference while still splitting around it', () => {
-    expect(splitOutsideReferences('kb_a,<start.pick(x,y)>,kb_b')).toEqual([
-      'kb_a',
-      '<start.pick(x,y)>',
-      'kb_b',
-    ])
-  })
-
-  it('keeps a comma inside an env-var placeholder', () => {
-    expect(splitOutsideReferences('{{A,B}},kb_a')).toEqual(['{{A,B}}', 'kb_a'])
-  })
-
-  it('returns a single entry when there is no separator', () => {
-    expect(splitOutsideReferences('kb_a')).toEqual(['kb_a'])
-  })
-
-  it('stays linear on a large value instead of rescanning tokens per comma', () => {
-    // A per-comma `tokens.some()` is O(commas x tokens) and took ~2.5s on this input.
-    const value = '{{A}},'.repeat(40000)
-    const startedAt = performance.now()
-    const parts = splitOutsideReferences(value)
-    const elapsedMs = performance.now() - startedAt
-
-    expect(parts).toHaveLength(40000)
-    expect(elapsedMs).toBeLessThan(1000)
-  })
-
-  it('protects a comma inside a reference that nests an env-var placeholder', () => {
-    // `findWorkflowReferenceTokens` is non-overlapping, so it reports only the inner `{{A}}` and
-    // drops the outer candidate. The candidate pass is what keeps the outer region protected.
-    expect(splitOutsideReferences('<start.body.pick({{A}},b)>,kb_literal')).toEqual([
-      '<start.body.pick({{A}},b)>',
-      'kb_literal',
-    ])
-    expect(splitOutsideReferences('<a.{{B}}x,y>,kb_literal')).toEqual([
-      '<a.{{B}}x,y>',
-      'kb_literal',
-    ])
-  })
-
-  it('still splits a near-miss that does not read as a reference', () => {
-    // `<a.b+c,d>` fails `isLikelyReferenceSegment` (the `+`), so it is not a protected region.
-    // Loud rather than silent: the fragments are reported as ids that do not resolve.
-    expect(splitOutsideReferences('<a.b+c,d>')).toEqual(['<a.b+c', 'd>'])
-  })
-})