From 19291dc9f424f64176f5cc0bda392f416dd66772 Mon Sep 17 00:00:00 2001 From: alexanderkirtzel Date: Tue, 20 Oct 2026 23:34:17 +0000 Subject: [PATCH 01/41] failure contract --- .../src/examples/flow-complete.manifest.ts | 4 + .../src/__tests__/boundary-error.test.ts | 10 +- .../src/__tests__/buffers-policy.test.ts | 238 +++ .../src/__tests__/build-flags.test.ts | 45 +- .../destination-batch-correctness.test.ts | 36 +- .../__tests__/destination-on-async.test.ts | 57 +- .../destination-on-timeout-overlap.test.ts | 60 +- .../destination-outcome-state.test.ts | 484 +++++ .../__tests__/destination-state-order.test.ts | 12 +- .../src/__tests__/destination.test.ts | 70 +- .../collector/src/__tests__/fanout.test.ts | 138 ++ .../src/__tests__/init-contract.test.ts | 43 + .../src/__tests__/invalid-event.test.ts | 12 +- .../src/__tests__/mapping-failures.test.ts | 147 ++ .../src/__tests__/on-callback-error.test.ts | 6 +- .../src/__tests__/on-without-init.test.ts | 106 + .../collector/src/__tests__/outcome.test.ts | 63 + .../src/__tests__/prerun-hold.test.ts | 25 +- .../src/__tests__/queue-bounds.test.ts | 33 +- .../src/__tests__/registration-checks.test.ts | 156 ++ .../src/__tests__/report-error.test.ts | 43 +- .../src/__tests__/step-state.test.ts | 17 +- .../src/__tests__/store-read-failure.test.ts | 152 ++ .../src/__tests__/transformer-branch.test.ts | 8 +- .../__tests__/transformer-init-error.test.ts | 91 +- .../src/__tests__/transformer-mapping.test.ts | 6 +- packages/collector/src/buffers.ts | 21 +- packages/collector/src/collector-next.ts | 8 + packages/collector/src/collector.ts | 37 +- packages/collector/src/constants.ts | 1 + .../src/destination-outcome-state.ts | 88 + packages/collector/src/destination.ts | 1841 ++++++++++------- packages/collector/src/flow.ts | 2 +- packages/collector/src/handle.ts | 8 +- packages/collector/src/observerEmit.ts | 30 +- packages/collector/src/on.ts | 91 +- packages/collector/src/outcome.ts | 175 ++ packages/collector/src/pending.ts | 8 +- packages/collector/src/push.ts | 149 +- packages/collector/src/report-error.ts | 100 +- packages/collector/src/shutdown.ts | 106 +- packages/collector/src/source.ts | 61 +- packages/collector/src/state.ts | 5 +- packages/collector/src/store.ts | 8 +- packages/collector/src/timeout.ts | 13 +- packages/collector/src/transformer.ts | 174 +- packages/collector/src/types/collector.ts | 1 + packages/core/src/__tests__/causeLog.test.ts | 148 ++ .../mapping-error-visibility.test.ts | 104 +- .../core/src/__tests__/mapping-no-ops.test.ts | 54 + packages/core/src/__tests__/mapping.test.ts | 57 +- packages/core/src/__tests__/state.test.ts | 20 +- packages/core/src/__tests__/stepError.test.ts | 91 + packages/core/src/cache.ts | 31 +- packages/core/src/causeLog.ts | 230 ++ packages/core/src/index.ts | 15 + packages/core/src/logger.ts | 58 +- packages/core/src/mapping-no-ops.ts | 109 + packages/core/src/mapping.ts | 126 +- packages/core/src/mappingValue.ts | 99 +- packages/core/src/schemas/collector.ts | 6 + packages/core/src/state.ts | 69 +- packages/core/src/stepError.ts | 111 + packages/core/src/transformer-mapping.ts | 68 +- packages/core/src/types/collector.ts | 22 + packages/core/src/types/destination.ts | 39 +- packages/core/src/types/elb.ts | 7 + packages/core/src/types/journey.ts | 22 +- packages/core/src/types/logger.ts | 8 +- packages/core/src/types/telemetry.ts | 15 +- packages/core/src/types/transformer.ts | 19 +- .../destinations/snowplow/src/index.test.ts | 30 +- 72 files changed, 5070 insertions(+), 1477 deletions(-) create mode 100644 packages/collector/src/__tests__/buffers-policy.test.ts create mode 100644 packages/collector/src/__tests__/destination-outcome-state.test.ts create mode 100644 packages/collector/src/__tests__/fanout.test.ts create mode 100644 packages/collector/src/__tests__/init-contract.test.ts create mode 100644 packages/collector/src/__tests__/mapping-failures.test.ts create mode 100644 packages/collector/src/__tests__/on-without-init.test.ts create mode 100644 packages/collector/src/__tests__/outcome.test.ts create mode 100644 packages/collector/src/__tests__/registration-checks.test.ts create mode 100644 packages/collector/src/__tests__/store-read-failure.test.ts create mode 100644 packages/collector/src/destination-outcome-state.ts create mode 100644 packages/collector/src/outcome.ts create mode 100644 packages/core/src/__tests__/causeLog.test.ts create mode 100644 packages/core/src/__tests__/mapping-no-ops.test.ts create mode 100644 packages/core/src/__tests__/stepError.test.ts create mode 100644 packages/core/src/causeLog.ts create mode 100644 packages/core/src/mapping-no-ops.ts create mode 100644 packages/core/src/stepError.ts diff --git a/packages/cli/src/examples/flow-complete.manifest.ts b/packages/cli/src/examples/flow-complete.manifest.ts index 4bc14f6ff..e05f42d47 100644 --- a/packages/cli/src/examples/flow-complete.manifest.ts +++ b/packages/cli/src/examples/flow-complete.manifest.ts @@ -2057,6 +2057,10 @@ export const flowCompleteCoverage = { queueMax: { excluded: 'the default bound fits; dlqMax is the knob this file turns', }, + consentDenied: { + excluded: + 'walkeros bundle sets it for server flows; this web example keeps the default queue', + }, name: RUNTIME('name'), release: { excluded: 'set by walkeros bundle --release, never written by hand', diff --git a/packages/collector/src/__tests__/boundary-error.test.ts b/packages/collector/src/__tests__/boundary-error.test.ts index d9cc8483c..6e9e9cba9 100644 --- a/packages/collector/src/__tests__/boundary-error.test.ts +++ b/packages/collector/src/__tests__/boundary-error.test.ts @@ -103,7 +103,7 @@ describe('push boundary', () => { // the ingest payload are deliberately absent; see the context-shape tests // below for why. expect(errorMock).toHaveBeenCalledWith( - 'push failed', + 'push failed at source.test-source', expect.objectContaining({ event: 'page view', error: 'boom', @@ -136,7 +136,13 @@ describe('push boundary', () => { .error as jest.Mock; const context: unknown = errorMock.mock.calls[0][1]; - expect(context).toEqual({ error: 'boom', event: 'order complete' }); + expect(context).toEqual({ + step: 'collector', + code: 'transport', + retryable: true, + error: 'boom', + event: 'order complete', + }); // What actually reaches Loki and the on-disk jsonl is the serialized // form, so assert on that too: no PII, whatever the key names. expect(JSON.stringify(context)).not.toContain('buyer@example.com'); diff --git a/packages/collector/src/__tests__/buffers-policy.test.ts b/packages/collector/src/__tests__/buffers-policy.test.ts new file mode 100644 index 000000000..d27f0819e --- /dev/null +++ b/packages/collector/src/__tests__/buffers-policy.test.ts @@ -0,0 +1,238 @@ +import { Level } from '@walkeros/core'; +import type { Collector, Destination, Logger, Source } from '@walkeros/core'; +import { startFlow } from '..'; + +type Line = { level: Level; message: string; context: Logger.LogContext }; + +function capture() { + const lines: Line[] = []; + const logger: Logger.Config = { + level: 'DEBUG', + handler: (level, message, context) => { + lines.push({ level, message, context }); + }, + }; + const at = (level: Level) => lines.filter((line) => line.level === level); + return { logger, at }; +} + +function batching( + pushBatch: Destination.PushBatchFn, + batch: Destination.Config['batch'] = { wait: 60_000 }, +): Destination.Instance { + return { + type: 'batcher', + config: { batch }, + push: jest.fn(), + pushBatch, + }; +} + +describe('buffer policy (design Q4)', () => { + it('pre-run overflow refuses the newest event: ok false, queue-full, one ERROR', async () => { + const { at, logger } = capture(); + const { elb, collector } = await startFlow({ + run: false, + queueMax: 2, + logger, + }); + + const held = [await elb('page view'), await elb('page view')]; + const refused = await elb('page view'); + + for (const r of held) expect(r.ok).toBe(true); + expect(refused.ok).toBe(false); + expect(refused.reason).toMatchObject({ + code: 'queue-full', + retryable: true, + }); + expect(refused.error).toContain('collector.preRunQueue full'); + expect(collector.preRunQueue).toHaveLength(2); + expect(at(Level.ERROR)).toHaveLength(1); + expect(at(Level.ERROR)[0].context.code).toBe('queue-full'); + }); + + it('DLQ overflow drops the oldest entry with an ERROR cause line', async () => { + const { at, logger } = capture(); + const { elb, collector } = await startFlow({ + logger, + destinations: { + d: { + code: { + type: 'down', + config: { dlqMax: 2 }, + push: async () => { + throw new Error('down'); + }, + }, + }, + }, + }); + + await elb('page view'); + await elb('page view'); + await elb('page view'); + + expect(collector.destinations.d.dlq).toHaveLength(2); + expect(collector.status.dropped['destination.d'].dlq).toBe(1); + const overflow = at(Level.ERROR).filter( + (line) => line.context.code === 'dlq-overflow', + ); + expect(overflow).toHaveLength(1); + expect(overflow[0].message).toContain('status.dropped'); + }); + + it('collector.queue overflow is INFO: no present destination loses an event', async () => { + const { at, logger } = capture(); + const { elb } = await startFlow({ logger, queueMax: 2 }); + + await elb('page view'); + await elb('page view'); + await elb('page view'); + + expect(at(Level.INFO).map((line) => line.message)).toContain( + 'collector.queue overflow; oldest events dropped', + ); + expect(at(Level.WARN)).toHaveLength(0); + }); + + it('consent-wait overflow evicts the oldest at DEBUG and counts it (ruling 13)', async () => { + const { at, logger } = capture(); + const { elb, collector } = await startFlow({ + logger, + destinations: { + d: { + code: { + type: 'gated', + config: { consent: { marketing: true }, queueMax: 1 }, + push: jest.fn(), + }, + }, + }, + }); + + await elb('page view'); + await elb('page view'); + + expect(collector.status.dropped['destination.d'].queue).toBe(1); + expect( + at(Level.DEBUG).some((line) => + line.message.startsWith('destination.queuePush overflow'), + ), + ).toBe(true); + expect(at(Level.WARN)).toHaveLength(0); + }); + + it('batch entries flush at their size, never drop', async () => { + const sizes: number[] = []; + const { elb, collector } = await startFlow({ + destinations: { + d: { + code: batching( + async (batch) => { + sizes.push(batch.entries.length); + }, + { size: 2, wait: 60_000 }, + ), + }, + }, + }); + + for (let i = 0; i < 5; i++) await elb('page view'); + await elb('walker flush'); + + expect(sizes).toEqual([2, 2, 1]); + expect(collector.status.dropped['destination.d']).toBeUndefined(); + }); + + it('invalid input is WARN through the cause log, once per source per window', async () => { + const { at, logger } = capture(); + const probe: Source.Init< + Source.Types + > = async (context) => ({ + type: 'probe', + config: context.config, + push: context.env.push, + }); + const { elb, collector } = await startFlow({ + logger, + sources: { probe: { code: probe } }, + }); + + const r1 = await elb({ name: 'nospace' }); + const r2 = await elb({ name: 'nospace' }); + + expect(r1).toMatchObject({ ok: false, invalid: true }); + expect(r2).toMatchObject({ ok: false, invalid: true }); + expect(collector.status.sources.probe.rejected).toBe(2); + const warns = at(Level.WARN); + expect(warns).toHaveLength(1); + expect(warns[0].context).toMatchObject({ + step: 'source.probe', + code: 'invalid', + }); + expect(warns[0].message).toContain('status.sources.probe.rejected'); + }); +}); + +describe('walker flush (A17)', () => { + it('delivers a pending batch in one pushBatch and names the destination done', async () => { + const pushBatch = jest.fn(); + const { elb } = await startFlow({ + destinations: { d: { code: batching(pushBatch) } }, + }); + await elb('page view'); + await elb('page view'); + expect(pushBatch).not.toHaveBeenCalled(); + + const r = await elb('walker flush'); + + expect(pushBatch).toHaveBeenCalledTimes(1); + expect(pushBatch.mock.calls[0][0].entries).toHaveLength(2); + expect(r.ok).toBe(true); + expect(Object.keys(r.done ?? {})).toEqual(['d']); + }); + + it('with nothing pending answers ok and calls nothing; the collector keeps running', async () => { + const pushBatch = jest.fn(); + const { elb, collector } = await startFlow({ + destinations: { d: { code: batching(pushBatch) } }, + }); + + const idle = await elb('walker flush'); + expect(idle).toEqual({ ok: true }); + expect(pushBatch).not.toHaveBeenCalled(); + + const r = await elb('page view'); + expect(r.queued?.d.reason?.code).toBe('batch'); + expect(collector.hasShutdown).toBeFalsy(); + await elb('walker flush'); + expect(pushBatch).toHaveBeenCalledTimes(1); + }); + + it('a failing flush is failed through the outcome sink', async () => { + const { logger } = capture(); + const { elb, collector } = await startFlow({ + logger, + destinations: { + d: { + code: batching(async () => { + throw new Error('batch endpoint down'); + }), + }, + }, + }); + await elb('page view'); + await elb('page view'); + + const r = await elb('walker flush'); + + expect(r.ok).toBe(false); + expect(r.failed?.d.reason).toMatchObject({ + code: 'transport', + message: 'batch endpoint down', + }); + expect(collector.status.destinations.d.failed).toBe(2); + expect(collector.destinations.d.dlq).toHaveLength(2); + }); +}); diff --git a/packages/collector/src/__tests__/build-flags.test.ts b/packages/collector/src/__tests__/build-flags.test.ts index 4191ceaf7..bfed331be 100644 --- a/packages/collector/src/__tests__/build-flags.test.ts +++ b/packages/collector/src/__tests__/build-flags.test.ts @@ -29,13 +29,16 @@ function setFlag(flag: BuildFlag, value: boolean | undefined): void { globalThis[flag] = value; } -/** Capture WARN-level messages via an injected logger handler. */ -function warnCapture(): { warns: string[]; logger: Logger.Config } { - const warns: string[] = []; +/** + * Capture ERROR-level messages via an injected logger handler: a lean build + * ignoring configured features is a config value without effect (ERROR). + */ +function errorCapture(): { errors: string[]; logger: Logger.Config } { + const errors: string[] = []; const handler: Logger.Handler = (level, message) => { - if (level === Level.WARN) warns.push(message); + if (level === Level.ERROR) errors.push(message); }; - return { warns, logger: { level: 'WARN', handler } }; + return { errors, logger: { level: 'ERROR', handler } }; } interface ApiEnv extends Destination.BaseEnv { @@ -59,9 +62,9 @@ afterEach(() => { }); describe('__WALKEROS_OBSERVE__ false', () => { - it('installs no telemetry for config.observe, warns, and keeps caller observers', async () => { + it('installs no telemetry for config.observe, logs an ERROR, and keeps caller observers', async () => { setFlag('__WALKEROS_OBSERVE__', false); - const { warns, logger } = warnCapture(); + const { errors, logger } = errorCapture(); const observer: ObserverFn = () => undefined; const { collector } = await startFlow({ @@ -77,7 +80,7 @@ describe('__WALKEROS_OBSERVE__ false', () => { expect([...collector.observers]).toEqual([observer]); expect(collector.observeLevel).toBeUndefined(); - expect(warns).toContain('observe: not in this build, config ignored'); + expect(errors).toContain('observe: not in this build, config ignored'); }); it('skips trace-level call capture and hands the destination its env untouched', async () => { @@ -119,9 +122,9 @@ describe('__WALKEROS_OBSERVE__ false', () => { }); describe('__WALKEROS_STORES__ false', () => { - it('skips declared stores with a warning and keeps the default __cache', async () => { + it('skips declared stores with an ERROR and keeps the default __cache', async () => { setFlag('__WALKEROS_STORES__', false); - const { warns, logger } = warnCapture(); + const { errors, logger } = errorCapture(); const storeCode: Store.Init = () => { throw new Error('a declared store must not initialize'); }; @@ -132,7 +135,7 @@ describe('__WALKEROS_STORES__ false', () => { }); expect(Object.keys(collector.stores)).toEqual(['__cache']); - expect(warns).toContain('stores: not in this build, config ignored'); + expect(errors).toContain('stores: not in this build, config ignored'); await elb('walker shutdown'); expect(collector.hasShutdown).toBe(true); @@ -190,9 +193,9 @@ describe('__WALKEROS_STORES__ false', () => { }); describe('__WALKEROS_STATE__ false', () => { - it('ignores step state and warns once per step that carries it', async () => { + it('ignores step state and logs an ERROR once per step that carries it', async () => { setFlag('__WALKEROS_STATE__', false); - const { warns, logger } = warnCapture(); + const { errors, logger } = errorCapture(); const read: WalkerOS.Event[] = []; const sourceCode: Source.Init< Source.Types @@ -235,7 +238,7 @@ describe('__WALKEROS_STATE__ false', () => { expect(read).toHaveLength(2); expect(read.every((event) => event.data?.fetched === undefined)).toBe(true); - expect(warns).toEqual( + expect(errors).toEqual( expect.arrayContaining([ 'state: not in this build, source.seeder state ignored', 'state: not in this build, transformer.stasher state ignored', @@ -244,9 +247,9 @@ describe('__WALKEROS_STATE__ false', () => { ); }); - it('warns for a runtime destination that carries state', async () => { + it('logs an ERROR for a runtime destination that carries state', async () => { setFlag('__WALKEROS_STATE__', false); - const { warns, logger } = warnCapture(); + const { errors, logger } = errorCapture(); const { elb } = await startFlow({ logger }); await elb('walker destination', { @@ -255,7 +258,7 @@ describe('__WALKEROS_STATE__ false', () => { state: { mode: 'set', key: 'event.user.id', value: 'event.data.x' }, }); - expect(warns).toContain( + expect(errors).toContain( 'state: not in this build, destination.late state ignored', ); }); @@ -269,7 +272,7 @@ describe('__WALKEROS_VALIDATE__ false', () => { it('registers a transformer entry without runtime validation', async () => { setFlag('__WALKEROS_VALIDATE__', false); - const { warns, logger } = warnCapture(); + const { errors, logger } = errorCapture(); const { collector } = await startFlow({ logger, @@ -277,11 +280,11 @@ describe('__WALKEROS_VALIDATE__ false', () => { }); expect(collector.transformers.bad).toBeDefined(); - expect(warns).toEqual([]); + expect(errors).toEqual([]); }); it('flag absent: the same entry is validated and skipped', async () => { - const { warns, logger } = warnCapture(); + const { errors, logger } = errorCapture(); const { collector } = await startFlow({ logger, @@ -289,6 +292,6 @@ describe('__WALKEROS_VALIDATE__ false', () => { }); expect(collector.transformers.bad).toBeUndefined(); - expect(warns[0]).toMatch(/^Transformer bad invalid \(UNKNOWN_KEY\)/); + expect(errors[0]).toMatch(/^Transformer bad invalid \(UNKNOWN_KEY\)/); }); }); diff --git a/packages/collector/src/__tests__/destination-batch-correctness.test.ts b/packages/collector/src/__tests__/destination-batch-correctness.test.ts index e1f222f97..6e1221ed1 100644 --- a/packages/collector/src/__tests__/destination-batch-correctness.test.ts +++ b/packages/collector/src/__tests__/destination-batch-correctness.test.ts @@ -16,18 +16,6 @@ import type { Destination, Logger, WalkerOS } from '@walkeros/core'; import { createEvent, createIngest, Level } from '@walkeros/core'; import { pushToDestinations, startFlow } from '..'; -// Capture WARN-level log messages (+ their context) via a custom handler. -function makeWarnCapture(): { - warnings: Array<{ message: string; context: Logger.LogContext }>; - handler: Logger.Handler; -} { - const warnings: Array<{ message: string; context: Logger.LogContext }> = []; - const handler: Logger.Handler = (level, message, context) => { - if (level === Level.WARN) warnings.push({ message, context }); - }; - return { warnings, handler }; -} - describe('Destination batch correctness (PROD-004)', () => { beforeEach(() => { jest.useFakeTimers(); @@ -425,8 +413,11 @@ describe('Destination batch correctness (PROD-004)', () => { expect(destStatus.count).toBe(0); }); - it('caps the DLQ at dlqMax on a whole-batch failure, counts the overflow dropped, and warns once', async () => { - const { warnings, handler } = makeWarnCapture(); + it('caps the DLQ at dlqMax on a whole-batch failure, counts the overflow dropped, and logs it once', async () => { + const warnings: Array<{ message: string; context: Logger.LogContext }> = []; + const handler: Logger.Handler = (level, message, context) => { + if (level === Level.ERROR) warnings.push({ message, context }); + }; const mockPushBatch = jest.fn(() => { throw new Error('whole batch boom'); }); @@ -445,8 +436,7 @@ describe('Destination batch correctness (PROD-004)', () => { const { collector } = await startFlow({ destinations: { d: { code: dest } }, - // Default log level is ERROR; raise to WARN so the overflow warn surfaces. - logger: { level: 'WARN', handler }, + logger: { handler }, }); for (let i = 0; i < 5; i++) { @@ -478,14 +468,14 @@ describe('Destination batch correctness (PROD-004)', () => { expect(collector.status.destinations['d'].failed).toBe(5); expect(collector.status.destinations['d'].dlqSize).toBe(2); - // Overflow warn fired exactly once, with the cap + cumulative drop count. - const overflowWarns = warnings.filter((w) => - /destination\.dlq overflow/.test(w.message), + // The eviction is one ERROR cause line (replay material lost), with the + // cap and the cumulative drop count. + const overflow = warnings.filter((w) => w.context.code === 'dlq-overflow'); + expect(overflow.length).toBe(1); + expect(overflow[0].message).toContain( + 'destination.dlq full (dlqMax 2): oldest failed entries dropped', ); - expect(overflowWarns.length).toBe(1); - expect(overflowWarns[0].context.buffer).toBe('dlq'); - expect(overflowWarns[0].context.cap).toBe(2); - expect(overflowWarns[0].context.droppedCount).toBe(3); + expect(overflow[0].context.count).toBe(3); }); it('a returned BatchOutcome DLQs only the failed entries and counts the rest delivered', async () => { diff --git a/packages/collector/src/__tests__/destination-on-async.test.ts b/packages/collector/src/__tests__/destination-on-async.test.ts index 3f1c88e69..9d45efcff 100644 --- a/packages/collector/src/__tests__/destination-on-async.test.ts +++ b/packages/collector/src/__tests__/destination-on-async.test.ts @@ -1,5 +1,5 @@ import type { Destination, WalkerOS } from '@walkeros/core'; -import { createMockLogger, FatalError } from '@walkeros/core'; +import { createCauseLog, createMockLogger, FatalError } from '@walkeros/core'; import { startFlow } from '..'; import { getStateHold } from '../on'; @@ -107,26 +107,30 @@ describe('async destination on() settles before the first push', () => { }), }, }); - Object.assign(collector, { logger }); + // The cause log writes through the logger it was created with. + Object.assign(collector, { logger, causeLog: createCauseLog(logger) }); await elb('page view', {}); await elb('walker consent', { marketing: true }); expect(log).toEqual(['d:init', 'd:on:start']); expect(getStateHold(collector.destinations.d)?.type).toBe('consent'); // Held, not dropped: the event is back in the destination's own queue. expect(collector.destinations.d.queuePush).toHaveLength(1); - const errorCall = findLoggerError(logger, 'on callback failed'); + // A failed on(consent) is consent-not-applied: one line per cause. + const errorCall = findLoggerError( + logger, + 'consent not applied: the destination on(consent) handler failed; its events are held until a retry succeeds', + ); expect(errorCall?.[1]).toEqual( expect.objectContaining({ - kind: 'destination', - destId: 'd', - type: 'consent', - name: 'DestinationTimeoutError', + step: 'destination.d', + code: 'consent-not-applied', + error: 'Destination "d" on(consent) did not settle within 20ms', }), ); expect(collector.status.failed).toBe(0); }); - test('a rejecting on() holds the destination and is logged as an on callback failure', async () => { + test('a rejecting on(consent) holds the destination and is logged as consent-not-applied', async () => { const log: string[] = []; const logger = createMockLogger(); const { elb, collector } = await startFlow({ @@ -134,16 +138,20 @@ describe('async destination on() settles before the first push', () => { d: asyncRecorder(log, () => Promise.reject(new Error('sdk down'))), }, }); - Object.assign(collector, { logger }); + // The cause log writes through the logger it was created with. + Object.assign(collector, { logger, causeLog: createCauseLog(logger) }); await elb('page view', {}); await elb('walker consent', { marketing: true }); expect(log).toEqual(['d:init', 'd:on:start']); expect(getStateHold(collector.destinations.d)?.type).toBe('consent'); - const errorCall = findLoggerError(logger, 'on callback failed'); + const errorCall = findLoggerError( + logger, + 'consent not applied: the destination on(consent) handler failed; its events are held until a retry succeeds', + ); expect(errorCall?.[1]).toEqual( expect.objectContaining({ - kind: 'destination', - destId: 'd', + step: 'destination.d', + code: 'consent-not-applied', error: 'sdk down', }), ); @@ -344,10 +352,11 @@ describe('async destination on() settles before the first push', () => { ]); }); - test('a failed queued delta is re-delivered as the cell by the next queued entry, before the first push', async () => { + test('deltas queued while waiting coalesce into the whole cell; its failure holds the first push', async () => { const log: string[] = []; let calls = 0; const { elb, collector } = await startFlow({ + logger: { handler: () => undefined }, destinations: { d: dataRecorder( log, @@ -360,19 +369,29 @@ describe('async destination on() settles before the first push', () => { }, }); await elb('page view', {}); - // Not granted yet: the destination stays uninitialized and both deltas - // wait in its queue for the init flush. + // Not granted yet: the destination stays uninitialized and the deltas + // coalesce into one queued consent entry, the cell's latest value. await elb('walker consent', { functional: true }); expect(log).toEqual([]); + expect( + collector.destinations.d.queueOn?.filter( + (entry) => entry.type === 'consent', + ), + ).toHaveLength(1); await elb('walker consent', { marketing: true }); - expect(getStateHold(collector.destinations.d)).toBeUndefined(); + // The one delivery carries the whole cell and fails: the destination is + // held and the event is not pushed before its consent reached on(). expect(log).toEqual([ 'init', - 'start:consent:{"functional":true}', 'start:consent:{"functional":true,"marketing":true}', - 'end:consent', - 'push:page view', ]); + expect(getStateHold(collector.destinations.d)?.type).toBe('consent'); + expect(log).not.toContain('push:page view'); + + // The next state command re-delivers the cell; the held event follows. + await elb('walker consent', { analytics: true }); + expect(getStateHold(collector.destinations.d)).toBeUndefined(); + expect(log).toContain('push:page view'); }); }); diff --git a/packages/collector/src/__tests__/destination-on-timeout-overlap.test.ts b/packages/collector/src/__tests__/destination-on-timeout-overlap.test.ts index b24867351..cc2a33b68 100644 --- a/packages/collector/src/__tests__/destination-on-timeout-overlap.test.ts +++ b/packages/collector/src/__tests__/destination-on-timeout-overlap.test.ts @@ -1,5 +1,5 @@ import type { Destination, On, WalkerOS } from '@walkeros/core'; -import { createMockLogger } from '@walkeros/core'; +import { createCauseLog, createMockLogger } from '@walkeros/core'; import { startFlow } from '..'; import { getStateHold, isStateDeliveryInFlight } from '../on'; @@ -93,7 +93,7 @@ describe('a timed-out on() still running', () => { jest.useRealTimers(); }); - test('a grant that lands after its timeout is overwritten by the later deny before any event is delivered', async () => { + test('a grant that lands after its timeout is overwritten by the later deny; events past one window fail', async () => { const log: string[] = []; const hung = deferred(); const logger = createMockLogger(); @@ -101,17 +101,28 @@ describe('a timed-out on() still running', () => { destinations: { d: overlapRecorder(log, hung.promise) }, }); await elb('page view', {}); - Object.assign(collector, { logger }); + // The cause log writes through the logger it was created with. + Object.assign(collector, { logger, causeLog: createCauseLog(logger) }); log.length = 0; await grantThatTimesOut(elb); - await elb('page view', {}); + const held = await elb('page view', {}); await elb('walker consent', { marketing: false }); await jest.advanceTimersByTimeAsync(TIMEOUT); - await elb('page view', {}); + const blocked = await elb('page view', {}); + + // Within the window the event is held; once the failed grant has not + // recovered for a whole window the destination is blocked: the held + // event and this one fail instead of parking. + expect(held).toMatchObject({ queued: { d: { reason: { code: 'held' } } } }); + expect(blocked).toMatchObject({ + ok: false, + failed: { d: { reason: { code: 'blocked' } } }, + }); + expect(collector.status.destinations.d.failed).toBe(2); // The late grant lands on the vendor now. Its settle re-delivers the - // current consent and drains the held events, with no further push. + // current consent and releases the hold; nothing is left to drain. hung.resolve(); await jest.advanceTimersByTimeAsync(0); expect(log).toEqual([ @@ -119,12 +130,16 @@ describe('a timed-out on() still running', () => { 'end:{"marketing":true}', 'start:{"marketing":false}', 'end:{"marketing":false}', - 'push:page view', - 'push:page view', ]); expect(getStateHold(collector.destinations.d)).toBeUndefined(); expect(isStateDeliveryInFlight(collector.destinations.d)).toBe(false); - expect(loggerCalls(logger, 'error', 'on callback failed')).toHaveLength(1); + expect( + loggerCalls( + logger, + 'error', + 'consent not applied: the destination on(consent) handler failed; its events are held until a retry succeeds', + ), + ).toHaveLength(1); expect( loggerCalls(logger, 'debug', 'late settle after timeout'), ).toHaveLength(1); @@ -149,7 +164,9 @@ describe('a timed-out on() still running', () => { expect(log).toEqual(['start:{"marketing":true}']); expect(isStateDeliveryInFlight(collector.destinations.d)).toBe(true); - expect(collector.destinations.d.queuePush).toHaveLength(3); + // Each push comes a window after the failed grant: blocked, failed. + expect(collector.destinations.d.queuePush).toHaveLength(0); + expect(collector.status.destinations.d.failed).toBe(3); }); test('a sibling destination keeps receiving events during the hang', async () => { @@ -184,7 +201,7 @@ describe('a timed-out on() still running', () => { expect(getStateHold(collector.destinations.d)).toBeDefined(); }); - test('a handler that never settles keeps the destination held, and walker run discards its held events', async () => { + test('a handler that never settles keeps the destination blocked: its events fail, nothing parks', async () => { const log: string[] = []; const { elb, collector } = await startFlow({ destinations: { @@ -200,14 +217,16 @@ describe('a timed-out on() still running', () => { await jest.advanceTimersByTimeAsync(TIMEOUT * 10); await elb('page view', {}); } - expect(collector.destinations.d.queuePush).toHaveLength(3); + expect(collector.destinations.d.queuePush).toHaveLength(0); + expect(collector.status.destinations.d.failed).toBe(3); await elb('walker run'); await elb('page view', {}); expect(log).toEqual(['start:{"marketing":true}']); expect(getStateHold(collector.destinations.d)).toBeDefined(); - expect(collector.destinations.d.queuePush).toHaveLength(1); + expect(collector.destinations.d.queuePush).toHaveLength(0); + expect(collector.status.destinations.d.failed).toBe(4); }); test('a grant that rejects after its timeout behaves like one that resolves late', async () => { @@ -218,7 +237,8 @@ describe('a timed-out on() still running', () => { destinations: { d: overlapRecorder(log, hung.promise) }, }); await elb('page view', {}); - Object.assign(collector, { logger }); + // The cause log writes through the logger it was created with. + Object.assign(collector, { logger, causeLog: createCauseLog(logger) }); log.length = 0; await grantThatTimesOut(elb); @@ -237,7 +257,13 @@ describe('a timed-out on() still running', () => { ]); expect(getStateHold(collector.destinations.d)).toBeUndefined(); expect(isStateDeliveryInFlight(collector.destinations.d)).toBe(false); - expect(loggerCalls(logger, 'error', 'on callback failed')).toHaveLength(1); + expect( + loggerCalls( + logger, + 'error', + 'consent not applied: the destination on(consent) handler failed; its events are held until a retry succeeds', + ), + ).toHaveLength(1); }); test('a late settle while frequent other-cell commands arrive still recovers at once', async () => { @@ -484,7 +510,9 @@ describe('a handler that re-emits state during a catch-up outside a command', () expect(calls).toBeLessThanOrEqual(10); expect(log).toEqual(['push:page view']); expect(getStateHold(collector.destinations.d)).toBeDefined(); - expect(collector.destinations.d.queuePush).toHaveLength(1); + // The retry at the window end did not recover: blocked, the event fails. + expect(collector.destinations.d.queuePush).toHaveLength(0); + expect(collector.destinations.d.dlq).toHaveLength(1); expect( loggerCalls(logger, 'error', 'state delivery did not converge'), ).toHaveLength(1); diff --git a/packages/collector/src/__tests__/destination-outcome-state.test.ts b/packages/collector/src/__tests__/destination-outcome-state.test.ts new file mode 100644 index 000000000..a198a5749 --- /dev/null +++ b/packages/collector/src/__tests__/destination-outcome-state.test.ts @@ -0,0 +1,484 @@ +import { assembleJourneys, Level } from '@walkeros/core'; +import type { + Collector, + Destination, + FlowState, + Logger, + Source, + WalkerOS, +} from '@walkeros/core'; +import { startFlow } from '..'; + +type Line = { level: Level; message: string; context: Logger.LogContext }; + +/** Every log line at every level, through the logger's one handler. */ +function capture() { + const lines: Line[] = []; + const logger: Logger.Config = { + level: 'DEBUG', + handler: (level, message, context) => { + lines.push({ level, message, context }); + }, + }; + const at = (level: Level) => lines.filter((line) => line.level === level); + return { lines, logger, at }; +} + +function observe() { + const states: FlowState[] = []; + return { states, observer: (state: FlowState) => void states.push(state) }; +} + +describe('destination states, consent first', () => { + afterEach(() => { + jest.useRealTimers(); + }); + + it('(a) a consent-denied event on a breaker-open destination waits for consent, never fails', async () => { + const { at, logger } = capture(); + const push = jest.fn(async () => { + throw new Error('down'); + }); + const { elb, collector } = await startFlow({ + logger, + consent: { marketing: true }, + destinations: { + d: { + code: { + type: 'probe', + config: { + consent: { marketing: true }, + breaker: { threshold: 1, cooldown: 60_000 }, + }, + push, + }, + }, + }, + }); + await elb('page view'); + expect(collector.status.breakers['destination.d'].state).toBe('open'); + const errorsBefore = at(Level.ERROR).length; + + await elb('walker consent', { marketing: false }); + const r = await elb('page view'); + + expect(r.ok).toBe(true); + expect(r.failed).toBeUndefined(); + expect(r.queued?.d.reason).toEqual({ + code: 'consent', + message: 'required consent is not granted', + }); + expect(collector.destinations.d.dlq).toHaveLength(1); + expect(collector.status.destinations.d.failed).toBe(1); + expect(at(Level.ERROR)).toHaveLength(errorsBefore); + }); + + it('(b) consentDenied skip: counted, DEBUG, Observe skip, nothing queued, ok without queued', async () => { + const { at, logger } = capture(); + const { states, observer } = observe(); + const push = jest.fn(); + const { elb, collector } = await startFlow({ + logger, + consentDenied: 'skip', + observers: [observer], + destinations: { + d: { + code: { + type: 'probe', + config: { consent: { marketing: true } }, + push, + }, + }, + }, + }); + + const r = await elb('page view'); + + expect(r).toMatchObject({ ok: true }); + expect(r.queued).toBeUndefined(); + expect(push).not.toHaveBeenCalled(); + expect(collector.destinations.d.queuePush ?? []).toHaveLength(0); + expect(collector.status.destinations.d.skipped).toBe(1); + expect( + at(Level.DEBUG).some( + (line) => + line.message === 'event skipped' && line.context.reason === 'consent', + ), + ).toBe(true); + expect( + states.some( + (state) => + state.stepId === 'destination.d' && + state.phase === 'skip' && + state.skipReason === 'consent', + ), + ).toBe(true); + expect(at(Level.ERROR)).toHaveLength(0); + expect(at(Level.WARN)).toHaveLength(0); + }); + + it('(c) a failed on(consent) holds within one window, then blocks', async () => { + jest.useFakeTimers({ now: 1_000_000 }); + const push = jest.fn(); + const dest: Destination.Instance = { + type: 'probe', + config: { timeout: 1000 }, + on: (type) => { + if (type === 'consent') throw new Error('vendor gone'); + }, + push, + }; + const { states, observer } = observe(); + const { logger } = capture(); + const { elb, collector } = await startFlow({ + logger, + observers: [observer], + destinations: { d: { code: dest } }, + }); + await elb('walker consent', { marketing: true }); + + const held = await elb('page view'); + expect(held.queued?.d.reason).toEqual({ + code: 'held', + message: 'a state delivery to the destination is running or failed', + }); + expect(held.failed).toBeUndefined(); + expect( + states.some( + (state) => state.phase === 'skip' && state.skipReason === 'held', + ), + ).toBe(true); + + jest.setSystemTime(1_000_000 + 1001); + const blocked = await elb('page view'); + expect(blocked.ok).toBe(false); + expect(blocked.failed?.d.reason?.code).toBe('blocked'); + // The held event and this one fail; nothing parks for the page lifetime. + expect(collector.status.destinations.d.failed).toBe(2); + expect(collector.destinations.d.dlq).toHaveLength(2); + expect(collector.destinations.d.queuePush ?? []).toHaveLength(0); + expect(push).not.toHaveBeenCalled(); + }); + + it('(d) breaker open: every event in the cooldown fails breaker-open, counted, one line', async () => { + const { at, logger } = capture(); + const push = jest.fn(async () => { + throw new Error('down'); + }); + const { elb, collector } = await startFlow({ + logger, + destinations: { + d: { + code: { + type: 'probe', + config: { breaker: { threshold: 2, cooldown: 60_000 } }, + push, + }, + }, + }, + }); + await elb('page view'); + await elb('page view'); + const results = [ + await elb('page view'), + await elb('page view'), + await elb('page view'), + ]; + + expect(push).toHaveBeenCalledTimes(2); + for (const r of results) { + expect(r.ok).toBe(false); + expect(r.failed?.d.reason).toMatchObject({ + code: 'breaker-open', + retryable: true, + }); + } + expect(collector.status.destinations.d.failed).toBe(5); + expect(collector.destinations.d.dlq).toHaveLength(5); + const codes = at(Level.ERROR).map((line) => line.context.code); + expect(codes).toEqual(['transport', 'breaker-open']); + }); + + it('(e) held overflow evicts consent waits first, then refuses the newest held event', async () => { + const { at, logger } = capture(); + const push = jest.fn(); + const dest: Destination.Instance = { + type: 'probe', + config: { consent: { marketing: true }, queueMax: 2, timeout: 60_000 }, + on: (type) => { + if (type === 'user') throw new Error('user sync down'); + }, + push, + }; + const { elb, collector } = await startFlow({ + logger, + consent: { marketing: true }, + destinations: { d: { code: dest } }, + }); + await elb('page view'); // initialized and delivered + await elb('walker user', { id: 'u1' }); // on(user) throws: held + await elb('walker consent', { marketing: false }); + await elb('page view'); // consent wait + await elb('page view'); // consent wait + await elb('walker consent', { marketing: true }); + expect( + (collector.destinations.d.queuePush ?? []).map((entry) => entry.reason), + ).toEqual(['consent', 'consent']); + + const first = await elb('page view'); + const second = await elb('page view'); + expect(first.queued?.d.reason?.code).toBe('held'); + expect(second.queued?.d.reason?.code).toBe('held'); + expect( + (collector.destinations.d.queuePush ?? []).map((entry) => entry.reason), + ).toEqual(['held', 'held']); + expect(collector.status.dropped['destination.d'].queue).toBe(2); + + const refused = await elb('page view'); + expect(refused.ok).toBe(false); + expect(refused.failed?.d.reason?.code).toBe('queue-full'); + expect(at(Level.ERROR).map((line) => line.context.code)).toContain( + 'queue-full', + ); + + // A new consent wait never evicts a held event: it is the one dropped. + await elb('walker consent', { marketing: false }); + const denied = await elb('page view'); + expect(denied.queued?.d.reason?.code).toBe('consent'); + expect( + (collector.destinations.d.queuePush ?? []).map((entry) => entry.reason), + ).toEqual(['held', 'held']); + expect(collector.status.dropped['destination.d'].queue).toBe(3); + // Consent evictions are never above DEBUG. + expect( + at(Level.WARN).filter((line) => line.message.includes('queuePush')), + ).toHaveLength(0); + }); + + it('(e) consent-only overflow keeps its bound: oldest dropped, DEBUG once, ok unchanged', async () => { + const { at, logger } = capture(); + const { elb, collector } = await startFlow({ + logger, + destinations: { + d: { + code: { + type: 'probe', + config: { consent: { marketing: true }, queueMax: 2 }, + push: jest.fn(), + }, + }, + }, + }); + const results = [ + await elb('page view'), + await elb('page view'), + await elb('page view'), + await elb('page view'), + ]; + for (const r of results) { + expect(r.ok).toBe(true); + expect(r.queued?.d.reason?.code).toBe('consent'); + } + expect(collector.destinations.d.queuePush).toHaveLength(2); + expect(collector.status.dropped['destination.d'].queue).toBe(2); + const overflow = at(Level.DEBUG).filter((line) => + line.message.startsWith('destination.queuePush overflow'), + ); + expect(overflow).toHaveLength(1); + expect(at(Level.WARN)).toHaveLength(0); + expect(at(Level.ERROR)).toHaveLength(0); + }); + + it('(f) a throwing on(consent) is consent-not-applied; a throwing on(user) stays on callback failed', async () => { + const { at, logger } = capture(); + const failingOn = + (failing: string): Destination.Instance['on'] => + (type) => { + if (type === failing) throw new Error('handler broke'); + }; + const dest: Destination.Instance = { + type: 'probe', + config: {}, + on: failingOn('consent'), + push: jest.fn(), + }; + const { elb } = await startFlow({ + logger, + destinations: { d: { code: dest } }, + }); + await elb('page view'); // initializes the destination + await elb('walker consent', { marketing: true }); + + const consentLines = at(Level.ERROR); + expect(consentLines).toHaveLength(1); + expect(consentLines[0].context).toMatchObject({ + step: 'destination.d', + code: 'consent-not-applied', + }); + expect( + consentLines.some((line) => line.message === 'on callback failed'), + ).toBe(false); + + const { at: atUser, logger: userLogger } = capture(); + const { elb: elbUser } = await startFlow({ + logger: userLogger, + destinations: { d: { code: { ...dest, on: failingOn('user') } } }, + }); + await elbUser('page view'); + await elbUser('walker user', { id: 'u1' }); + expect(atUser(Level.ERROR).map((line) => line.message)).toContain( + 'on callback failed', + ); + }); + + it('(g) a batched event waits with reason batch and keeps its batched out; a held event emits skip held', async () => { + const { logger } = capture(); + const { states, observer } = observe(); + const pushBatch = jest.fn(); + const held: Destination.Instance = { + type: 'held', + config: { timeout: 60_000 }, + on: (type) => { + if (type === 'consent') throw new Error('vendor gone'); + }, + push: jest.fn(), + }; + const { elb } = await startFlow({ + logger, + observers: [observer], + destinations: { + d: { + code: { + type: 'probe', + config: { batch: { wait: 60_000 } }, + push: jest.fn(), + pushBatch, + }, + }, + h: { code: held }, + }, + }); + await elb('walker consent', { marketing: true }); // h is held from here + + const r = await elb('page view'); + + expect(r.ok).toBe(true); + expect(r.done).toBeUndefined(); + expect(r.queued?.d.reason).toEqual({ + code: 'batch', + message: 'waiting in a batch window until it flushes', + }); + expect(r.queued?.h.reason?.code).toBe('held'); + expect(pushBatch).not.toHaveBeenCalled(); + + // Observe: the batch-stamped out stays the batch signal (no skip for a + // batch); the held destination emits skip with reason held. + const eventId = r.event?.id; + const ofEvent = states.filter((state) => state.eventId === eventId); + expect( + ofEvent.some( + (state) => state.stepId === 'destination.d' && state.phase === 'skip', + ), + ).toBe(false); + expect( + ofEvent.some( + (state) => + state.stepId === 'destination.h' && + state.phase === 'skip' && + state.skipReason === 'held', + ), + ).toBe(true); + const hopOf = () => + assembleJourneys(states) + .journeys.find((journey) => journey.id === eventId) + ?.hops.find((hop) => hop.stepId === 'destination.d'); + expect(hopOf()).toMatchObject({ terminalPhase: 'out', batched: true }); + + await elb('walker flush'); + expect(pushBatch).toHaveBeenCalledTimes(1); + expect(hopOf()).toMatchObject({ batched: true, flushConfirmed: true }); + }); + + it('(h) an ignored event is a skip at a destination, not a delivery', async () => { + const { at, logger } = capture(); + const { states, observer } = observe(); + const push = jest.fn(); + const { elb, collector } = await startFlow({ + logger, + observers: [observer], + destinations: { + d: { + code: { type: 'probe', config: {}, push }, + config: { mapping: { page: { view: { ignore: true } } } }, + }, + }, + }); + const outBefore = collector.status.out; + + const r = await elb('page view'); + + expect(r.ok).toBe(true); + expect(r.done).toBeUndefined(); + expect(push).not.toHaveBeenCalled(); + expect(collector.status.destinations.d.skipped).toBe(1); + expect(collector.status.destinations.d.count).toBe(0); + expect(collector.status.out).toBe(outBefore); + expect( + at(Level.DEBUG).filter( + (line) => + line.message === 'event skipped' && line.context.reason === 'ignored', + ), + ).toHaveLength(1); + expect( + states.some( + (state) => + state.stepId === 'destination.d' && + state.phase === 'skip' && + state.skipReason === 'ignored', + ), + ).toBe(true); + }); + + it('(h) an ignored event at a source answers dropped and is a collector drop', async () => { + const { states, observer } = observe(); + const delivered: WalkerOS.Event[] = []; + const probe: Source.Init< + Source.Types + > = async (context) => ({ + type: 'probe', + config: context.config, + push: context.env.push, + }); + const { elb, collector } = await startFlow({ + observers: [observer], + sources: { + probe: { + code: probe, + config: { mapping: { page: { view: { ignore: true } } } }, + }, + }, + destinations: { + d: { + code: { + type: 'sink', + config: {}, + push: async (event) => void delivered.push(event), + }, + }, + }, + }); + const inBefore = collector.status.in; + + const r = await elb({ name: 'page view' }); + + expect(r).toEqual({ ok: true, dropped: true }); + expect(delivered).toHaveLength(0); + expect(collector.status.in).toBe(inBefore + 1); + expect(collector.status.out).toBe(0); + const drop = states.find( + (state) => state.stepId === 'collector.push' && state.phase === 'skip', + ); + expect(drop?.skipReason).toBe('ignored'); + expect(drop?.meta).toEqual({ at: 'source.probe' }); + }); +}); diff --git a/packages/collector/src/__tests__/destination-state-order.test.ts b/packages/collector/src/__tests__/destination-state-order.test.ts index ea492d443..e593e5b73 100644 --- a/packages/collector/src/__tests__/destination-state-order.test.ts +++ b/packages/collector/src/__tests__/destination-state-order.test.ts @@ -187,13 +187,9 @@ describe('destination on(consent) precedes the first push (further activation pa expect(collector.destinations.rec).toBeDefined(); expect(log).toEqual([]); await elb('walker consent', { marketing: true }); - // The denied snapshot delivered at activation stays in queueOn and is - // replayed in FIFO order at init; order is preserved, nothing collapsed. - expect(log).toEqual([ - 'init', - 'on:consent:{"marketing":false}', - GRANT, - 'push:page view', - ]); + // While the destination waits, a consent delivery replaces the queued + // one: the cell's latest value supersedes the denied snapshot, so the + // grant alone reaches on(), still before the queued event. + expect(log).toEqual(['init', GRANT, 'push:page view']); }); }); diff --git a/packages/collector/src/__tests__/destination.test.ts b/packages/collector/src/__tests__/destination.test.ts index f64b2fcc1..d5529f468 100644 --- a/packages/collector/src/__tests__/destination.test.ts +++ b/packages/collector/src/__tests__/destination.test.ts @@ -127,13 +127,14 @@ describe('Destination', () => { }); test('failing init', async () => { - // Simulate a failed init + // Simulate a failed init: a returned false is not-ready, a failure. mockInit.mockImplementation(() => false); - await pushToDestinations(createWalkerjs(), event); + const result = await pushToDestinations(createWalkerjs(), event); expect(mockInit).toHaveBeenCalledTimes(1); expect(mockPush).toHaveBeenCalledTimes(0); expect(destination.config.init).toBeFalsy(); + expect(result.failed?.foo.reason?.code).toBe('not-ready'); }); test('logs an ERROR and counts when destination init throws', async () => { @@ -153,12 +154,11 @@ describe('Destination', () => { // The collector must surface the failure at ERROR on the scoped logger // AND increment status.failed (Category A - internal pipeline failure). - expect(collector.logger.scope).toHaveBeenCalledWith('unknown'); - const scopedLogger = (collector.logger.scope as jest.Mock).mock.results[0] - .value; - expect(scopedLogger.error).toHaveBeenCalledWith( + // One cause line for the failure (no cause log on this hand-built + // collector: written through its logger, filtered). + expect(collector.logger.error).toHaveBeenCalledWith( 'destination init failed', - expect.objectContaining({ error: 'boom' }), + expect.objectContaining({ step: 'destination.foo', error: 'boom' }), ); expect(collector.status.failed).toBe(1); }); @@ -178,17 +178,16 @@ describe('Destination', () => { expect(mockInit).toHaveBeenCalledTimes(1); expect(mockPush).toHaveBeenCalledTimes(0); - expect(collector.logger.scope).toHaveBeenCalledWith('unknown'); - const scopedLogger = (collector.logger.scope as jest.Mock).mock.results[0] - .value; - expect(scopedLogger.error).toHaveBeenCalledWith( + // One cause line for the failure (no cause log on this hand-built + // collector: written through its logger, filtered). + expect(collector.logger.error).toHaveBeenCalledWith( 'destination init failed', - expect.objectContaining({ error: 'boom' }), + expect.objectContaining({ step: 'destination.foo', error: 'boom' }), ); expect(collector.status.failed).toBe(1); }); - test('logs an ERROR and counts when queueOn-only init throws', async () => { + test('logs an ERROR when queueOn-only init throws', async () => { // Mirrors the init-throw test above but exercises the alternate code path // at destination.ts:190 (no push events, only queueOn events). That branch // previously used tryCatchAsync(destinationInit) without an onError, which @@ -209,17 +208,17 @@ describe('Destination', () => { expect(mockInit).toHaveBeenCalledTimes(1); expect(mockPush).toHaveBeenCalledTimes(0); - expect(collector.logger.scope).toHaveBeenCalledWith('unknown'); - const scopedLogger = (collector.logger.scope as jest.Mock).mock.results[0] - .value; - expect(scopedLogger.error).toHaveBeenCalledWith( + // One cause line for the failure (no cause log on this hand-built + // collector: written through its logger, filtered). No event was lost, + // so nothing counts as failed. + expect(collector.logger.error).toHaveBeenCalledWith( 'destination init failed', - expect.objectContaining({ error: 'boom' }), + expect.objectContaining({ step: 'destination.foo', error: 'boom' }), ); - expect(collector.status.failed).toBe(1); + expect(collector.status.failed).toBe(0); }); - test('logs an ERROR and counts when queueOn-only init rejects', async () => { + test('logs an ERROR when queueOn-only init rejects', async () => { // Async-rejection variant of the queueOn-only init throw test. Guards // against future refactors that drop the await on the queueOn-only branch. mockInit.mockRejectedValue(new Error('boom')); @@ -233,14 +232,14 @@ describe('Destination', () => { expect(mockInit).toHaveBeenCalledTimes(1); expect(mockPush).toHaveBeenCalledTimes(0); - expect(collector.logger.scope).toHaveBeenCalledWith('unknown'); - const scopedLogger = (collector.logger.scope as jest.Mock).mock.results[0] - .value; - expect(scopedLogger.error).toHaveBeenCalledWith( + // One cause line for the failure (no cause log on this hand-built + // collector: written through its logger, filtered). No event was lost, + // so nothing counts as failed. + expect(collector.logger.error).toHaveBeenCalledWith( 'destination init failed', - expect.objectContaining({ error: 'boom' }), + expect.objectContaining({ step: 'destination.foo', error: 'boom' }), ); - expect(collector.status.failed).toBe(1); + expect(collector.status.failed).toBe(0); }); test('logs init lifecycle', async () => { @@ -327,11 +326,16 @@ describe('Destination', () => { { destination: createDestination() }, ); - const call = (scoped.error as jest.Mock).mock.calls.find( + const call = (rootLogger.error as jest.Mock).mock.calls.find( (args) => args[0] === 'Push failed', ); - expect(call).toBeDefined(); - expect(call?.[1]).toEqual({ error: 'kaputt', event: event.name }); + expect(call?.[1]).toEqual({ + step: 'destination.destination', + code: 'transport', + retryable: true, + error: 'kaputt', + event: event.name, + }); // A raw Error has no enumerable own properties: serialized it collapses // to `"error":{}` and the cause is gone. expect(JSON.stringify(call?.[1])).toContain('kaputt'); @@ -381,7 +385,7 @@ describe('Destination', () => { deniedEvent.name = 'denied action'; deniedEvent.entity = 'denied'; deniedEvent.action = 'action'; - destinationWithConsent.queuePush = [deniedEvent]; + destinationWithConsent.queuePush = [{ event: deniedEvent }]; // Push a new event that carries individual marketing consent. // New events are cloned (not consent-overwritten), so this one @@ -396,7 +400,9 @@ describe('Destination', () => { // The denied (queued) event should still be in the queue expect(destinationWithConsent.queuePush).toHaveLength(1); - expect(destinationWithConsent.queuePush[0].name).toBe('denied action'); + expect(destinationWithConsent.queuePush?.[0].event.name).toBe( + 'denied action', + ); }); describe('destination on method', () => { @@ -942,7 +948,7 @@ describe('Destination', () => { // queue (queuePush on the live instance), proving gate-not-drop. const live = collector.destinations.d; expect(live.queuePush).toHaveLength(1); - expect(live.queuePush?.[0].name).toBe('page view'); + expect(live.queuePush?.[0].event.name).toBe('page view'); // Grant consent: the gated event must replay through the batch buffer. await elb('walker consent', { marketing: true }); diff --git a/packages/collector/src/__tests__/fanout.test.ts b/packages/collector/src/__tests__/fanout.test.ts new file mode 100644 index 000000000..64f6ae611 --- /dev/null +++ b/packages/collector/src/__tests__/fanout.test.ts @@ -0,0 +1,138 @@ +import type { + Collector, + Destination, + Logger, + Source, + Transformer, + WalkerOS, +} from '@walkeros/core'; +import { startFlow } from '..'; + +/** The failure lines are asserted elsewhere; keep the output clean. */ +const logger: Logger.Config = { handler: () => undefined }; + +/** Forks every event into two copies, `data.copy` 1 and 2. */ +const fork: Transformer.Instance = { + type: 'fork', + config: {}, + push: (event) => [ + { event: { ...event, data: { copy: 1 } } }, + { event: { ...event, data: { copy: 2 } } }, + ], +}; + +/** Refuses copy 2, accepts the rest. */ +function picky(delivered: WalkerOS.Event[]): Destination.Instance { + return { + type: 'picky', + config: {}, + push: async (event) => { + if (event.data.copy === 2) throw new Error('copy 2 refused'); + delivered.push(event); + }, + }; +} + +const probe: Source.Init< + Source.Types +> = async (context) => ({ + type: 'probe', + config: context.config, + push: context.env.push, +}); + +const throwing: Transformer.Instance = { + type: 'broken', + config: {}, + push: () => { + throw new Error('transformer bug'); + }, +}; + +const dropping: Transformer.Instance = { + type: 'filter', + config: {}, + push: () => false, +}; + +describe('fan-out reports every copy (C3)', () => { + it.each<[string, (sink: Destination.Instance) => Collector.InitConfig]>([ + [ + 'collector next', + (sink) => ({ + logger, + next: 'fork', + transformers: { fork: { code: async () => fork } }, + destinations: { d: { code: sink } }, + }), + ], + [ + 'source next', + (sink) => ({ + logger, + sources: { probe: { code: probe, next: 'fork' } }, + transformers: { fork: { code: async () => fork } }, + destinations: { d: { code: sink } }, + }), + ], + [ + 'source before', + (sink) => ({ + logger, + sources: { probe: { code: probe, before: 'fork' } }, + transformers: { fork: { code: async () => fork } }, + destinations: { d: { code: sink } }, + }), + ], + ])('%s: a copy failing at a destination fails the push', async (_, init) => { + const delivered: WalkerOS.Event[] = []; + const { elb } = await startFlow(init(picky(delivered))); + + const r = await elb({ name: 'page view', data: {} }); + + expect(delivered).toHaveLength(1); + expect(r.ok).toBe(false); + expect(r.failed?.d.reason?.message).toBe('copy 2 refused'); + expect(r.done?.d).toBeUndefined(); + }); +}); + +describe('transformer errors are failures, drops are drops (C6)', () => { + it('a throwing transformer fails the push with its reason, not dropped', async () => { + const delivered: WalkerOS.Event[] = []; + const { elb, collector } = await startFlow({ + logger, + next: 'broken', + transformers: { broken: { code: async () => throwing } }, + destinations: { d: { code: picky(delivered) } }, + }); + + const r = await elb({ name: 'page view', data: {} }); + + expect(r.ok).toBe(false); + expect(r.dropped).toBeUndefined(); + expect(r.error).toContain('broken'); + expect(r.reason).toEqual({ + code: 'transformer', + message: 'transformer bug', + retryable: false, + }); + expect(delivered).toHaveLength(0); + expect(collector.status.failed).toBe(1); + }); + + it('a transformer returning false is a deliberate drop', async () => { + const delivered: WalkerOS.Event[] = []; + const { elb } = await startFlow({ + next: 'filter', + transformers: { filter: { code: async () => dropping } }, + destinations: { d: { code: picky(delivered) } }, + }); + + const r = await elb({ name: 'page view', data: {} }); + + expect(r).toMatchObject({ ok: true, dropped: true }); + expect(r.reason).toBeUndefined(); + expect(delivered).toHaveLength(0); + }); +}); diff --git a/packages/collector/src/__tests__/init-contract.test.ts b/packages/collector/src/__tests__/init-contract.test.ts new file mode 100644 index 000000000..a68d3a76a --- /dev/null +++ b/packages/collector/src/__tests__/init-contract.test.ts @@ -0,0 +1,43 @@ +import { StepError } from '@walkeros/core'; +import type { Destination } from '@walkeros/core'; +import { startFlow } from '..'; + +describe('init contract', () => { + it('a thrown StepError from init fails the push with its reason (B1, B2)', async () => { + const dest: Destination.Instance = { + type: 'needs-key', + config: {}, + init: () => { + throw new StepError('Config settings apiKey missing', { + code: 'config', + }); + }, + push: jest.fn(), + }; + const { elb, collector } = await startFlow({ + destinations: { d: { code: dest } }, + }); + const r = await elb('page view'); + expect(r.ok).toBe(false); + expect(r.queued).toBeUndefined(); + expect(r.failed?.d.reason?.code).toBe('config'); + expect(collector.status.destinations.d.failed).toBe(1); + }); + + it('a false returned at runtime (untyped bundle code) is reported as not-ready (guard)', async () => { + const initFalse = (): false => false; + const legacy: Destination.Instance = { + type: 'legacy', + config: {}, + init: initFalse, + push: jest.fn(), + }; + const { elb, collector } = await startFlow({ + destinations: { l: { code: legacy } }, + }); + const r = await elb('page view'); + expect(r.ok).toBe(false); + expect(r.failed?.l.reason?.code).toBe('not-ready'); + expect(collector.status.destinations.l.failed).toBe(1); + }); +}); diff --git a/packages/collector/src/__tests__/invalid-event.test.ts b/packages/collector/src/__tests__/invalid-event.test.ts index 3f3ef3152..c6004b559 100644 --- a/packages/collector/src/__tests__/invalid-event.test.ts +++ b/packages/collector/src/__tests__/invalid-event.test.ts @@ -53,16 +53,22 @@ describe('invalid inbound events', () => { }, ); - it('logs one warn carrying only the reason', async () => { + it('logs one warn per source and window, carrying only the reason', async () => { const collector = await createTestCollector(records); + await collector.push({}, { id: 'web' }); await collector.push({}, { id: 'web' }); expect(records).toEqual([ { level: Level.WARN, - message: 'invalid event rejected', - context: { error: 'Event name is required' }, + message: + 'invalid event rejected (repeats within a minute are counted in status.sources.web.rejected)', + context: { + step: 'source.web', + code: 'invalid', + error: 'Event name is required', + }, }, ]); }); diff --git a/packages/collector/src/__tests__/mapping-failures.test.ts b/packages/collector/src/__tests__/mapping-failures.test.ts new file mode 100644 index 000000000..fe514265c --- /dev/null +++ b/packages/collector/src/__tests__/mapping-failures.test.ts @@ -0,0 +1,147 @@ +import { Level } from '@walkeros/core'; +import type { + Collector, + Destination, + Logger, + Mapping, + Source, +} from '@walkeros/core'; +import { startFlow } from '..'; + +function errors() { + const lines: Array<{ message: string; context: Logger.LogContext }> = []; + const logger: Logger.Config = { + handler: (level, message, context) => { + if (level === Level.ERROR) lines.push({ message, context }); + }, + }; + return { lines, logger }; +} + +const broken: Mapping.Condition = () => { + throw new Error('condition bug'); +}; + +const probe: Source.Init< + Source.Types +> = async (context) => ({ + type: 'probe', + config: context.config, + push: context.env.push, +}); + +describe('mapping failures (ruling 2)', () => { + it('(a) a throwing rule condition fails the destination, never falls through', async () => { + const { lines, logger } = errors(); + const push = jest.fn(); + const { elb, collector } = await startFlow({ + logger, + destinations: { + d: { + code: { type: 'probe', config: {}, push }, + config: { + mapping: { + order: { + complete: [ + { condition: broken, ignore: true }, + { name: 'purchase' }, + ], + }, + }, + }, + }, + }, + }); + + const r = await elb('order complete', { email: 'person@example.com' }); + + expect(r.ok).toBe(false); + expect(r.failed?.d.reason).toMatchObject({ + code: 'mapping', + retryable: false, + }); + expect(push).not.toHaveBeenCalled(); + expect(collector.status.destinations.d.failed).toBe(1); + expect(lines).toHaveLength(1); + expect(JSON.stringify(lines)).not.toContain('person@example.com'); + }); + + it('(a) a throwing rule condition at a source fails the push with reason mapping', async () => { + const { logger } = errors(); + const push = jest.fn(); + const { elb } = await startFlow({ + logger, + sources: { + probe: { + code: probe, + config: { mapping: { order: { complete: { condition: broken } } } }, + }, + }, + destinations: { d: { code: { type: 'probe', config: {}, push } } }, + }); + + const r = await elb({ name: 'order complete', data: {} }); + + expect(r.ok).toBe(false); + expect(r.reason).toMatchObject({ code: 'mapping', retryable: false }); + expect(push).not.toHaveBeenCalled(); + }); + + it('(b, c, d) a throwing fn and an operator-less value drop the value only: delivered, reported, no payload', async () => { + const { lines, logger } = errors(); + const received: unknown[] = []; + const order: Record = { total: 'data.total' }; + const destination: Destination.Instance = { + type: 'probe', + config: {}, + push: (_event, context) => void received.push(context.data), + }; + const { elb, collector } = await startFlow({ + logger, + destinations: { + d: { + code: destination, + config: { + mapping: { + order: { + complete: { + data: { + map: { + order, + currency: 'data.currency', + broken: { + fn: () => { + throw new Error('fn bug'); + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }); + + const r = await elb('order complete', { + total: 42, + currency: 'EUR', + email: 'person@example.com', + }); + + expect(r.ok).toBe(true); + expect(received).toEqual([{ currency: 'EUR' }]); + expect(collector.status.failed).toBe(0); + expect(lines.map((line) => line.context)).toEqual([ + expect.objectContaining({ + step: 'mapping', + code: 'mapping', + event: 'order complete', + mappingKey: 'order complete', + }), + ]); + expect(lines[0].message).toContain('mapping value has no operator (total)'); + expect(JSON.stringify(lines)).not.toContain('person@example.com'); + }); +}); diff --git a/packages/collector/src/__tests__/on-callback-error.test.ts b/packages/collector/src/__tests__/on-callback-error.test.ts index 73cd91b02..28951c274 100644 --- a/packages/collector/src/__tests__/on-callback-error.test.ts +++ b/packages/collector/src/__tests__/on-callback-error.test.ts @@ -55,9 +55,9 @@ describe('on.ts user-callback throws (Category B)', () => { }, } as unknown as Destination.Instance; - callDestinationOn(collector, destination, 'd1', 'consent', { - marketing: true, - }); + // A failed on(consent) is consent-not-applied (destination-outcome-state + // test f); every other type keeps the user-code line. + callDestinationOn(collector, destination, 'd1', 'user', { id: 'u1' }); expect(collector.status.failed).toBe(0); const errorCall = findLoggerError(collector, 'on callback failed'); diff --git a/packages/collector/src/__tests__/on-without-init.test.ts b/packages/collector/src/__tests__/on-without-init.test.ts new file mode 100644 index 000000000..2d4a4ebbc --- /dev/null +++ b/packages/collector/src/__tests__/on-without-init.test.ts @@ -0,0 +1,106 @@ +import { Level } from '@walkeros/core'; +import type { Destination, Logger, On } from '@walkeros/core'; +import { startFlow } from '..'; + +/** A destination with `on` and no `init`, recording every delivery. */ +function recorder(config: Destination.Config = {}) { + const calls: Array<{ type: On.Types; data: unknown }> = []; + const destination: Destination.Instance = { + type: 'recorder', + config, + on: (type, context) => { + calls.push({ type, data: context.data }); + }, + push: jest.fn(), + }; + return { calls, destination }; +} + +function errors() { + const lines: string[] = []; + const logger: Logger.Config = { + handler: (level, message) => { + if (level === Level.ERROR) lines.push(message); + }, + }; + return { lines, logger }; +} + +describe('on() without init', () => { + it('(a) receives on(consent) after a consent command; queueOn drains (B3)', async () => { + const { calls, destination } = recorder(); + const { elb, collector } = await startFlow({ + destinations: { d: { code: destination } }, + }); + + await elb('walker consent', { marketing: true }); + + expect(calls.map((call) => call.type)).toContain('consent'); + expect(collector.destinations.d.queueOn ?? []).toHaveLength(0); + expect(collector.destinations.d.config.init).toBe(true); + }); + + it('(b) a consent-gated destination gets no state before the grant, then all of it in order', async () => { + const { calls, destination } = recorder({ consent: { marketing: true } }); + const { elb } = await startFlow({ + destinations: { d: { code: destination } }, + }); + + await elb('walker user', { id: 'u1' }); + await elb('walker globals', { pagegroup: 'docs' }); + expect(calls).toHaveLength(0); + + await elb('walker consent', { marketing: true }); + + expect( + calls + .map((call) => call.type) + .filter((type) => type === 'user' || type === 'globals'), + ).toEqual(['user', 'globals']); + }); + + it('(c) state deliveries coalesce per type while waiting: one entry, the latest value', async () => { + const { lines, logger } = errors(); + const { calls, destination } = recorder({ consent: { marketing: true } }); + const { elb, collector } = await startFlow({ + logger, + destinations: { d: { code: destination } }, + }); + + for (let i = 0; i < 50; i++) await elb('walker user', { id: `u${i}` }); + + const queued = collector.destinations.d.queueOn ?? []; + expect(queued.filter((entry) => entry.type === 'user')).toEqual([ + { type: 'user', data: { id: 'u49' } }, + ]); + + await elb('walker consent', { marketing: true }); + + const userCalls = calls.filter((call) => call.type === 'user'); + expect(userCalls).toEqual([{ type: 'user', data: { id: 'u49' } }]); + expect(lines).toHaveLength(0); + expect(collector.status.dropped['destination.d']).toBeUndefined(); + }); + + it('(d) lifecycle deliveries beyond queueMax drop the oldest with one ERROR', async () => { + const { lines, logger } = errors(); + const { destination } = recorder({ + consent: { marketing: true }, + queueMax: 2, + }); + const { elb, collector } = await startFlow({ + logger, + destinations: { d: { code: destination } }, + }); + + // The run delivery is queued already; three session deliveries follow. + await elb('walker session'); + await elb('walker session'); + await elb('walker session'); + + expect(collector.destinations.d.queueOn).toHaveLength(2); + expect(collector.status.dropped['destination.d'].queue).toBe(2); + expect(lines).toHaveLength(1); + expect(lines[0]).toContain('destination.queueOn full (queueMax 2)'); + }); +}); diff --git a/packages/collector/src/__tests__/outcome.test.ts b/packages/collector/src/__tests__/outcome.test.ts new file mode 100644 index 000000000..f73e73928 --- /dev/null +++ b/packages/collector/src/__tests__/outcome.test.ts @@ -0,0 +1,63 @@ +import { Level, StepError } from '@walkeros/core'; +import type { Destination, Logger } from '@walkeros/core'; +import { startFlow } from '..'; + +function capture() { + const errors: Array<{ message: string; context: Logger.LogContext }> = []; + const logger: Logger.Config = { + level: 'DEBUG', + handler: (level, message, context) => { + if (level === Level.ERROR) errors.push({ message, context }); + }, + }; + return { errors, logger }; +} + +const failing: Destination.Instance = { + type: 'probe', + config: {}, + push: async () => { + throw new StepError('vendor said no', { + code: 'transport', + status: 503, + body: { email: 'a@b.c' }, + }); + }, +}; + +describe('outcome sink', () => { + it('reports a thrown StepError as failed with a reason, DLQ, counter and one ERROR line', async () => { + const { errors, logger } = capture(); + const { elb, collector } = await startFlow({ + destinations: { probe: { code: failing } }, + logger, + }); + const r1 = await elb('page view'); + await elb('page view'); + expect(r1.ok).toBe(false); + expect(r1.failed?.probe.reason).toEqual({ + code: 'transport', + message: 'vendor said no', + retryable: true, + status: 503, + }); + expect(collector.status.destinations.probe.failed).toBe(2); + expect(collector.destinations.probe.dlq).toHaveLength(2); + expect(errors).toHaveLength(1); + expect(JSON.stringify(errors)).not.toContain('a@b.c'); + }); + + it('flushes the suppressed count through the logger at shutdown', async () => { + const { errors, logger } = capture(); + const { elb, collector } = await startFlow({ + destinations: { probe: { code: failing } }, + logger, + }); + expect(collector.causeLog).toBeDefined(); + await elb('page view'); + await elb('page view'); + await elb('walker shutdown'); + expect(errors).toHaveLength(2); + expect(errors[1].context.suppressed).toBe(1); + }); +}); diff --git a/packages/collector/src/__tests__/prerun-hold.test.ts b/packages/collector/src/__tests__/prerun-hold.test.ts index 54537793b..845aee094 100644 --- a/packages/collector/src/__tests__/prerun-hold.test.ts +++ b/packages/collector/src/__tests__/prerun-hold.test.ts @@ -27,38 +27,43 @@ describe('pre-run hold buffer', () => { expect(collector.preRunQueue[0].event.name).toBe('page view'); }); - test('the buffer is bounded by queueMax with drop-oldest', async () => { + test('the buffer is bounded by queueMax and refuses the newest event', async () => { const { collector, elb } = await startFlow({ run: false, queueMax: 2, + logger: { handler: () => undefined }, }); await elb({ name: 'a one', data: {} }); await elb({ name: 'b two', data: {} }); - await elb({ name: 'c three', data: {} }); + const refused = await elb({ name: 'c three', data: {} }); + // The held events were promised already; the newest one is told. + expect(refused.ok).toBe(false); expect(collector.preRunQueue).toHaveLength(2); expect(collector.preRunQueue.map((h) => h.event.name)).toEqual([ + 'a one', 'b two', - 'c three', ]); }); - test('overflow is counted in status.dropped, separately from the replay buffer', async () => { - const { collector, elb } = await startFlow({ run: false, queueMax: 2 }); + test('a refused event is counted failed, never as a drop of a held one', async () => { + const { collector, elb } = await startFlow({ + run: false, + queueMax: 2, + logger: { handler: () => undefined }, + }); await elb({ name: 'a one', data: {} }); await elb({ name: 'b two', data: {} }); - // Third push evicts the oldest: one real event lost, so it must be counted - // and not merely debug-logged. await elb({ name: 'c three', data: {} }); - expect(collector.status.dropped['collector.preRun']?.queue).toBe(1); - // Kept apart from the post-run replay buffer's counter. + expect(collector.status.failed).toBe(1); + expect(collector.status.dropped['collector.preRun']).toBeUndefined(); expect(collector.status.dropped.collector?.queue).toBeUndefined(); await elb({ name: 'd four', data: {} }); - expect(collector.status.dropped['collector.preRun']?.queue).toBe(2); + expect(collector.status.failed).toBe(2); }); test('a failing replayed event surfaces in the run result', async () => { diff --git a/packages/collector/src/__tests__/queue-bounds.test.ts b/packages/collector/src/__tests__/queue-bounds.test.ts index c14d16247..d4d72ae23 100644 --- a/packages/collector/src/__tests__/queue-bounds.test.ts +++ b/packages/collector/src/__tests__/queue-bounds.test.ts @@ -131,14 +131,14 @@ describe('queue bounds', () => { // After pushToDestinations runs, queuePush carries the denied events // (consent not granted). Length must equal the capped queue size. expect(destination.queuePush?.length).toBe(3); - expect(destination.queuePush?.map((e) => e.name)).toEqual([ + expect(destination.queuePush?.map((e) => e.event.name)).toEqual([ 'event 2', 'event 3', 'event 4', ]); }); - test('warn-once-on-transition log emitted on overflow', async () => { + test('once-on-transition INFO log emitted on overflow', async () => { const collector = createTestCollector({ config: { queueMax: 2 } }); // Push 3 events; the third should overflow once. @@ -146,21 +146,22 @@ describe('queue bounds', () => { await pushToDestinations(collector, makeEvent('b')); await pushToDestinations(collector, makeEvent('c')); - // The scoped warn() should have been called for the queue overflow. - // We capture both root and scoped loggers via the mock. + // The replay buffer only feeds destinations added later: INFO, not + // WARN. We capture both root and scoped loggers via the mock. const scopedLogger = (collector.logger.scope as jest.Mock).mock.results[0] ?.value; - const allWarns = [ - ...(collector.logger.warn as jest.Mock).mock.calls, - ...((scopedLogger?.warn as jest.Mock | undefined)?.mock.calls ?? []), + const allInfos = [ + ...(collector.logger.info as jest.Mock).mock.calls, + ...((scopedLogger?.info as jest.Mock | undefined)?.mock.calls ?? []), ]; - const queueWarn = allWarns.find((call) => + const queueInfo = allInfos.find((call) => String(call[0]).toLowerCase().includes('queue'), ); - expect(queueWarn).toBeDefined(); + expect(queueInfo).toBeDefined(); + expect(collector.logger.warn).not.toHaveBeenCalled(); }); - test('warn suppressed on subsequent drops within same window', async () => { + test('log suppressed on subsequent drops within same window', async () => { const collector = createTestCollector({ config: { queueMax: 2 } }); for (let i = 0; i < 5; i++) { @@ -169,15 +170,15 @@ describe('queue bounds', () => { const scopedLogger = (collector.logger.scope as jest.Mock).mock.results[0] ?.value; - const allWarns = [ - ...(collector.logger.warn as jest.Mock).mock.calls, - ...((scopedLogger?.warn as jest.Mock | undefined)?.mock.calls ?? []), + const allInfos = [ + ...(collector.logger.info as jest.Mock).mock.calls, + ...((scopedLogger?.info as jest.Mock | undefined)?.mock.calls ?? []), ]; - const queueWarns = allWarns.filter((call) => + const queueInfos = allInfos.filter((call) => String(call[0]).toLowerCase().includes('queue'), ); // Three overflows but only the first transition logs. - expect(queueWarns.length).toBe(1); + expect(queueInfos.length).toBe(1); // Drop counter still tracks every dropped event. expect(collector.status.dropped[stepId('collector')]?.queue ?? 0).toBe(3); }); @@ -200,7 +201,7 @@ describe('queue bounds', () => { } expect(destination.queuePush?.length).toBe(3); - expect(destination.queuePush?.map((e) => e.name)).toEqual([ + expect(destination.queuePush?.map((e) => e.event.name)).toEqual([ 'event 2', 'event 3', 'event 4', diff --git a/packages/collector/src/__tests__/registration-checks.test.ts b/packages/collector/src/__tests__/registration-checks.test.ts new file mode 100644 index 000000000..5a77914ae --- /dev/null +++ b/packages/collector/src/__tests__/registration-checks.test.ts @@ -0,0 +1,156 @@ +import { Level } from '@walkeros/core'; +import type { + Collector, + Destination, + Logger, + Source, + Transformer, + WalkerOS, +} from '@walkeros/core'; +import { startFlow } from '..'; + +function errors() { + const lines: string[] = []; + const logger: Logger.Config = { + handler: (level, message) => { + if (level === Level.ERROR) lines.push(message); + }, + }; + return { lines, logger }; +} + +function sink(delivered: WalkerOS.Event[]): Destination.Instance { + return { + type: 'sink', + config: {}, + push: async (event) => void delivered.push(event), + }; +} + +const probe: Source.Init< + Source.Types +> = async (context) => ({ + type: 'probe', + config: context.config, + push: context.env.push, +}); + +const pass: Transformer.Instance = { + type: 'pass', + config: {}, + push: (event) => ({ event }), +}; + +afterEach(() => { + globalThis.__WALKEROS_VALIDATE__ = undefined; +}); + +describe('registration checks', () => { + it('(E3) a batch on a destination without pushBatch is one ERROR at registration', async () => { + const { lines, logger } = errors(); + const delivered: WalkerOS.Event[] = []; + const { elb } = await startFlow({ + logger, + destinations: { + d: { code: sink(delivered), config: { batch: { wait: 100 } } }, + }, + }); + expect(lines).toEqual([ + 'Destination d: `config.batch` has no effect: the destination has no pushBatch, so events go to push one by one', + ]); + + await elb('page view'); + await elb('page view'); + expect(lines).toHaveLength(1); + expect(delivered).toHaveLength(2); + }); + + it('an unknown transformer in a chain is one ERROR at registration, never per event', async () => { + const { lines, logger } = errors(); + const delivered: WalkerOS.Event[] = []; + const { elb } = await startFlow({ + logger, + next: ['missing', 'pass'], + transformers: { pass: { code: async () => pass } }, + destinations: { d: { code: sink(delivered) } }, + }); + expect(lines).toEqual([ + 'Transformer not found: missing (named in collector.next); the chain goes on without it', + ]); + + await elb('page view'); + await elb('page view'); + expect(lines).toHaveLength(1); + expect(delivered).toHaveLength(2); + }); +}); + +describe('mapping keys with no effect at their position (D4)', () => { + it('a destination rule extend and a source rule data: one ERROR each, both still run', async () => { + const { lines, logger } = errors(); + const delivered: WalkerOS.Event[] = []; + const { elb } = await startFlow({ + logger, + sources: { + probe: { + code: probe, + config: { + mapping: { page: { view: { data: { map: { id: 'data.id' } } } } }, + }, + }, + }, + destinations: { + d: { + code: sink(delivered), + config: { + mapping: { page: { view: { name: 'pv', extend: { name: 'x' } } } }, + }, + }, + }, + }); + + expect(lines).toEqual([ + "Destination d: `config.mapping.page.view.extend` does nothing at the destination position; a rule's `extend` is merged only by a package onto its own shipped rules; write the fields on the rule itself.", + "Source probe: `config.mapping.page.view.data` does nothing at the source position; only `consent`, `policy` and a rule's `condition`, `policy`, `name` and `ignore` apply.", + ]); + + await elb({ name: 'page view', data: { id: 1 } }); + expect(delivered).toHaveLength(1); + expect(delivered[0].name).toBe('pv'); + }); + + it('a transformer no-op keeps its text and is now an ERROR', async () => { + const { lines, logger } = errors(); + await startFlow({ + logger, + transformers: { + t: { config: { mapping: { data: 'data' } } }, + }, + }); + expect(lines).toEqual([ + "Transformer t: `config.mapping.data` does nothing at the transformer position; only `policy` and a rule's `condition`, `policy`, `name` and `ignore` apply.", + ]); + }); + + it('with __WALKEROS_VALIDATE__ false nothing is checked', async () => { + globalThis.__WALKEROS_VALIDATE__ = false; + const { lines, logger } = errors(); + await startFlow({ + logger, + sources: { + probe: { + code: probe, + config: { mapping: { page: { view: { data: 'data' } } } }, + }, + }, + transformers: { t: { config: { mapping: { data: 'data' } } } }, + destinations: { + d: { + code: sink([]), + config: { mapping: { page: { view: { extend: { name: 'x' } } } } }, + }, + }, + }); + expect(lines).toEqual([]); + }); +}); diff --git a/packages/collector/src/__tests__/report-error.test.ts b/packages/collector/src/__tests__/report-error.test.ts index 340d0756d..71ab78b80 100644 --- a/packages/collector/src/__tests__/report-error.test.ts +++ b/packages/collector/src/__tests__/report-error.test.ts @@ -179,30 +179,41 @@ describe('reportError', () => { expect(c.status.failed).toBe(1); }); - test('event-bearing log meta carries the transport code when present', async () => { - const c = await collector({}); - const logger = createMockLogger(); + test('the event-bearing line goes through the cause log: one line per cause, no payload', async () => { + const lines: Array<{ message: string; context: unknown }> = []; + const c = await collector({ + logger: { + handler: (_level, message, context) => { + lines.push({ message, context }); + }, + }, + }); const dlq: Destination.DLQ = []; const reportError = buildReportError( c, 'destination', 'bigquery', - logger, + createMockLogger(), makeDestination(dlq), ); - const err: Error & { code?: number } = Object.assign( - new Error('append failed'), - { code: 14 }, - ); - - reportError(err, event); - - expect(logger.error).toHaveBeenCalledWith('report error', { - error: 'append failed', - code: 14, - event: event.name, - }); + reportError(new Error('append failed'), event); + reportError(new Error('append failed'), event); + + expect(lines).toEqual([ + { + message: + 'report error (repeats within a minute are counted in status.destinations.bigquery.failed)', + context: { + step: 'destination.bigquery', + code: 'transport', + retryable: true, + event: event.name, + error: 'append failed', + }, + }, + ]); + expect(c.status.destinations.bigquery.failed).toBe(2); }); }); diff --git a/packages/collector/src/__tests__/step-state.test.ts b/packages/collector/src/__tests__/step-state.test.ts index 9945b4917..f135e5415 100644 --- a/packages/collector/src/__tests__/step-state.test.ts +++ b/packages/collector/src/__tests__/step-state.test.ts @@ -118,7 +118,12 @@ describe('store records', () => { const error = states.find((s) => s.phase === 'error'); expect(error?.stepId).toBe('store.kv'); expect(error?.meta).toEqual({ op: 'get', key: 'k' }); - expect(error?.error).toEqual({ name: 'TypeError', message: 'store down' }); + expect(error?.error).toEqual({ + name: 'TypeError', + message: 'store down', + code: 'transport', + retryable: true, + }); expect(typeof error?.durationMs).toBe('number'); }); }); @@ -149,19 +154,25 @@ describe('error projection', () => { )?.error; } - it('keeps the name of an Error', async () => { + it('keeps the name of an Error, with the failure code of the position', async () => { class VendorError extends Error { name = 'VendorError'; } expect(await errorOf(new VendorError('rejected'))).toEqual({ name: 'VendorError', message: 'rejected', + code: 'transformer', + retryable: false, }); }); it('has no name for a non-Error throw', async () => { const error = await errorOf('boom'); - expect(error).toEqual({ message: 'boom' }); + expect(error).toEqual({ + message: 'boom', + code: 'transformer', + retryable: false, + }); expect(error).not.toHaveProperty('name'); }); }); diff --git a/packages/collector/src/__tests__/store-read-failure.test.ts b/packages/collector/src/__tests__/store-read-failure.test.ts new file mode 100644 index 000000000..c73d55da8 --- /dev/null +++ b/packages/collector/src/__tests__/store-read-failure.test.ts @@ -0,0 +1,152 @@ +import { createMockLogger, Level, StepError } from '@walkeros/core'; +import type { Destination, Logger, Store, WalkerOS } from '@walkeros/core'; +import { startFlow } from '..'; +import { createCacheStore } from '../cache-store'; +import { wrapStoreWithCache } from '../store-cache-wrapper'; + +/** A store whose reads are refused (403), and whose writes are kept. */ +function refusing(written: Map): Store.Instance { + return { + type: 'refusing', + config: {}, + get: async () => { + throw new StepError('HTTP 403', { code: 'transport', status: 403 }); + }, + set: async (key: string, value: Store.StoreValue) => { + written.set(key, value); + }, + delete: async () => undefined, + }; +} + +function refusingStore(written: Map): Store.Init { + return () => refusing(written); +} + +function errors() { + const lines: Array<{ message: string; context: Logger.LogContext }> = []; + const logger: Logger.Config = { + handler: (level, message, context) => { + if (level === Level.ERROR) lines.push({ message, context }); + }, + }; + return { lines, logger }; +} + +function sink(delivered: WalkerOS.Event[]): Destination.Instance { + return { + type: 'sink', + config: {}, + push: async (event) => void delivered.push(event), + }; +} + +describe('failed store reads (ruling 10)', () => { + it('(a) a state get that throws delivers without the enrichment, one ERROR per cause', async () => { + const { lines, logger } = errors(); + const delivered: WalkerOS.Event[] = []; + const { elb } = await startFlow({ + logger, + stores: { s: { code: refusingStore(new Map()) } }, + destinations: { + d: { + code: sink(delivered), + state: { + mode: 'get', + store: 's', + key: 'event.name', + value: 'event.data.enriched', + }, + }, + }, + }); + + const first = await elb('page view', { id: 1 }); + await elb('page view', { id: 2 }); + + expect(first.ok).toBe(true); + expect(delivered).toHaveLength(2); + expect(delivered[0].data).toEqual({ id: 1 }); + expect(lines).toHaveLength(1); + expect(lines[0].context).toMatchObject({ + step: 'store.s', + code: 'transport', + status: 403, + }); + }); + + it('(b) a step cache backed by that store treats the throw as a miss', async () => { + const { lines, logger } = errors(); + const delivered: WalkerOS.Event[] = []; + const { elb } = await startFlow({ + logger, + stores: { s: { code: refusingStore(new Map()) } }, + destinations: { + d: { + code: sink(delivered), + cache: { + store: 's', + rules: [{ key: ['event.name'], ttl: 60 }], + }, + }, + }, + }); + + const r = await elb('page view'); + await elb('page view'); + + expect(r.ok).toBe(true); + expect(delivered).toHaveLength(2); + expect(lines).toHaveLength(1); + expect(lines[0].context).toMatchObject({ + step: 'store.s', + code: 'transport', + status: 403, + }); + }); + + it('(c) a state merge whose read throws skips its write and reports once', async () => { + const { lines, logger } = errors(); + const written = new Map(); + const { elb } = await startFlow({ + logger, + stores: { s: { code: refusingStore(written) } }, + destinations: { + d: { + code: sink([]), + state: { + mode: 'set', + store: 's', + key: 'event.name', + value: 'event.data', + mapping: { map: { last: 'id' } }, + }, + }, + }, + }); + + await elb('page view', { id: 'a' }); + await elb('page view', { id: 'b' }); + + expect(written.size).toBe(0); + expect(lines).toHaveLength(1); + expect(lines[0].context).toMatchObject({ step: 'store.s', status: 403 }); + }); + + it('(d) the read-through wrapper passes a backing throw on, caches nothing, logs nothing', async () => { + const logger = createMockLogger(); + const cacheStore = createCacheStore(); + const wrapped = wrapStoreWithCache(refusing(new Map()), { + storeId: 's', + cacheConfig: { rules: [{ ttl: 60 }] }, + cacheStore, + namespace: 's', + logger, + }); + + await expect(wrapped.get('k')).rejects.toThrow('HTTP 403'); + expect(cacheStore.get('s:k')).toBeUndefined(); + expect(logger.error).not.toHaveBeenCalled(); + expect(logger.warn).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/collector/src/__tests__/transformer-branch.test.ts b/packages/collector/src/__tests__/transformer-branch.test.ts index ebcabb541..60b892a22 100644 --- a/packages/collector/src/__tests__/transformer-branch.test.ts +++ b/packages/collector/src/__tests__/transformer-branch.test.ts @@ -348,11 +348,11 @@ describe('chain branching', () => { undefined, ); - // One rule for every unknown id in a route: warn and continue with the - // event the router returned. + // One rule for every unknown id in a route: one ERROR per id, and the + // copy continues with the event the router returned. expect(result.copies[0].event).toEqual({}); - expect(collector.logger.warn).toHaveBeenCalledWith( - 'Transformer not found: nonexistent-parser', + expect(collector.logger.error).toHaveBeenCalledWith( + 'Transformer not found: nonexistent-parser (named in a chain); the chain goes on without it', ); }); diff --git a/packages/collector/src/__tests__/transformer-init-error.test.ts b/packages/collector/src/__tests__/transformer-init-error.test.ts index 1b9e17c1d..5c100001f 100644 --- a/packages/collector/src/__tests__/transformer-init-error.test.ts +++ b/packages/collector/src/__tests__/transformer-init-error.test.ts @@ -1,13 +1,19 @@ -import type { Collector, Transformer, WalkerOS } from '@walkeros/core'; -import { createMockLogger } from '@walkeros/core'; +import type { + Collector, + Destination, + Transformer, + WalkerOS, +} from '@walkeros/core'; +import { createMockLogger, StepError } from '@walkeros/core'; import { runTransformerChain } from '../transformer'; +import { startFlow } from '..'; /** * Transformer init error visibility: when transformerInit throws inside - * runTransformerChain, the outer tryCatchAsync wrap MUST log the original - * cause via the scoped logger AND increment `collector.status.failed` - * (Category A - internal pipeline failure). Downstream behavior is - * unchanged: the chain stops, the event is dropped. + * runTransformerChain, the cause is reported once (cause line under the + * transformer's step) AND `collector.status.failed` increments. The chain + * stops for that copy and the failure travels in the chain result, so the + * push answers `ok: false` instead of reading as a drop. */ function createTestCollector( @@ -81,11 +87,15 @@ describe('transformer init throws (Category A)', () => { // Pipeline failure counts. expect(collector.status.failed).toBe(1); + expect(result.failure).toEqual({ + by: 'bad', + reason: { code: 'transformer', message: 'init boom', retryable: false }, + }); + const errorCall = findLoggerError(collector, 'transformer init failed'); - expect(errorCall).toBeDefined(); expect(errorCall?.[1]).toEqual( expect.objectContaining({ - transformer: 'bad', + step: 'transformer.bad', error: 'init boom', }), ); @@ -109,13 +119,76 @@ describe('transformer init throws (Category A)', () => { const context = findLoggerError(collector, 'transformer init failed')?.[1]; - expect(context).toEqual({ transformer: 'bad', error: 'init boom' }); + expect(context).toEqual({ + step: 'transformer.bad', + code: 'transformer', + type: 'mock', + retryable: false, + error: 'init boom', + }); // A raw Error serializes to `"error":{}`, blanking the cause in the // output an operator reads. expect(JSON.stringify(context)).toContain('init boom'); }); }); +function sink(delivered: WalkerOS.Event[]): Destination.Instance { + return { + type: 'sink', + config: {}, + push: async (event) => { + delivered.push(event); + }, + }; +} + +describe('transformer init contract (C7)', () => { + it('an init returning false at runtime fails the push instead of stopping it silently', async () => { + const delivered: WalkerOS.Event[] = []; + const initFalse = (): false => false; + const enrich: Transformer.Instance = { + type: 'enrich', + config: {}, + init: initFalse, + push: (event) => ({ event }), + }; + const { elb } = await startFlow({ + next: 'enrich', + transformers: { enrich: { code: async () => enrich } }, + destinations: { sink: { code: sink(delivered) } }, + }); + const r = await elb('page view'); + expect(r.ok).toBe(false); + expect(r.dropped).toBeUndefined(); + expect(r.reason?.code).toBe('not-ready'); + expect(r.error).toContain('enrich'); + expect(delivered).toHaveLength(0); + }); + + it('an init throwing a StepError fails the push with its reason', async () => { + const delivered: WalkerOS.Event[] = []; + const enrich: Transformer.Instance = { + type: 'enrich', + config: {}, + init: () => { + throw new StepError('Config settings salt missing', { + code: 'config', + }); + }, + push: (event) => ({ event }), + }; + const { elb } = await startFlow({ + next: 'enrich', + transformers: { enrich: { code: async () => enrich } }, + destinations: { sink: { code: sink(delivered) } }, + }); + const r = await elb('page view'); + expect(r.ok).toBe(false); + expect(r.reason?.code).toBe('config'); + expect(delivered).toHaveLength(0); + }); +}); + // --- helpers --- function findLoggerError( diff --git a/packages/collector/src/__tests__/transformer-mapping.test.ts b/packages/collector/src/__tests__/transformer-mapping.test.ts index f550ba564..fbc99fc08 100644 --- a/packages/collector/src/__tests__/transformer-mapping.test.ts +++ b/packages/collector/src/__tests__/transformer-mapping.test.ts @@ -144,13 +144,13 @@ describe('mapping-only transformer steps', () => { }); }); -describe('transformer mapping init warnings', () => { +describe('transformer mapping init notices (ERROR: config without effect)', () => { function collectWarnings() { const warnings: string[] = []; const handler: Logger.Handler = (level, message) => { - if (level === Level.WARN) warnings.push(message); + if (level === Level.ERROR) warnings.push(message); }; - return { warnings, logger: { level: 'WARN' as const, handler } }; + return { warnings, logger: { level: 'ERROR' as const, handler } }; } it('names every field that does nothing at the transformer position, array rules included', async () => { diff --git a/packages/collector/src/buffers.ts b/packages/collector/src/buffers.ts index 9e51305e2..3d05a0d71 100644 --- a/packages/collector/src/buffers.ts +++ b/packages/collector/src/buffers.ts @@ -15,10 +15,16 @@ import type { Logger } from '@walkeros/core'; * * Caller responsibility: increment the appropriate status counter * under `status.dropped[stepId]` (see `stepId()` in @walkeros/core) - * and emit the warn-once-on-transition log. The helper stays pure: no logger + * and emit the once-on-transition log. The helper stays pure: no logger * coupling, no collector reference. */ +/** + * Default bound of a destination's `queuePush` and of its queued `on()` + * lifecycle deliveries. + */ +export const DEFAULT_QUEUE_MAX = 1_000; + /** * The collector queue bound. `collector()` seeds `config.queueMax`, so a * missing one is a wiring bug, never a user setting. @@ -98,22 +104,25 @@ export function pushBounded( const overflowState = new WeakMap(); /** - * Emit a warn-once log on the transition into overflow. Subsequent drops - * within the same overflow window stay silent until the buffer drains - * (the caller signals "below-cap again" by calling `resetOverflow`). + * Emit a log once on the transition into overflow, at the buffer's level: + * INFO for the replay buffer (no present destination loses an event), DEBUG + * for the consent wait (unmet consent is never above DEBUG). Subsequent + * drops within the same overflow window stay silent until the buffer drains + * (the caller signals "below-cap again" by calling `resetOverflowFlag`). * * The cumulative drop counter is still incremented on every drop by the * caller, so operators retain rate signal via `collector.status.dropped[stepId]`. */ -export function warnOverflowOnce( +export function logOverflowOnce( buffer: object, logger: Logger.Instance, + level: 'info' | 'debug', message: string, context: Record, ): void { if (overflowState.get(buffer)) return; overflowState.set(buffer, true); - logger.warn(message, context); + logger[level](message, context); } /** diff --git a/packages/collector/src/collector-next.ts b/packages/collector/src/collector-next.ts index 59495aa9b..96c00c6aa 100644 --- a/packages/collector/src/collector-next.ts +++ b/packages/collector/src/collector-next.ts @@ -26,6 +26,8 @@ export interface CollectorNextResult { respond?: RespondFn; /** Set when a route stop or a transformer ended every copy. */ dropped?: true; + /** A transformer of the chain failed: the push fails with its reason. */ + failure?: Transformer.ChainFailure; } /** @@ -61,6 +63,11 @@ export async function runCollectorNext( COLLECTOR_NEXT_PATH, ); const respond = result.respond ?? meta.respond; + const failure = result.failure; + + // A transformer failed and no copy goes on: not a drop, the push fails. + if (failure && result.copies.length === 0) + return { copies: [], respond, failure }; if (result.copies.length === 0) { collector.logger.debug( @@ -88,5 +95,6 @@ export async function runCollectorNext( ingest: copy.ingest, })), respond, + ...(failure ? { failure } : {}), }; } diff --git a/packages/collector/src/collector.ts b/packages/collector/src/collector.ts index 51ad0a312..048b38fdc 100644 --- a/packages/collector/src/collector.ts +++ b/packages/collector/src/collector.ts @@ -1,8 +1,8 @@ import type { Collector, Elb, Logger, WalkerOS } from '@walkeros/core'; -import { assign, createLogger } from '@walkeros/core'; +import { assign, createCauseLog, createLogger } from '@walkeros/core'; import { commonHandleCommand, prepareEvent } from './handle'; import { initDestinations } from './destination'; -import { initTransformers } from './transformer'; +import { checkChainTargets, initTransformers } from './transformer'; import { createPush } from './push'; import { createCommand } from './command'; import { createElb } from './elb'; @@ -18,6 +18,7 @@ export async function collector( sessionStatic: {}, run: true, queueMax: 1_000, + consentDenied: 'queue', }; const config: Collector.Config = assign(defaultConfig, initConfig, { @@ -47,6 +48,8 @@ export async function collector( hooks: initConfig.hooks || {}, observers: new Set(), logger, + // The one cause log: every failure line, one per cause per minute. + causeLog: createCauseLog(logger), on: {}, queue: [], preRunQueue: [], @@ -113,7 +116,7 @@ export async function collector( // these raw references with the actual Store.Instance objects. resolveStoreReferences(rawStores, collector.stores, initConfig); } else if (Object.keys(rawStores).length > 0) { - logger.warn('stores: not in this build, config ignored'); + logger.error('stores: not in this build, config ignored'); } // Create default cache store for steps that use cache without explicit store. @@ -136,5 +139,33 @@ export async function collector( initConfig.transformers || {}, ); + // Registration check of every chain known now: each transformer it names + // must exist (one ERROR per missing id, not one per event). + checkChainTargets(collector, 'collector.next', config.next); + for (const [id, destination] of Object.entries(collector.destinations)) { + checkChainTargets( + collector, + `destination.${id}.before`, + destination.config.before, + ); + checkChainTargets( + collector, + `destination.${id}.next`, + destination.config.next, + ); + } + for (const [id, transformer] of Object.entries(collector.transformers)) { + checkChainTargets( + collector, + `transformer.${id}.before`, + transformer.config.before, + ); + checkChainTargets( + collector, + `transformer.${id}.next`, + transformer.config.next, + ); + } + return collector; } diff --git a/packages/collector/src/constants.ts b/packages/collector/src/constants.ts index dedc32a9e..be820308c 100644 --- a/packages/collector/src/constants.ts +++ b/packages/collector/src/constants.ts @@ -10,6 +10,7 @@ export const Commands: Record = { Custom: 'custom', Destination: 'destination', Elb: 'elb', + Flush: 'flush', Globals: 'globals', Hook: 'hook', Init: 'init', diff --git a/packages/collector/src/destination-outcome-state.ts b/packages/collector/src/destination-outcome-state.ts new file mode 100644 index 000000000..7d7b02151 --- /dev/null +++ b/packages/collector/src/destination-outcome-state.ts @@ -0,0 +1,88 @@ +import type { Collector, Destination, QueuedReason } from '@walkeros/core'; +import { StepError, stepId } from '@walkeros/core'; +import { getStateHold } from './on'; +import { isBreakerOpen, resolveBreakerConfig } from './breaker'; + +/** + * O3: the state of a destination for one push, decided here and nowhere + * else. Ready: deliver. Waiting: `queued` with a reason, the events are kept + * and delivered when the condition clears. Blocked: `failed` with a reason, + * through the one outcome sink. Consent is decided per event before any + * blocked classification (a consent-denied event only ever waits), so the + * callers ask `heldState` and `breakerState` for admitted events only. + */ +export type DestinationState = + | { state: 'ready' } + | { state: 'waiting'; reason: QueuedReason } + | { state: 'blocked'; error: StepError }; + +/** Waiting for consent: the event's required consent is not granted. */ +export const CONSENT_WAIT: QueuedReason = { + code: 'consent', + message: 'required consent is not granted', +}; + +/** Waiting in a batch window until it flushes. */ +export const BATCH_WAIT: QueuedReason = { + code: 'batch', + message: 'waiting in a batch window until it flushes', +}; + +/** Waiting while a state delivery to the destination runs or failed. */ +export const HELD_WAIT: QueuedReason = { + code: 'held', + message: 'a state delivery to the destination is running or failed', +}; + +const READY: DestinationState = { state: 'ready' }; + +/** + * A held destination waits while a state delivery to it runs, and for one + * retry window after a delivery failed; then it is blocked until a retry + * succeeds: its events fail instead of parking for the page lifetime. + */ +export function heldState( + destination: Destination.Instance, + timeout: number, + now = Date.now(), +): DestinationState { + const hold = getStateHold(destination); + if (!hold) return READY; + if (hold.failedAt === undefined || now - hold.failedAt < timeout) + return { state: 'waiting', reason: HELD_WAIT }; + return { + state: 'blocked', + error: new StepError( + `destination blocked: its ${String(hold.type)} delivery failed and was not recovered within ${timeout}ms; events fail until a retry succeeds`, + { code: 'blocked' }, + ), + }; +} + +/** + * While a destination's breaker is open its events fail with + * `breaker-open`. Presence-gated: ready unless `config.breaker` is set. A + * half-open breaker admits this push as its single probe (the caller settles + * it). + */ +export function breakerState( + collector: Collector.Instance, + destination: Destination.Instance, + canonicalId: string, +): DestinationState { + const config = resolveBreakerConfig(destination.config.breaker); + if (!config) return READY; + const key = stepId('destination', canonicalId); + if (!isBreakerOpen(collector.status.breakers, key, config.cooldown)) + return READY; + const openUntil = collector.status.breakers[key]?.openUntil; + return { + state: 'blocked', + error: new StepError( + `breaker open after ${config.threshold} consecutive failures; events fail until ${ + openUntil ? new Date(openUntil).toISOString() : 'a probe succeeds' + }; see the first failure's line`, + { code: 'breaker-open' }, + ), + }; +} diff --git a/packages/collector/src/destination.ts b/packages/collector/src/destination.ts index bf8870df2..a26ecee9e 100644 --- a/packages/collector/src/destination.ts +++ b/packages/collector/src/destination.ts @@ -4,6 +4,8 @@ import type { WalkerOS, Elb, Destination, + FailureReason, + QueuedReason, Transformer, Ingest, RespondFn, @@ -21,16 +23,28 @@ import { emitStep, getId, getGrantedConsent, + getMappingNoOpMessage, + getMappingNoOpPaths, + isArray, isDefined, isFunction, isObject, OBSERVE_ENV_KEY, processEventMapping, + StepError, stepId, tryCatchAsync, useHooks, } from '@walkeros/core'; -import { journeyFields, stepError, stepState } from './observerEmit'; +import { journeyFields, observerStepError, stepState } from './observerEmit'; +import { isTransportFailure, recordFailure, recordSkip } from './outcome'; +import { + BATCH_WAIT, + CONSENT_WAIT, + HELD_WAIT, + breakerState, + heldState, +} from './destination-outcome-state'; import { compileStepState, runStepState, warnStateOff } from './state'; import { wrapEnv } from './wrapEnv'; import { attachCalls } from './sanitizeArgs'; @@ -48,31 +62,32 @@ import { } from './on'; import type { DestinationDelivery } from './on'; import { resolveDestinationTimeout, withTimeout } from './timeout'; -import { runTransformerChain, extractChainProperty } from './transformer'; +import { + checkChainTargets, + runTransformerChain, + extractChainProperty, +} from './transformer'; import { getCacheStore } from './cache'; import { + DEFAULT_QUEUE_MAX, + logOverflowOnce, pushBounded, resetOverflowFlag, seededQueueMax, - warnOverflowOnce, } from './buffers'; import { - DEFAULT_DLQ_MAX, bumpDropped, ensureDestStatus, buildReportError, - errorMeta, } from './report-error'; import { reconcilePending } from './pending'; import { runCollectorNext } from './collector-next'; import { - isBreakerOpen, recordStepOutcome, releaseProbe, resolveBreakerConfig, } from './breaker'; -const DEFAULT_QUEUE_MAX = 1_000; /** Default upper-bound on entries per batch. Caps unbounded growth under sustained load. */ const DEFAULT_BATCH_SIZE = 1_000; /** Default upper-bound on batch age in ms. Forces flush even if debounce keeps resetting. */ @@ -92,6 +107,17 @@ function isBatchedResult(value: unknown): boolean { return value === BATCHED_RESULT; } +/** + * Sentinel returned by {@link destinationPush} when a mapping rule with + * `ignore: true` matched: the event was deliberately not sent (a skip, never + * `done`). + */ +const IGNORED_RESULT: { ignored: true } = Object.freeze({ ignored: true }); + +function isIgnoredResult(value: unknown): boolean { + return value === IGNORED_RESULT; +} + /** * Narrows a `pushBatch` return value to a {@link Destination.BatchOutcome}. * A destination returning `void` (the historical contract) yields whole-batch @@ -205,13 +231,14 @@ export async function addDestination( config, env: mergeEnvironments(code.env, env), }; + checkDestinationRegistration(collector, id, destination, true); // Add the destination collector.destinations[id] = destination; // Process previous events if not disabled if (destination.config.queue !== false) - destination.queuePush = [...collector.queue]; + destination.queuePush = collector.queue.map((event) => ({ event })); // A runtime destination catches up on state recorded before it existed, the // same way a pending destination does on activation, so its on() runs @@ -264,43 +291,71 @@ export async function settleDestinationDeliveries( } /** - * Fold the results of per-destination drains into a flush result, so a - * command reports every destination its events reached. A destination queued - * in the flush and done or failed after its drain is reported by its latest - * outcome. + * Fold push results into one, worst first: `ok` only if every result is ok, + * a destination failed in any of them is failed, otherwise a destination's + * latest outcome wins (a destination queued in a flush and done after its + * drain is done). A `reason` and an `error` from any result survive. The one + * merge for a command's drains and for every fan-out (source `before`, + * source `next`, `collector.next`). */ export function mergePushResults( base: Elb.PushResult, - drains: Elb.PushResult[], + others: Elb.PushResult[], ): Elb.PushResult { - if (!drains.length) return base; + if (!others.length) return base; const done = { ...(base.done ?? {}) }; const queued = { ...(base.queued ?? {}) }; const failed = { ...(base.failed ?? {}) }; - for (const drain of drains) { - for (const [id, ref] of Object.entries(drain.done ?? {})) { + let reason = base.reason; + let error = base.error; + for (const other of others) { + for (const [id, ref] of Object.entries(other.done ?? {})) { + if (failed[id]) continue; done[id] = ref; delete queued[id]; } - for (const [id, ref] of Object.entries(drain.failed ?? {})) { + for (const [id, ref] of Object.entries(other.failed ?? {})) { failed[id] = ref; + delete done[id]; delete queued[id]; } - Object.assign(queued, drain.queued ?? {}); + for (const [id, ref] of Object.entries(other.queued ?? {})) { + if (!failed[id]) queued[id] = ref; + } + reason = reason ?? other.reason; + error = error ?? other.error; } const merged: Elb.PushResult = { ...base, - ok: base.ok && drains.every((drain) => drain.ok), + ok: base.ok && others.every((other) => other.ok), }; delete merged.done; delete merged.queued; delete merged.failed; + delete merged.reason; + delete merged.error; if (Object.keys(done).length) merged.done = done; if (Object.keys(queued).length) merged.queued = queued; if (Object.keys(failed).length) merged.failed = failed; + if (reason) merged.reason = reason; + if (error !== undefined) merged.error = error; return merged; } +/** + * The push result of a chain whose transformer failed: `ok: false`, the + * reason, and an error naming the transformer. + */ +export function chainFailureResult( + failure: Transformer.ChainFailure, +): Elb.PushResult { + return createPushResult({ + ok: false, + reason: failure.reason, + error: `Transformer "${failure.by}" failed: ${failure.reason.message}`, + }); +} + /** * Pushes an event to all or a subset of destinations. * @@ -332,14 +387,18 @@ export async function pushToDestinations( return deliverToDestinations(collector, undefined, meta, destinations); const next = await runCollectorNext(collector, event, meta); + const failure = next.failure ? chainFailureResult(next.failure) : undefined; if (next.copies.length === 0) - return createPushResult({ - ok: true, - ...(next.dropped ? { dropped: true } : {}), - }); + return ( + failure ?? + createPushResult({ + ok: true, + ...(next.dropped ? { dropped: true } : {}), + }) + ); - // Each copy is delivered in order, as its own event with its own ingest. - // The first delivery's result is reported, as for a source-level fan-out. + // Each copy is delivered in order, as its own event with its own ingest, + // and every copy's outcome is reported (merged worst first). const results: Elb.PushResult[] = []; for (const copy of next.copies) { results.push( @@ -351,13 +410,38 @@ export async function pushToDestinations( ), ); } - return results[0]; + const [first, ...rest] = results; + return mergePushResults(first, failure ? [...rest, failure] : rest); +} + +/** What one destination did with one delivery pass. */ +interface DestinationOutcome { + id: string; + destination: Destination.Instance; + /** Nothing to report for this destination (disabled, nothing to send). */ + skipped?: true; + /** Why its events wait. */ + queued?: QueuedReason; + /** Why it failed; the failure is recorded already. */ + failed?: FailureReason; + error?: unknown; + response?: unknown; + totalDuration?: number; + /** Events delivered synchronously. */ + delivered?: number; + /** Events that entered a batch window. */ + batched?: number; } /** * Delivers one event (or, without one, the queued events only) to all or a * subset of destinations. Module-private: every caller goes through * `pushToDestinations`, so `collector.next` runs exactly once per event. + * + * Per destination and pass, one state (O3): ready (`done`), waiting + * (`queued` with reason `consent`, `batch` or `held`), blocked (`failed` + * with a reason, through the one outcome sink). Consent is decided first: + * a consent-denied event never reaches a failure path. */ async function deliverToDestinations( collector: Collector.Instance, @@ -371,7 +455,9 @@ async function deliverToDestinations( ): Promise { const { user } = collector; - // Add event to the collector queue (bounded; FIFO drop-oldest on overflow) + // Add event to the collector queue (bounded; FIFO drop-oldest on overflow). + // The replay buffer only feeds destinations added later, so an eviction + // loses nothing for a present destination: INFO. if (event) { const queueMax = seededQueueMax(collector.config.queueMax); const result = pushBounded(collector.queue, event, { max: queueMax }); @@ -382,9 +468,10 @@ async function deliverToDestinations( 'queue', result.dropped, ); - warnOverflowOnce( + logOverflowOnce( collector.queue, collector.logger, + 'info', 'collector.queue overflow; oldest events dropped', { buffer: 'queue', @@ -407,677 +494,793 @@ async function deliverToDestinations( const results = await Promise.all( // Process all destinations in parallel - Object.entries(destinations || {}).map(async ([id, destination]) => { - // Disabled destinations are completely skipped — no queuing, no init, no processing - if (destination.config.disabled) { - return { id, destination, skipped: true }; - } - - // Canonical id: the breaker gate, the failure/success accounting, and any - // aggregation MUST all key on the SAME id. A destination whose runtime - // `config.id` differs from its map key would otherwise touch two - // different breaker entries (gate vs accounting), so failures never - // accumulate against the entry the gate inspects and the breaker never - // opens. Resolve once here and thread it through the result. - const canonicalId = destination.config.id || id; - const breakerKey = stepId('destination', canonicalId); - const breakerConfig = resolveBreakerConfig(destination.config.breaker); - - // Circuit-breaker gate (same precedence as `config.disabled`): when the - // breaker is open, skip the event (counted as skipped, never pushed). - // Presence-gated: inert unless `config.breaker` is set. - if ( - breakerConfig && - isBreakerOpen( - collector.status.breakers, - breakerKey, - breakerConfig.cooldown, - ) - ) { - return { id, destination, skipped: true }; - } - - // Probe-settle helpers. When the gate above admitted a half-open probe, - // EVERY post-gate path must settle it: either record an outcome (the - // event reached the transport) or release the probe (it never did). - // Otherwise `probing` stays true and the breaker deadlocks half-open. - // Both are presence-gated and internally no-op unless half-open+probing. - const recordProbe = (outcome: 'transport-failure' | 'success') => { - if (breakerConfig) { - recordStepOutcome( - collector.status.breakers, - breakerKey, - outcome, - breakerConfig.threshold, - breakerConfig.cooldown, - ); + Object.entries(destinations || {}).map( + async ([id, destination]): Promise => { + // Disabled destinations are completely skipped — no queuing, no init, no processing + if (destination.config.disabled) { + return { id, destination, skipped: true }; } - }; - const releaseProbeSlot = () => { - if (breakerConfig) releaseProbe(collector.status.breakers, breakerKey); - }; - // Return events to this destination's own queue. Bounded; FIFO - // drop-oldest on overflow. - const requeue = (events: WalkerOS.Events) => { - if (!events.length) return; - if (!destination.queuePush) destination.queuePush = []; - const queuePush = destination.queuePush; - const destId = destination.config.id || id; - const bound = { - max: destination.config.queueMax ?? DEFAULT_QUEUE_MAX, + // Canonical id: the breaker gate, the failure/success accounting, and + // any aggregation MUST all key on the SAME id. A destination whose + // runtime `config.id` differs from its map key would otherwise touch + // two different breaker entries (gate vs accounting), so failures + // never accumulate against the entry the gate inspects and the + // breaker never opens. Resolve once here and thread it through. + const canonicalId = destination.config.id || id; + const breakerKey = stepId('destination', canonicalId); + const breakerConfig = resolveBreakerConfig(destination.config.breaker); + + // Probe-settle helpers. When the breaker gate below admitted a + // half-open probe, EVERY later path must settle it: either record an + // outcome (the event reached the transport) or release the probe (it + // never did). Otherwise `probing` stays true and the breaker + // deadlocks half-open. Both are presence-gated and internally no-op + // unless half-open+probing. + const recordProbe = (outcome: 'transport-failure' | 'success') => { + if (breakerConfig) { + recordStepOutcome( + collector.status.breakers, + breakerKey, + outcome, + breakerConfig.threshold, + breakerConfig.cooldown, + ); + } + }; + const releaseProbeSlot = () => { + if (breakerConfig) + releaseProbe(collector.status.breakers, breakerKey); }; - let totalDropped = 0; - for (const queued of events) { - const r = pushBounded(queuePush, queued, bound); - totalDropped += r.dropped; - } - if (totalDropped > 0) { - // Ensure status entry exists for early-overflow paths. - ensureDestStatus(collector, destId); - const droppedCount = bumpDropped( - collector.status, - stepId('destination', destId), - 'queue', - totalDropped, - ); - warnOverflowOnce( - queuePush, - collector.logger.scope(destination.type || 'unknown'), - 'destination.queuePush overflow; oldest events dropped', - { - buffer: 'queuePush', - destination: destId, - cap: bound.max, - droppedCount, - }, - ); - } else if (queuePush.length < bound.max) { - resetOverflowFlag(queuePush); - } - }; - - // A held destination retries its owed state before its push, so a - // handler that failed once and would settle now recovers without waiting - // for another state command. Rate-limited by the hold's own age to one - // attempt per timeout window, and never while a state delivery to it is - // running: a permanently hung handler costs one bounded wait per window, - // never one per event. The retry is inside this destination's own - // closure, so siblings are untouched. - // An initialized destination that is not held but owes state (a cell - // merged at `walker run` before the run's re-delivery reached it) - // receives that state before this push, the same way a held one - // retries, but without waiting for a window. - if ( - destination.on && - destination.config.init && - !getStateHold(destination) && - !isStateDeliveryInFlight(destination) && - owesDestinationState(collector, destination) - ) - await catchUpDestinationState(collector, destination, id); - const hold = getStateHold(destination); - if ( - hold && - !isStateDeliveryInFlight(destination) && - Date.now() - hold.since >= - resolveDestinationTimeout(destination.config.timeout) - ) - await catchUpDestinationState(collector, destination, id); - - // State hold: a state delivery to this destination is running or did - // not settle. Its events stay in its own queue, nothing is delivered, - // and the queue is not drained. Held, not dropped. - if (getStateHold(destination)) { - releaseProbeSlot(); // probe admitted but the destination is held - if (!event) return { id, destination, skipped: true }; - const held = [clone(event)]; - requeue(held); - return { id, destination, queue: held }; - } + const queueMax = destination.config.queueMax ?? DEFAULT_QUEUE_MAX; + const queueStep = stepId('destination', canonicalId); + const destLogger = collector.logger.scope( + destination.type || 'unknown', + ); - // Queued events: refresh consent (full replace — stale consent must not persist). - // User merge happens for all events below in allowedEvents.map. - let currentQueue = (destination.queuePush || []).map((event) => ({ - ...event, - consent: collector.consent, - })); - destination.queuePush = []; - - // Current event: added as-is (consent is already fresh from createEvent). - if (event) currentQueue.push(clone(event)); - - // Clone ingest for this destination (prevents cross-destination races in Promise.all) - const destIngest: Ingest = meta.ingest - ? { - ...meta.ingest, - _meta: { ...meta.ingest._meta, path: [...meta.ingest._meta.path] }, + // One bounded queue, two kinds of waiting entries. A consent wait is + // a documented bound: when the queue is full the oldest consent wait + // is evicted (`status.dropped`, DEBUG once: unmet consent is never + // above DEBUG), and a new consent wait meeting a queue of held events + // only is the one dropped. A held event is a delivery promise: it + // evicts the oldest consent wait, and with none left it is refused + // (returned, to fail with `queue-full`). Entries keep their identity; + // only their reason is set. The once-flag is keyed on the destination: + // its queue array is replaced on every pass. + const enqueue = ( + entries: Destination.Queued[], + reason: 'consent' | 'held', + ): Destination.Queued[] => { + const refused: Destination.Queued[] = []; + if (!entries.length) return refused; + if (!destination.queuePush) destination.queuePush = []; + const queuePush = destination.queuePush; + let evicted = 0; + for (const entry of entries) { + entry.reason = reason; + if (queuePush.length >= queueMax) { + const oldest = queuePush.findIndex( + (queued) => queued.reason === 'consent', + ); + if (oldest !== -1) { + queuePush.splice(oldest, 1); + evicted++; + } else if (reason === 'consent') { + evicted++; + continue; + } else { + refused.push(entry); + continue; + } + } + queuePush.push(entry); } - : createIngest('unknown'); - - // If no events and no queued on events, skip this destination - if (!currentQueue.length && !destination.queueOn?.length) { - releaseProbeSlot(); // probe admitted but no event to push - return { id, destination, skipped: true }; - } - - // If only on events queued (no push events), still init to flush queueOn. - // Direct try/catch + logger.error here is intentional. Previously this - // used tryCatchAsync(destinationInit) without an onError callback, which - // silently returned undefined when init threw, hiding real failures on - // the queueOn-only path. Mirrors the same pattern used below at the main - // init call site so failures are never silent on either branch. - if (!currentQueue.length && destination.queueOn?.length) { - // Consent gate (sole-gate invariant): this branch would init the - // destination purely to flush queued on() events, but there is no push - // event whose individual consent could apply, so collector consent is - // the complete basis. Never init a consent-gated destination while its - // required consent is denied. Self-heals: handle.ts runs - // pushToDestinations after every state command, so the grant command - // re-enters here with consent satisfied and inits then. - if (!getGrantedConsent(destination.config.consent, collector.consent)) { - releaseProbeSlot(); // probe admitted but consent gate denies push - return { id, destination, skipped: true }; - } - let isInitialized = false; - try { - isInitialized = await destinationInit( + if (evicted > 0) { + ensureDestStatus(collector, canonicalId); + const droppedCount = bumpDropped( + collector.status, + queueStep, + 'queue', + evicted, + ); + logOverflowOnce( + destination, + destLogger, + 'debug', + 'destination.queuePush overflow; oldest consent-waiting events dropped', + { + buffer: 'queuePush', + destination: canonicalId, + cap: queueMax, + droppedCount, + }, + ); + } else if (queuePush.length < queueMax) { + resetOverflowFlag(destination); + } + return refused; + }; + const requeue = (entries: Destination.Queued[]) => + void enqueue(entries, 'consent'); + const requeueHeld = (entries: Destination.Queued[]) => + enqueue(entries, 'held'); + + // Fail refused held entries through the one outcome sink. + const refuseHeld = (refused: Destination.Queued[]) => + recordFailure( collector, destination, id, - true, + refused.map((entry) => entry.event), + new StepError( + `destination.queuePush full (queueMax ${queueMax}) while the destination is held: newest event refused`, + { code: 'queue-full' }, + ), ); - } catch (err) { - collector.status.failed++; - const destType = destination.type || 'unknown'; - collector.logger.scope(destType).error('destination init failed', { - error: err instanceof Error ? err.message : String(err), - }); - // A probe whose init throws is a real transport failure: re-open. - recordProbe('transport-failure'); - } - // The flush held the destination: there are no events of this call - // to requeue, and any parked by a concurrent push stay queued. - if (getStateHold(destination)) { - releaseProbeSlot(); - return { id, destination, skipped: true }; - } - // Events parked by a concurrent push while the flush was running go - // out with this call, through the consent gate below. - if (isInitialized && destination.queuePush?.length) { - currentQueue = destination.queuePush.map((queued) => ({ - ...queued, - consent: collector.consent, - })); - destination.queuePush = []; - } else { - // queueOn-only flush exercises no push, so a successful/false-returning - // init releases the probe (the transport-failure path already re-opened - // it, where release is a no-op). - releaseProbeSlot(); - return { id, destination, skipped: !isInitialized }; - } - } - // Per-event consent gate: granted events keep only their granted - // states and are delivered, denied events emit a skip state so - // observers see the gate decision, and stay queued. The collector state - // is read at call time: this closure awaits, and a consent command can - // run in between. - const admit = (queue: WalkerOS.Events) => { - const consent = collector.consent; - const allowed: WalkerOS.Events = []; - const skipped = queue.filter((queuedEvent) => { - const grantedConsent = getGrantedConsent( - destination.config.consent, // Required - consent, // Current collector state - queuedEvent.consent, // Individual event state - ); + // Per-event consent gate: granted events keep only their granted + // states and are delivered; denied events emit a skip state so + // observers see the gate decision, and wait (or, under + // `consentDenied: 'skip'`, are skipped and counted). The collector + // state is read at call time: this closure awaits, and a consent + // command can run in between. + const consentDenied = collector.config.consentDenied ?? 'queue'; + const destIngest: Ingest = meta.ingest + ? { + ...meta.ingest, + _meta: { + ...meta.ingest._meta, + path: [...meta.ingest._meta.path], + }, + } + : createIngest('unknown'); + const admit = (queue: Destination.Queued[]) => { + const consent = collector.consent; + const allowed: Destination.Queued[] = []; + const denied: Destination.Queued[] = []; + for (const entry of queue) { + const queuedEvent = entry.event; + const grantedConsent = getGrantedConsent( + destination.config.consent, // Required + consent, // Current collector state + queuedEvent.consent, // Individual event state + ); - if (grantedConsent) { - queuedEvent.consent = grantedConsent; // Save granted consent states only + if (grantedConsent) { + queuedEvent.consent = grantedConsent; // Save granted consent states only + allowed.push(entry); + continue; + } + + const skipState = stepState( + collector, + stepId('destination', id), + 'destination', + 'skip', + typeof queuedEvent.id === 'string' ? queuedEvent.id : '', + Date.now(), + journeyFields(queuedEvent, destIngest, collector), + ); + skipState.skipReason = 'consent'; + if (consent) skipState.consent = { ...consent }; + if (destination.config.consent) { + skipState.meta = { + required: { ...destination.config.consent }, + }; + } + emitStep(collector, skipState); - allowed.push(queuedEvent); // Add to allowed queue - return false; // Remove from destination queue + if (consentDenied === 'skip') + recordSkip(collector, destination, id, queuedEvent, 'consent'); + else denied.push(entry); } + return { allowed, denied }; + }; - const skipState = stepState( - collector, - stepId('destination', id), - 'destination', - 'skip', - typeof queuedEvent.id === 'string' ? queuedEvent.id : '', - Date.now(), - journeyFields(queuedEvent, destIngest, collector), - ); - skipState.skipReason = 'consent'; - if (consent) skipState.consent = { ...consent }; - if (destination.config.consent) { - skipState.meta = { required: { ...destination.config.consent } }; + // A held destination retries its owed state before its push, so a + // handler that failed once and would settle now recovers without + // waiting for another state command. Rate-limited by the hold's own + // age to one attempt per timeout window, and never while a state + // delivery to it is running: a permanently hung handler costs one + // bounded wait per window, never one per event. The retry is inside + // this destination's own closure, so siblings are untouched. + // An initialized destination that is not held but owes state (a cell + // merged at `walker run` before the run's re-delivery reached it) + // receives that state before this push, the same way a held one + // retries, but without waiting for a window. + if ( + destination.on && + destination.config.init && + !getStateHold(destination) && + !isStateDeliveryInFlight(destination) && + owesDestinationState(collector, destination) + ) + await catchUpDestinationState(collector, destination, id); + + const timeout = resolveDestinationTimeout(destination.config.timeout); + const hold = getStateHold(destination); + if ( + hold && + !isStateDeliveryInFlight(destination) && + Date.now() - hold.since >= timeout + ) + await catchUpDestinationState(collector, destination, id); + + // State hold: a state delivery to this destination is running or did + // not settle. Consent first: a denied event waits for consent as + // anywhere else. A granted one is held in the destination's own queue + // for one retry window (`queued`, reason `held`); once the hold + // outlasts that window the destination is blocked: the event and the + // held ones before it fail (`blocked`) until a retry succeeds. + const held = heldState(destination, timeout); + if (held.state !== 'ready') { + if (!event) return { id, destination, skipped: true }; + const { allowed, denied } = admit([{ event: clone(event) }]); + requeue(denied); + if (!allowed.length) + return denied.length + ? { id, destination, queued: CONSENT_WAIT } + : { id, destination, skipped: true }; + + if (held.state === 'blocked') { + const heldBefore = (destination.queuePush || []).filter( + (queued) => queued.reason === 'held', + ); + destination.queuePush = (destination.queuePush || []).filter( + (queued) => queued.reason !== 'held', + ); + const error = held.error; + const failed = recordFailure( + collector, + destination, + id, + [...heldBefore, ...allowed].map((entry) => entry.event), + error, + ); + return { id, destination, failed, error }; } - emitStep(collector, skipState); - return true; // Keep denied events in the queue - }); - return { allowed, skipped }; - }; + const refused = requeueHeld(allowed); + for (const entry of allowed) { + if (refused.includes(entry)) continue; + const skipState = stepState( + collector, + stepId('destination', id), + 'destination', + 'skip', + typeof entry.event.id === 'string' ? entry.event.id : '', + Date.now(), + journeyFields(entry.event, destIngest, collector), + ); + skipState.skipReason = 'held'; + emitStep(collector, skipState); + } + if (refused.length) { + const failed = refuseHeld(refused); + return { id, destination, failed }; + } + return { id, destination, queued: HELD_WAIT }; + } - const admittedWith = collector.consent; - const { allowed: allowedEvents, skipped: skippedEvents } = - admit(currentQueue); + // Queued events: refresh consent (full replace — stale consent must + // not persist). Each entry keeps its identity; the event inside is + // copied. User merge happens for all events below. + let currentQueue: Destination.Queued[] = ( + destination.queuePush || [] + ).map((entry) => { + entry.event = { ...entry.event, consent: collector.consent }; + return entry; + }); + destination.queuePush = []; - // Add skipped (consent-denied) events back to the queue. - requeue(skippedEvents); + // Current event: added as-is (consent is already fresh from createEvent). + if (event) currentQueue.push({ event: clone(event) }); - // Execution shall not pass if no events are allowed - if (!allowedEvents.length) { - releaseProbeSlot(); // probe admitted but every event was re-queued - return { id, destination, queue: currentQueue }; // Don't push if not allowed - } + // If no events and no queued on events, skip this destination + if (!currentQueue.length && !destination.queueOn?.length) { + return { id, destination, skipped: true }; + } - // Events present in the queue before init: the consent-denied ones just - // requeued. Anything else found there after init was parked by a - // concurrent push while the init flush was running. - const parked = new Set(destination.queuePush); - - // Initialize the destination if needed. - // Direct try/catch + logger.error here is intentional. Previously this - // used tryCatchAsync(destinationInit) without an onError callback, which - // silently returned undefined when init threw, hiding real failures. The - // destination itself may also log a more specific error before throwing; - // this is the boundary safety net so failures are never silent. - // Allowed: at least one event cleared the per-event consent gate above, - // so initialization is authorized (true) for the chokepoint guard. - let isInitialized = false; - try { - isInitialized = await destinationInit(collector, destination, id, true); - } catch (err) { - collector.status.failed++; - const destType = destination.type || 'unknown'; - collector.logger.scope(destType).error('destination init failed', { - error: err instanceof Error ? err.message : String(err), - }); - // A probe whose init throws is a real transport failure (the down - // destination is exactly the one whose init throws): re-open with a - // fresh window so self-heal does not deadlock. - recordProbe('transport-failure'); - } + // If only on events are queued (no push events), still init to flush + // queueOn. An init that throws here loses no event: it is reported + // through the outcome sink with none. + if (!currentQueue.length && destination.queueOn?.length) { + // Consent gate (sole-gate invariant): this branch would init the + // destination purely to flush queued on() events, but there is no + // push event whose individual consent could apply, so collector + // consent is the complete basis. Never init a consent-gated + // destination while its required consent is denied. Self-heals: + // handle.ts runs pushToDestinations after every state command, so + // the grant command re-enters here with consent satisfied and inits + // then. + if ( + !getGrantedConsent(destination.config.consent, collector.consent) + ) { + return { id, destination, skipped: true }; + } + let isInitialized = false; + try { + isInitialized = await destinationInit( + collector, + destination, + id, + true, + ); + } catch (err) { + recordFailure(collector, destination, id, [], err, { + message: 'destination init failed', + }); + } + // The flush held the destination: there are no events of this call + // to requeue, and any parked by a concurrent push stay queued. + if (getStateHold(destination)) { + return { id, destination, skipped: true }; + } + // Events parked by a concurrent push while the flush was running go + // out with this call, through the consent gate below. + if (isInitialized && destination.queuePush?.length) { + currentQueue = destination.queuePush.map((entry) => { + entry.event = { ...entry.event, consent: collector.consent }; + return entry; + }); + destination.queuePush = []; + } else { + return { id, destination, skipped: true }; + } + } - if (!isInitialized) { - // Init returned false (no throw): the probe never pushed, so release it - // (no-op if the throw path above already re-opened). - releaseProbeSlot(); - return { id, destination, queue: currentQueue }; - } + const admittedWith = collector.consent; + const { allowed: allowedEvents, denied: deniedEvents } = + admit(currentQueue); - // The init flush held the destination: a state delivery did not settle. - // The allowed events go back to its queue; the denied ones are there - // already. - if (getStateHold(destination)) { - releaseProbeSlot(); - requeue(allowedEvents); - return { id, destination, queue: allowedEvents }; - } + // Denied events wait for consent in the destination's own queue. + requeue(deniedEvents); - // A consent command ran while init was awaited: the events admitted - // before it pass the gate again with the consent that holds now. - const deniedAfter: WalkerOS.Events = []; - if (collector.consent !== admittedWith) { - const regated = admit( - allowedEvents.map((queued) => ({ - ...queued, - consent: collector.consent, - })), - ); - allowedEvents.splice(0, allowedEvents.length, ...regated.allowed); - deniedAfter.push(...regated.skipped); - } + // Execution shall not pass if no events are allowed + if (!allowedEvents.length) { + return deniedEvents.length + ? { id, destination, queued: CONSENT_WAIT } + : { id, destination, skipped: true }; + } - // Events parked by a concurrent push while the init flush was running - // pass the same consent gate, with the consent that holds now, and go - // out with this call. - const queuedAfter = destination.queuePush || []; - const arrived = queuedAfter.filter((queued) => !parked.has(queued)); - if (arrived.length) { - destination.queuePush = queuedAfter.filter((queued) => - parked.has(queued), - ); - const late = admit( - arrived.map((queued) => ({ ...queued, consent: collector.consent })), - ); - allowedEvents.push(...late.allowed); - deniedAfter.push(...late.skipped); - } - requeue(deniedAfter); + // Circuit-breaker gate, after the consent decision: while the breaker + // is open, every admitted event fails with `breaker-open` (counted, + // DLQ, one cause line). Presence-gated: inert unless `config.breaker` + // is set. A half-open breaker admits this pass as its single probe. + const breaker = breakerState(collector, destination, canonicalId); + if (breaker.state === 'blocked') { + const error = breaker.error; + const failed = recordFailure( + collector, + destination, + id, + allowedEvents.map((entry) => entry.event), + error, + ); + return { id, destination, failed, error }; + } - if (!allowedEvents.length) { - releaseProbeSlot(); // every event was denied after init - return deniedAfter.length - ? { id, destination, queue: deniedAfter } - : { id, destination, skipped: true }; - } + // Events present in the queue before init: the consent-denied ones + // just requeued. Anything else found there after init was parked by + // a concurrent push while the init flush was running. + const parked = new Set(destination.queuePush); + + // Initialize the destination if needed. An init that throws, or + // returns false (runtime guard: `not-ready`), fails the admitted + // events through the outcome sink; init is re-attempted on the next + // push. Allowed: at least one event cleared the per-event consent + // gate above, so initialization is authorized (true) for the + // chokepoint guard. + try { + if (!(await destinationInit(collector, destination, id, true))) + throw new StepError('destination init was refused', { + code: 'not-ready', + }); + } catch (err) { + const failed = recordFailure( + collector, + destination, + id, + allowedEvents.map((entry) => entry.event), + err, + { message: 'destination init failed' }, + ); + // A refused init (not-ready) never reached the transport. + if (!isTransportFailure(failed)) releaseProbeSlot(); + return { id, destination, failed, error: err }; + } - // Process the destinations event queue - let error: unknown; - let response: unknown; - if (!destination.dlq) destination.dlq = []; - - // The before and next routes; each resolves per event, hop by hop, - // with `{ ingest, event }` (the runner resolves nothing ahead). - const before = destination.config.before; - const nextConfig = destination.config.next; - - // Compile destination cache once per batch (not per-event). - // Destination caches operate on events (HIT/MISS keyed by event fields), - // so the rule shape is always EventCacheRule, not StoreCacheRule. - const destCacheConfig = destination.config?.cache as - | Cache.Cache - | undefined; - const compiledDCache = destCacheConfig - ? compileCache(destCacheConfig) - : undefined; - const dCacheStore = compiledDCache - ? getCacheStore(compiledDCache, collector) - : undefined; - - // Compile declarative state entries once per batch. `get` runs before - // the mapping-to-payload push (so it can shape the pushed event); `set` - // runs after a successful send. - const dStateEntries = compileStepState(destination.config?.state); - const dStateGet = dStateEntries?.filter((entry) => entry.mode === 'get'); - const dStateSet = dStateEntries?.filter((entry) => entry.mode === 'set'); - - // Process allowed events and store failed ones in the dead letter queue (DLQ) - let totalDuration = 0; - // Count of events enqueued into a batch in this pass; the aggregation - // below uses this to skip the synchronous `count++` for batched events - // (counters move to the flush callback per PROD-004 plan Q9). - let batchedCount = 0; - // Events the destination actually received. A before-chain fan-out makes - // this exceed allowedEvents.length, which counts what entered. - let pushedCount = 0; - await Promise.all( - allowedEvents.map(async (event) => { - // Merge collector user into event (collector as base, event overrides) - event.user = assign(user, event.user); - - // Full cache check: before the before chain (skips everything on HIT) - let cacheMiss: { key: string; ttl: number } | undefined; - let cacheValue: unknown; - if (compiledDCache?.stop && dCacheStore) { - const cacheContext = createMappingRoot(destIngest, event); - const cacheResult = await checkCache( - compiledDCache, - dCacheStore, - cacheContext, - ); - if (cacheResult?.status === 'HIT') { - return event; // Skip before chain + push - } - if (cacheResult?.status === 'MISS') { - cacheMiss = { key: cacheResult.key, ttl: cacheResult.rule.ttl }; - } + // The init flush held the destination: a state delivery did not + // settle. The allowed events wait as held; the denied ones are queued + // already. + if (getStateHold(destination)) { + releaseProbeSlot(); + const refused = requeueHeld(allowedEvents); + if (refused.length) { + const failed = refuseHeld(refused); + return { id, destination, failed }; } + return { id, destination, queued: HELD_WAIT }; + } - // Run post-collector transformer chain if configured for this destination - let children: Array<{ event: WalkerOS.Event; ingest: Ingest }> = [ - { event, ingest: destIngest }, - ]; - let destRespond = meta.respond; - if (before !== undefined) { - const chainPath = `destination.${id}.before`; - const chainResult = await runTransformerChain( - collector, - transformers, - before, - event, - destIngest, - meta.respond, - chainPath, - ); + // A consent command ran while init was awaited: the events admitted + // before it pass the gate again with the consent that holds now. + const deniedAfter: Destination.Queued[] = []; + if (collector.consent !== admittedWith) { + const regated = admit( + allowedEvents.map((entry) => { + entry.event = { ...entry.event, consent: collector.consent }; + return entry; + }), + ); + allowedEvents.splice(0, allowedEvents.length, ...regated.allowed); + deniedAfter.push(...regated.denied); + } - if (chainResult.copies.length === 0) { - // Dropped or stopped: THIS destination skips the event; other - // destinations are unaffected. - const skipState = stepState( - collector, - stepId('destination', id), - 'destination', - 'skip', - typeof event.id === 'string' ? event.id : '', - Date.now(), - journeyFields(event, destIngest, collector), - ); - skipState.skipReason = 'dropped'; - skipState.meta = { - by: chainResult.droppedBy ?? 'route', - at: chainPath, - }; - emitStep(collector, skipState); - return event; - } + // Events parked by a concurrent push while the init flush was running + // pass the same consent gate, with the consent that holds now, and go + // out with this call. + const queuedAfter = destination.queuePush || []; + const arrived = queuedAfter.filter((queued) => !parked.has(queued)); + if (arrived.length) { + destination.queuePush = queuedAfter.filter((queued) => + parked.has(queued), + ); + const late = admit( + arrived.map((entry) => { + entry.event = { ...entry.event, consent: collector.consent }; + return entry; + }), + ); + allowedEvents.push(...late.allowed); + deniedAfter.push(...late.denied); + } + requeue(deniedAfter); - // Update respond if the chain produced a wrapped one - if (chainResult.respond) destRespond = chainResult.respond; + if (!allowedEvents.length) { + releaseProbeSlot(); // every event was denied after init + return deniedAfter.length + ? { id, destination, queued: CONSENT_WAIT } + : { id, destination, skipped: true }; + } - // A before chain may fan one event into several. Every child is - // delivered, each with the ingest it finished with. - children = chainResult.copies.map((copy) => ({ - event: copy.event as WalkerOS.Event, - ingest: copy.ingest, - })); - } + // Process the destinations event queue + let error: unknown; + let failure: FailureReason | undefined; + // A push failed at the transport: the outcome sink recorded it for + // the breaker. + let pushRecorded = false; + let response: unknown; + + // The before and next routes; each resolves per event, hop by hop, + // with `{ ingest, event }` (the runner resolves nothing ahead). + const before = destination.config.before; + const nextConfig = destination.config.next; + + // Compile destination cache once per batch (not per-event). + // Destination caches operate on events (HIT/MISS keyed by event + // fields), so the rule shape is always EventCacheRule, not + // StoreCacheRule. + const destCacheConfig = destination.config?.cache as + | Cache.Cache + | undefined; + const compiledDCache = destCacheConfig + ? compileCache(destCacheConfig) + : undefined; + const dCacheStore = compiledDCache + ? getCacheStore(compiledDCache, collector) + : undefined; + + // Compile declarative state entries once per batch. `get` runs before + // the mapping-to-payload push (so it can shape the pushed event); + // `set` runs after a successful send. + const dStateEntries = compileStepState(destination.config?.state); + const dStateGet = dStateEntries?.filter( + (entry) => entry.mode === 'get', + ); + const dStateSet = dStateEntries?.filter( + (entry) => entry.mode === 'set', + ); - // Delivers one child of the before chain, from the step-level cache - // check through the destination.next chain. Runs once per child, so - // a fan-out reaches the destination in full. Returns whether the - // child was actually delivered. - const deliverOne = async ( - processedEvent: WalkerOS.Event | null, - childIngest: Ingest, - sharedMiss: { key: string; ttl: number } | undefined, - ): Promise => { - // The pre-chain key is derived from the one event that entered, so - // a fan-out shares it and one write covers every child. A - // step-level key is derived from THIS child, so it is the child's - // own and must be written here. - let cacheMiss = sharedMiss; - let ownKey = false; - - // Step-level cache check: after before chain, skip only push on HIT - if (compiledDCache && !compiledDCache.stop && dCacheStore) { - const cacheContext = createMappingRoot( - childIngest, - processedEvent ?? undefined, - ); + let totalDuration = 0; + // Events enqueued into a batch in this pass: they wait in the batch + // window (`queued`, reason `batch`); their counters move to the flush + // callback (per PROD-004 plan Q9). + let batchedCount = 0; + // Events the destination actually received. A before-chain fan-out + // makes this exceed allowedEvents.length, which counts what entered. + let pushedCount = 0; + await Promise.all( + allowedEvents.map(async (entry) => { + const event = entry.event; + // Merge collector user into event (collector as base, event overrides) + event.user = assign(user, event.user); + + // Full cache check: before the before chain (skips everything on HIT) + let cacheMiss: { key: string; ttl: number } | undefined; + let cacheValue: unknown; + if (compiledDCache?.stop && dCacheStore) { + const cacheContext = createMappingRoot(destIngest, event); const cacheResult = await checkCache( compiledDCache, dCacheStore, cacheContext, + undefined, + collector, ); if (cacheResult?.status === 'HIT') { - return false; // Skip push, deduplicated + return event; // Skip before chain + push } if (cacheResult?.status === 'MISS') { - cacheMiss = { key: cacheResult.key, ttl: cacheResult.rule.ttl }; - ownKey = true; + cacheMiss = { + key: cacheResult.key, + ttl: cacheResult.rule.ttl, + }; } } - // state[get]: enrich the event before the mapping-to-payload push. - if (dStateGet && dStateGet.length > 0 && processedEvent) { - processedEvent = await runStepState( + // Run post-collector transformer chain if configured for this + // destination + let children: Array<{ event: WalkerOS.Event; ingest: Ingest }> = [ + { event, ingest: destIngest }, + ]; + let destRespond = meta.respond; + if (before !== undefined) { + const chainPath = `destination.${id}.before`; + const chainResult = await runTransformerChain( collector, - dStateGet, - processedEvent, - childIngest, + transformers, + before, + event, + destIngest, + meta.respond, + chainPath, ); - } - const pushStart = Date.now(); - let pushFailed = false; - const result = await tryCatchAsync(destinationPush, (err) => { - // Log the error with destination scope - const destType = destination.type || 'unknown'; - collector.logger.scope(destType).error('Push failed', { - ...errorMeta(err), - event: processedEvent!.name, - }); - error = err; // oh no - pushFailed = true; - - // Add failed event to destinations DLQ (bounded; FIFO drop-oldest) - const dlq = destination.dlq!; - const destId = destination.config.id || id; - const dlqBound = { - max: destination.config.dlqMax ?? DEFAULT_DLQ_MAX, - }; - const dlqResult = pushBounded( - dlq, - [processedEvent!, err], - dlqBound, - ); - if (dlqResult.dropped > 0) { - ensureDestStatus(collector, destId); - const droppedCount = bumpDropped( - collector.status, - stepId('destination', destId), - 'dlq', - dlqResult.dropped, - ); - warnOverflowOnce( - dlq, - collector.logger.scope(destination.type || 'unknown'), - 'destination.dlq overflow; oldest entries dropped', - { - buffer: 'dlq', - destination: destId, - cap: dlqBound.max, - droppedCount, - }, - ); - } else if (dlq.length < dlqBound.max) { - resetOverflowFlag(dlq); + // A transformer of this destination's before chain failed: the + // destination did not get the event. The transformer's failure + // is counted and logged already; the destination reports it. + if (chainResult.failure) { + failure = chainResult.failure.reason; + error = `Transformer "${chainResult.failure.by}" failed: ${chainResult.failure.reason.message}`; + ensureDestStatus(collector, id).failed++; } - return undefined; - })( - collector, - destination, - id, - processedEvent!, - childIngest, - destRespond, - ); - totalDuration += Date.now() - pushStart; - - // Destination cache MISS. A per-child key is written now; the - // shared pre-chain key is recorded and written once after every - // child settles. The stored value is a dedup marker, since neither - // cache check reads it back. - if ( - cacheMiss && - dCacheStore && - destination.config.mock === undefined - ) { - if (ownKey) { - storeCache( - dCacheStore, - cacheMiss.key, - result ?? true, - cacheMiss.ttl, + if (chainResult.copies.length === 0) { + if (chainResult.failure) return event; + // Dropped or stopped: THIS destination skips the event; other + // destinations are unaffected. + const skipState = stepState( + collector, + stepId('destination', id), + 'destination', + 'skip', + typeof event.id === 'string' ? event.id : '', + Date.now(), + journeyFields(event, destIngest, collector), ); - } else if (cacheValue === undefined) { - cacheValue = result ?? true; + skipState.skipReason = 'dropped'; + skipState.meta = { + by: chainResult.droppedBy ?? 'route', + at: chainPath, + }; + emitStep(collector, skipState); + return event; } - } - // state[set]: stash from the event after a successful send. A - // batched-enqueue is not a real send (the sentinel is returned - // before delivery), so set is deferred until flush. The flush-path - // write is a documented follow-up, not handled here yet. - if ( - !pushFailed && - !isBatchedResult(result) && - dStateSet && - dStateSet.length > 0 && - processedEvent - ) { - processedEvent = await runStepState( - collector, - dStateSet, - processedEvent, - childIngest, - ); - } + // Update respond if the chain produced a wrapped one + if (chainResult.respond) destRespond = chainResult.respond; - // Capture the last response (for single event pushes). - // Batched-enqueue sentinel is NOT a real response; don't surface it. - if (result !== undefined && !isBatchedResult(result)) { - response = result; + // A before chain may fan one event into several. Every child is + // delivered, each with the ingest it finished with. + children = chainResult.copies.map((copy) => ({ + event: copy.event as WalkerOS.Event, + ingest: copy.ingest, + })); } - if (isBatchedResult(result)) batchedCount++; - // Run destination.next chain after successful push - if (!pushFailed && nextConfig) { - // Write push response to ingest for destination.next transformers - if (result !== undefined) { - childIngest._response = result; + // Delivers one child of the before chain, from the step-level + // cache check through the destination.next chain. Runs once per + // child, so a fan-out reaches the destination in full. Returns + // whether the child was actually delivered. + const deliverOne = async ( + processedEvent: WalkerOS.Event | null, + childIngest: Ingest, + sharedMiss: { key: string; ttl: number } | undefined, + ): Promise => { + // The pre-chain key is derived from the one event that entered, + // so a fan-out shares it and one write covers every child. A + // step-level key is derived from THIS child, so it is the + // child's own and must be written here. + let cacheMiss = sharedMiss; + let ownKey = false; + + // Step-level cache check: after before chain, skip only push on HIT + if (compiledDCache && !compiledDCache.stop && dCacheStore) { + const cacheContext = createMappingRoot( + childIngest, + processedEvent ?? undefined, + ); + const cacheResult = await checkCache( + compiledDCache, + dCacheStore, + cacheContext, + undefined, + collector, + ); + if (cacheResult?.status === 'HIT') { + return false; // Skip push, deduplicated + } + if (cacheResult?.status === 'MISS') { + cacheMiss = { + key: cacheResult.key, + ttl: cacheResult.rule.ttl, + }; + ownKey = true; + } + } + + // state[get]: enrich the event before the mapping-to-payload push. + if (dStateGet && dStateGet.length > 0 && processedEvent) { + processedEvent = await runStepState( + collector, + dStateGet, + processedEvent, + childIngest, + ); } - // Delivery already happened: a stop only ends this chain. - const nextResult = await runTransformerChain( + const pushStart = Date.now(); + let pushFailed = false; + const result = await tryCatchAsync(destinationPush, (err) => { + // The one outcome sink: counters, DLQ, breaker, one cause + // line. destinationPush emitted the Observe error already. + error = err; + failure = recordFailure( + collector, + destination, + id, + [processedEvent!], + err, + { observed: true }, + ); + if (isTransportFailure(failure)) pushRecorded = true; + pushFailed = true; + return undefined; + })( collector, - transformers, - nextConfig, + destination, + id, processedEvent!, childIngest, destRespond, - `destination.${id}.next`, ); - if (nextResult.respond) destRespond = nextResult.respond; - } + totalDuration += Date.now() - pushStart; + + // A rule that ignores the event: deliberately not sent, a skip. + if (isIgnoredResult(result)) return false; + + // Destination cache MISS. A per-child key is written now; the + // shared pre-chain key is recorded and written once after every + // child settles. The stored value is a dedup marker, since + // neither cache check reads it back. + if ( + cacheMiss && + dCacheStore && + destination.config.mock === undefined + ) { + if (ownKey) { + storeCache( + dCacheStore, + cacheMiss.key, + result ?? true, + cacheMiss.ttl, + ); + } else if (cacheValue === undefined) { + cacheValue = result ?? true; + } + } - return true; - }; + // state[set]: stash from the event after a successful send. A + // batched-enqueue is not a real send (the sentinel is returned + // before delivery), so set is deferred until flush. The + // flush-path write is a documented follow-up, not handled here + // yet. + if ( + !pushFailed && + !isBatchedResult(result) && + dStateSet && + dStateSet.length > 0 && + processedEvent + ) { + processedEvent = await runStepState( + collector, + dStateSet, + processedEvent, + childIngest, + ); + } - for (const child of children) { - // Each child gets its own step-level cache decision; the full-check - // miss recorded before the chain is shared, since it was computed - // from the one event that entered. Only a child that actually - // reached the destination counts as delivered. - if (await deliverOne(child.event, child.ingest, cacheMiss)) - pushedCount++; - } + // Capture the last response (for single event pushes). + // Batched-enqueue sentinel is NOT a real response; don't surface it. + if (result !== undefined && !isBatchedResult(result)) { + response = result; + } + if (isBatchedResult(result)) { + batchedCount++; + return false; + } + if (pushFailed) return false; + + // Run destination.next chain after successful push + if (nextConfig) { + // Write push response to ingest for destination.next transformers + if (result !== undefined) { + childIngest._response = result; + } + + // Delivery already happened: a stop only ends this chain. + const nextResult = await runTransformerChain( + collector, + transformers, + nextConfig, + processedEvent!, + childIngest, + destRespond, + `destination.${id}.next`, + ); + if (nextResult.respond) destRespond = nextResult.respond; + } - // One write per request, after every child settled. - if ( - cacheMiss && - dCacheStore && - cacheValue !== undefined && - destination.config.mock === undefined - ) { - storeCache(dCacheStore, cacheMiss.key, cacheValue, cacheMiss.ttl); - } + return true; + }; - return event; - }), - ); + for (const child of children) { + // Each child gets its own step-level cache decision; the + // full-check miss recorded before the chain is shared, since it + // was computed from the one event that entered. Only a child + // that actually reached the destination counts as delivered. + if (await deliverOne(child.event, child.ingest, cacheMiss)) + pushedCount++; + } - return { - id, - destination, - error, - response, - totalDuration, - batchedCount, - allowedCount: allowedEvents.length, - pushedCount, - canonicalId, - breakerConfig, - }; - }), + // One write per request, after every child settled. + if ( + cacheMiss && + dCacheStore && + cacheValue !== undefined && + destination.config.mock === undefined + ) { + storeCache(dCacheStore, cacheMiss.key, cacheValue, cacheMiss.ttl); + } + + return event; + }), + ); + + // Settle a probe this pass may hold: a transport failure was + // recorded by the outcome sink; otherwise a synchronous delivery + // closes the breaker, and a pass that reached no transport (every + // event ignored, deduplicated, dropped, failed by a before chain or + // by its mapping) releases it. A batched pass settles in its flush, + // which records its own outcome. + if (!pushRecorded) { + if (pushedCount) recordProbe('success'); + else if (!batchedCount) releaseProbeSlot(); + } + + return { + id, + destination, + ...(failure ? { failed: failure, error } : {}), + response, + totalDuration, + delivered: pushedCount, + batched: batchedCount, + }; + }, + ), ); // Build result objects @@ -1103,63 +1306,36 @@ async function deliverToDestinations( destStatus.queuePushSize = destination.queuePush?.length ?? 0; destStatus.dlqSize = destination.dlq?.length ?? 0; - // Circuit-breaker accounting keys on the canonical stepId so it matches the - // gate exactly. Presence-gated: only when this destination has a breaker. - const breakerConfig = result.breakerConfig; - const breakerKey = result.canonicalId - ? stepId('destination', result.canonicalId) - : undefined; - const recordBreaker = (outcome: 'transport-failure' | 'success') => { - if (breakerConfig && breakerKey) { - recordStepOutcome( - collector.status.breakers, - breakerKey, - outcome, - breakerConfig.threshold, - breakerConfig.cooldown, - ); - } - }; + // Events delivered synchronously count once per pass (the historical + // semantics: one bump per pushToDestinations call, regardless of the + // number of events), whatever else happened in the pass. + const delivered = result.delivered ?? 0; + if (delivered > 0) { + destStatus.count++; + destStatus.lastAt = now; + destStatus.duration += result.totalDuration || 0; + collector.status.out++; + } - if (result.error) { + if (result.failed) { + // Counted, DLQ'd and logged by the outcome sink already. ref.error = result.error; + ref.reason = result.failed; failed[result.id] = ref; - destStatus.failed++; destStatus.lastAt = now; - destStatus.duration += result.totalDuration || 0; - collector.status.failed++; - // A single-event push that threw/timed out is a transport failure. - recordBreaker('transport-failure'); - } else if (result.queue && result.queue.length) { - // Events already re-queued at destination.queuePush via skippedEvents push + if (!delivered) destStatus.duration += result.totalDuration || 0; + } else if (delivered > 0) { + done[result.id] = ref; + } else if ((result.batched ?? 0) > 0) { + // In a batch window: delivered (and counted) when the batch flushes. + ref.reason = BATCH_WAIT; + queued[result.id] = ref; + } else if (result.queued) { + ref.reason = result.queued; queued[result.id] = ref; - } else { - // If every allowed event was enqueued into a batch, success counters - // belong to the flush callback (PROD-004 plan Q9): don't increment - // here and don't mark `done`. The flush bumps `count` and `out` when - // pushBatch resolves. If only some events were batched, count the - // synchronously-delivered ones. - const batchedCount = result.batchedCount ?? 0; - const allowedCount = result.allowedCount ?? 0; - // What the destination actually received. A before-chain fan-out makes - // this exceed allowedCount, which counts what entered. - const deliveredCount = result.pushedCount ?? allowedCount; - const syncDelivered = Math.max(0, deliveredCount - batchedCount); - if (syncDelivered > 0 || allowedCount === 0) { - done[result.id] = ref; - // For non-batched destinations preserve the historical semantics - // (one bump per pushToDestinations call, regardless of allowed - // events count, since the original code only bumped once). - destStatus.count++; - destStatus.lastAt = now; - destStatus.duration += result.totalDuration || 0; - collector.status.out++; - // A synchronously-delivered single-event push is a success: reset and - // close the breaker. (Fully-batched passes settle in the flush - // callback instead, which records its own outcome.) - recordBreaker('success'); - } } + // Otherwise every event was skipped on purpose (ignored, deduplicated by + // the cache, dropped by a before chain): nothing to report. } return createPushResult({ @@ -1191,14 +1367,15 @@ function hasConsentRequirement(destination: Destination.Instance): boolean { /** * Run a destination's own `init()` and return the config to install, marked - * initialized, or false when `init()` returned false. The caller installs it. + * initialized. An `init()` that throws, or returns false, throws (the + * latter as `StepError` code `not-ready`). The caller installs the config. */ async function runDestinationInit( collector: Collector.Instance, destination: Destination.Instance, init: NonNullable, destId: string, -): Promise { +): Promise { // Create scoped logger for this destination: [type:id] or [unknown:id] const destType = destination.type || 'unknown'; const destLogger = collector.logger.scope(destType); @@ -1252,13 +1429,19 @@ async function runDestinationInit( initErrFinished, ); errState.durationMs = initErrFinished - initStarted; - errState.error = stepError(err); + errState.error = observerStepError(err); emitStep(collector, errState); throw err; } - // Actively check for errors (when false) - if (configResult === false) return false; // don't push if init is false + // Runtime guard: bundled code is not typechecked, so an init may still + // return false. Without this, `{ ...(false || config), init: true }` + // would install a destination with its settings missing. + if (configResult === false) + throw new StepError( + 'init returned false: throw a StepError that names the missing setting', + { code: 'not-ready' }, + ); // The config to install, marked initialized; the caller installs it. return { @@ -1276,12 +1459,14 @@ async function runDestinationInit( async function initAndFlush( collector: Collector.Instance, destination: Destination.Instance, - init: NonNullable, + init: Destination.Instance['init'], destId: string, ): Promise { - const config = await runDestinationInit(collector, destination, init, destId); - if (config === false) return false; // don't push if init is false - destination.config = config; + // A destination without an `init` has nothing to run: it is marked + // initialized, and its queued deliveries flush below. + destination.config = init + ? await runDestinationInit(collector, destination, init, destId) + : { ...destination.config, init: true }; // Flush queued on() events now that destination is initialized. The // queue stays live while the flush runs: a delivery that arrives in the @@ -1325,7 +1510,8 @@ async function initAndFlush( } } - collector.logger.scope(destination.type || 'unknown').debug('init done'); + if (init) + collector.logger.scope(destination.type || 'unknown').debug('init done'); return true; } @@ -1346,8 +1532,10 @@ export async function destinationInit( // consent-gated destination is never initialized without a cleared gate. allowed = false, ): Promise { - // Check if the destination was initialized properly or try to do so - if (destination.init && !destination.config.init) { + // Check if the destination was initialized properly or try to do so. A + // destination without an `init` runs a no-op one, through the same consent + // gate and queue flush, so its queued `on()` deliveries reach it. + if (!destination.config.init) { // Defense-in-depth: refuse to init a destination that declares a consent // requirement unless the gate was cleared. We do NOT re-derive consent from // collector state here, because the push path's decision may rest on an @@ -1403,9 +1591,45 @@ export async function destinationPush( ): Promise { const { config } = destination; - const processed = await processEventMapping(event, config, collector); + // A rule `condition` that throws fails the event (`mapping`): reported to + // observers here, like any failed push, then rethrown to the outcome sink. + let processed: Awaited< + ReturnType> + >; + try { + processed = await processEventMapping(event, config, collector); + } catch (err) { + const errState = stepState( + collector, + stepId('destination', destId), + 'destination', + 'error', + typeof event.id === 'string' ? event.id : '', + Date.now(), + journeyFields(event, ingest, collector), + ); + errState.error = observerStepError(err); + emitStep(collector, errState); + throw err; + } - if (processed.ignore) return false; + // An ignored event is a deliberate skip: counted in `skipped`, never `done`. + if (processed.ignore) { + recordSkip(collector, destination, destId, event, 'ignored'); + const skipState = stepState( + collector, + stepId('destination', destId), + 'destination', + 'skip', + typeof event.id === 'string' ? event.id : '', + Date.now(), + journeyFields(event, ingest, collector), + ); + skipState.skipReason = 'ignored'; + if (processed.mappingKey) skipState.mappingKey = processed.mappingKey; + emitStep(collector, skipState); + return IGNORED_RESULT; + } // Create scoped logger for this destination: [type] or [unknown] const destType = destination.type || 'unknown'; @@ -1481,10 +1705,12 @@ export async function destinationPush( // so the closure does NOT leak the first event's context. const baseEnv = mergeEnvironments(destination.env, config.env); - const flushBatch = async (): Promise => { + const flushBatch = async (): Promise< + Destination.BatchFlushOutcome | undefined + > => { const batchState = destination.batches![mappingKey]; const currentBatched = batchState.batched; - if (currentBatched.entries.length === 0) return; + if (currentBatched.entries.length === 0) return undefined; const snapshot: Destination.Batch = { key: currentBatched.key, @@ -1553,46 +1779,7 @@ export async function destinationPush( const failedRows = new Map(); let batchThrew = false; let batchError: unknown; - - // Routes a set of [event, error] pairs to this destination's DLQ, - // applying the bound and emitting the overflow warning once. Shared by - // the whole-batch failure path (pushBatch threw) and the partial-row - // failure path (pushBatch returned a BatchOutcome). - const routeToDlq = (failures: Array<[WalkerOS.Event, unknown]>) => { - const dlq = (destination.dlq = destination.dlq || []); - const dlqBound = { - max: destination.config.dlqMax ?? DEFAULT_DLQ_MAX, - }; - let totalDropped = 0; - for (const pair of failures) { - const r = pushBounded(dlq, pair, dlqBound); - totalDropped += r.dropped; - } - if (totalDropped > 0) { - const droppedCount = bumpDropped( - collector.status, - stepId('destination', destIdResolved), - 'dlq', - totalDropped, - ); - warnOverflowOnce( - dlq, - destLogger, - 'destination.dlq overflow; oldest entries dropped', - { - buffer: 'dlq', - destination: destIdResolved, - cap: dlqBound.max, - droppedCount, - }, - ); - } else if (dlq.length < dlqBound.max) { - resetOverflowFlag(dlq); - } - destStatus.failed += failures.length; - destStatus.dlqSize = dlq.length; - collector.status.failed += failures.length; - }; + let failure: FailureReason | undefined; // Number of entries treated as delivered after the flush settles. // Whole-batch success (void) = all; whole-batch failure (throw) = 0; @@ -1621,74 +1808,76 @@ export async function destinationPush( succeededCount = 0; batchThrew = true; batchError = err; - // Route the entire batch to DLQ on a thrown/whole-batch failure. - routeToDlq(snapshot.entries.map((entry) => [entry.event, err])); - // Whole-batch throw is a transport failure for the breaker. - recordFlushBreaker('transport-failure'); - destLogger.error('Push batch failed', { - error: err instanceof Error ? err.message : String(err), - entries: snapshot.entries.length, - }); + // The whole batch failed: every entry through the one outcome + // sink (counters, DLQ, breaker, one cause line). The per-entry + // Observe frames follow below. + failure = recordFailure( + collector, + destination, + destIdResolved, + snapshot.entries.map((entry) => entry.event), + err, + { observed: true, message: 'Push batch failed' }, + ); return undefined; }, )(snapshot, batchContext); // Partial-failure path: pushBatch resolved a BatchOutcome listing the - // entries that did not succeed. DLQ and fail-count only those; the rest - // are delivered. Out-of-range indices are ignored defensively. + // entries that did not succeed. Only those fail, each with its own + // error, through the outcome sink; the rest are delivered. One line + // summarizes the rows by cause (the error's `code`, else its + // `name`), with a count and the first message as a sample. + // Partial-failure rows are breaker-neutral. Out-of-range indices are + // ignored defensively. if (isBatchOutcome(outcome) && outcome.failed.length > 0) { - const failedPairs: Array<[WalkerOS.Event, unknown]> = []; - for (const failure of outcome.failed) { - const entry = snapshot.entries[failure.index]; - if (!entry || failedRows.has(failure.index)) continue; + for (const row of outcome.failed) { + const entry = snapshot.entries[row.index]; + if (!entry || failedRows.has(row.index)) continue; const rowError = - failure.error ?? + row.error ?? new Error( - `Push batch entry ${failure.index} failed (no per-row error provided)`, + `Push batch entry ${row.index} failed (no per-row error provided)`, ); - failedRows.set(failure.index, rowError); - failedPairs.push([entry.event, rowError]); + failedRows.set(row.index, rowError); } - if (failedPairs.length > 0) { - routeToDlq(failedPairs); - succeededCount = Math.max( - 0, - snapshot.entries.length - failedPairs.length, + const causes = new Map< + string, + { code: string; count: number; message: string } + >(); + for (const [index, rowError] of failedRows) { + const reason = recordFailure( + collector, + destination, + destIdResolved, + [snapshot.entries[index].event], + rowError, + { observed: true, report: false, breaker: false }, ); - // One entry per failure cause: the error's `code`, else its - // `name`, with a count and the first message as a sample. - const causes = new Map< - string, - { code: string; count: number; message: string } - >(); - for (const [, rowError] of failedPairs) { - const code = - rowError instanceof Error && - 'code' in rowError && - typeof rowError.code === 'string' - ? rowError.code - : rowError instanceof Error - ? rowError.name - : 'Error'; - const cause = causes.get(code); - if (cause) cause.count++; - else - causes.set(code, { - code, - count: 1, - message: - rowError instanceof Error - ? rowError.message - : String(rowError), - }); - } + failure = failure ?? reason; + const code = + rowError instanceof Error && + 'code' in rowError && + typeof rowError.code === 'string' + ? rowError.code + : rowError instanceof Error + ? rowError.name + : 'Error'; + const cause = causes.get(code); + if (cause) cause.count++; + else causes.set(code, { code, count: 1, message: reason.message }); + } + succeededCount = Math.max( + 0, + snapshot.entries.length - failedRows.size, + ); + if (failedRows.size > 0) destLogger.error('Push batch partial failure', { - failed: failedPairs.length, + failed: failedRows.size, delivered: succeededCount, entries: snapshot.entries.length, causes: [...causes.values()], }); - } } // Batch outcomes fan out one frame per entry, emitted once the flush @@ -1722,7 +1911,7 @@ export async function destinationPush( state.durationMs = settledAt - flushStarted; if (failed) { const err = batchThrew ? batchError : failedRows.get(index); - state.error = stepError(err); + state.error = observerStepError(err); } emitStep(collector, state); }); @@ -1740,10 +1929,16 @@ export async function destinationPush( destStatus.lastAt = Date.now(); collector.status.out += succeededCount; // Any delivered rows mean the transport worked: success closes the - // breaker. Partial-failure rows are breaker-neutral (handled by the - // absence of a transport-failure record for them above). + // breaker. Partial-failure rows are breaker-neutral (recorded with + // `breaker: false` above). recordFlushBreaker('success'); } + + return { + delivered: succeededCount, + failed: snapshot.entries.length - succeededCount, + ...(failure ? { reason: failure } : {}), + }; }; const scheduler = debounce(flushBatch, { @@ -1758,9 +1953,7 @@ export async function destinationPush( batchFn: () => { void scheduler(); }, - flush: async () => { - await scheduler.flush(); - }, + flush: async () => (await scheduler.flush()) ?? undefined, }; } @@ -1944,7 +2137,7 @@ export async function destinationPush( journey, ); errState.durationMs = pushFinished - pushStarted; - errState.error = stepError(err); + errState.error = observerStepError(err); // Surface the vendor calls recorded before the failure, so a mid-push // error doesn't discard them. attachCalls(errState, recordedCalls); @@ -1994,6 +2187,57 @@ export function registerDestination( return { ...code, config: mergedConfig, env: mergedEnv }; } +/** + * Registration checks of a destination, logged once at ERROR (a config value + * without effect): a `batch` without `pushBatch` (events would go to `push` + * one by one), and mapping keys that do nothing at the destination position + * (behind the `__WALKEROS_VALIDATE__` build flag: CLI bundles check at build + * time). The destination still runs. With `chains`, the transformers its + * `before` and `next` name must exist (checked once transformers exist). + */ +export function checkDestinationRegistration( + collector: Collector.Instance, + id: string, + destination: Destination.Instance, + chains = false, +): void { + const { config } = destination; + const logger = collector.logger.scope(destination.type || 'unknown'); + + if (!destination.pushBatch) { + const batchPaths: string[] = []; + if (config.batch !== undefined) batchPaths.push('config.batch'); + for (const [entity, actions] of Object.entries(config.mapping || {})) { + for (const [action, value] of Object.entries(actions || {})) { + const list = isArray(value) ? value : [value]; + list.forEach((rule, index) => { + if (rule?.batch === undefined) return; + batchPaths.push( + `config.mapping.${entity}.${action}${isArray(value) ? `[${index}]` : ''}.batch`, + ); + }); + } + } + if (batchPaths.length) + logger.error( + `Destination ${id}: ${batchPaths.map((path) => `\`${path}\``).join(', ')} has no effect: the destination has no pushBatch, so events go to push one by one`, + ); + } + + if (typeof __WALKEROS_VALIDATE__ === 'undefined' || __WALKEROS_VALIDATE__) { + const paths = getMappingNoOpPaths('destination', config, 'config'); + if (paths.length) + logger.error( + `Destination ${id}: ${getMappingNoOpMessage('destination', paths)}`, + ); + } + + if (chains) { + checkChainTargets(collector, `destination.${id}.before`, config.before); + checkChainTargets(collector, `destination.${id}.next`, config.next); + } +} + /** * Initializes a map of destinations using ONLY the unified code/config/env pattern. * Does NOT call destination.init() - that happens later during push with proper consent checks. @@ -2019,6 +2263,7 @@ export async function initDestinations( continue; } result[id] = registerDestination(def); + checkDestinationRegistration(collector, id, result[id]); } return result; diff --git a/packages/collector/src/flow.ts b/packages/collector/src/flow.ts index 07747f11a..b28cca3c9 100644 --- a/packages/collector/src/flow.ts +++ b/packages/collector/src/flow.ts @@ -290,7 +290,7 @@ export async function startFlow< if (typeof __WALKEROS_OBSERVE__ === 'undefined' || __WALKEROS_OBSERVE__) { installObserve(instance, config); } else if (config.observe) { - instance.logger.warn('observe: not in this build, config ignored'); + instance.logger.error('observe: not in this build, config ignored'); } // Initialize sources; the collector's elb adapter is already available diff --git a/packages/collector/src/handle.ts b/packages/collector/src/handle.ts index acd55c1ab..8ced52a92 100644 --- a/packages/collector/src/handle.ts +++ b/packages/collector/src/handle.ts @@ -29,7 +29,7 @@ import { processConsent } from './consent'; import { on, onApplyDeferred, startRunRedelivery, enterCascade } from './on'; import type { DestinationDelivery } from './on'; import { reconcilePending } from './pending'; -import { destroyAllSteps } from './shutdown'; +import { destroyAllSteps, flushDestinationBatches } from './shutdown'; import type { RunState } from './types/collector'; // Replaced at build time by tsup's `define` (see packages/config/tsup). @@ -138,6 +138,12 @@ export async function commonHandleCommand( } break; + case Const.Commands.Flush: + // Deliver every pending destination batch now (a page going away), + // through the same flush shutdown uses. The collector keeps running. + result = await flushDestinationBatches(collector); + break; + case Const.Commands.Hook: if ( isObject(data) && diff --git a/packages/collector/src/observerEmit.ts b/packages/collector/src/observerEmit.ts index 0aa9355f0..956ce1dcf 100644 --- a/packages/collector/src/observerEmit.ts +++ b/packages/collector/src/observerEmit.ts @@ -1,5 +1,5 @@ import type { Collector, FlowState, Ingest, WalkerOS } from '@walkeros/core'; -import { emitStep } from '@walkeros/core'; +import { emitStep, StepError, toFailureReason } from '@walkeros/core'; /** Journey-correlation trio shared by every FlowState-stamping site. */ export interface JourneyFields { @@ -96,12 +96,24 @@ export function buildBaseState( /** * The error a `FlowState` carries: name and message for an `Error`, the - * message alone for anything else thrown. + * message alone for anything else thrown, plus the failure code, whether a + * retry can help and the HTTP status. A throw that is not a `StepError` gets + * `code` (`transport` unless the position says otherwise). */ -export function stepError(err: unknown): NonNullable { - return err instanceof Error - ? { name: err.name, message: err.message } - : { message: String(err) }; +export function observerStepError( + err: unknown, + code = 'transport', +): NonNullable { + const message = err instanceof Error ? err.message : String(err); + const reason = toFailureReason( + err instanceof StepError ? err : new StepError(message, { code }), + ); + return { + ...(err instanceof Error ? { name: err.name, message } : { message }), + code: reason.code, + retryable: reason.retryable, + ...(reason.status !== undefined ? { status: reason.status } : {}), + }; } /** @@ -123,6 +135,7 @@ export function emit( * `collector.next`); `by` is the transformer id, `'route'` when omitted. * - `consent`: the source's `config.consent` is not granted; `required` is * that requirement and `consent` the state it was checked against. + * - `ignored`: a source mapping rule with `ignore: true` matched. */ export type CollectorDrop = | { reason: 'dropped'; at: string; by?: string } @@ -131,7 +144,8 @@ export type CollectorDrop = at: string; required: WalkerOS.Consent; consent: WalkerOS.Consent; - }; + } + | { reason: 'ignored'; at: string }; /** * A drop at a collector-owned position: the event never reaches the @@ -163,6 +177,8 @@ export function emitCollectorDrop( if (drop.reason === 'consent') { state.consent = { ...drop.consent }; state.meta = { at: drop.at, required: { ...drop.required } }; + } else if (drop.reason === 'ignored') { + state.meta = { at: drop.at }; } else { state.meta = { by: drop.by ?? 'route', at: drop.at }; } diff --git a/packages/collector/src/on.ts b/packages/collector/src/on.ts index 7604fc9f1..ce3c2c42f 100644 --- a/packages/collector/src/on.ts +++ b/packages/collector/src/on.ts @@ -6,10 +6,12 @@ import type { Source, Logger, } from '@walkeros/core'; -import { isArray, FatalError } from '@walkeros/core'; +import { isArray, FatalError, reportCause, stepId } from '@walkeros/core'; import { Const } from './constants'; import { tryCatch, tryCatchAsync } from '@walkeros/core'; import { mergeEnvironments, pushToDestinations } from './destination'; +import { DEFAULT_QUEUE_MAX } from './buffers'; +import { bumpDropped } from './report-error'; import { buildReportError, errorMeta } from './report-error'; import { reconcilePending } from './pending'; import { flushSourceQueueOn, isSourceStarted } from './source'; @@ -431,7 +433,7 @@ export async function callDestinationOn( gated && (overdueDeliveries.get(destination) || stateCalls.has(destination)) ) { - markStateLost(collector, destination, type); + markStateLost(collector, destination, type, false); if (!overdueDeliveries.get(destination)) deferredDuringCall.add(destination); return false; @@ -490,7 +492,23 @@ export async function callDestinationOn( (err) => { settled = false; if (err instanceof DestinationTimeoutError) timedOut = true; - logOnCallbackError(collector, 'destination', err, { destId, type }); + if (type === 'consent') { + // The vendor never got the consent decision: a destination + // failure (not an unmet condition), one line per cause. + if (err instanceof FatalError) throw err; + reportCause( + collector, + collector.logger, + { + step: stepId('destination', destId), + code: 'consent-not-applied', + }, + 'consent not applied: the destination on(consent) handler failed; its events are held until a retry succeeds', + { type: destination.type, ...errorMeta(err) }, + ); + } else { + logOnCallbackError(collector, 'destination', err, { destId, type }); + } }, )(); threw = false; @@ -624,7 +642,32 @@ function recoverDestinationState( */ export interface StateHold { type: On.Types; + /** When the hold was last raised or its delivery last failed: paces retries. */ since: number; + /** + * When a delivery first failed (threw, or timed out) while the hold lasts; + * absent while the hold only covers a running delivery. A hold failing for + * one retry window blocks the destination. + */ + failedAt?: number; +} + +/** + * Raise or refresh a destination's hold. A failed delivery dates the + * failure once; later failures and deliveries keep that date. + */ +function setStateHold( + destination: Destination.Instance, + type: On.Types, + failed: boolean, +): void { + const now = Date.now(); + const failedAt = stateHolds.get(destination)?.failedAt; + stateHolds.set(destination, { + type, + since: now, + ...(failedAt !== undefined || failed ? { failedAt: failedAt ?? now } : {}), + }); } const stateHolds = new WeakMap(); @@ -722,16 +765,18 @@ export function endInitFlush( * taken back, the next delivery of the cell carries all of it, and the * destination is held, dated now, until that delivery settles. Used for a * delivery that failed, one the init flush never reached, and one the cascade - * bound suppressed. + * bound suppressed. A delivery only deferred behind a running call + * (`failed: false`) does not start the hold's failure window. */ export function markStateLost( collector: Collector.Instance, destination: Destination.Instance, type: On.Types, + failed = true, ): void { clearMark(collector, destination, type); addLostCell(destination, type); - stateHolds.set(destination, { type, since: Date.now() }); + setStateHold(destination, type, failed); } /** @@ -762,7 +807,7 @@ export function openStateDelivery( type: On.Types, ): void { stateInFlight.set(destination, (stateInFlight.get(destination) || 0) + 1); - stateHolds.set(destination, { type, since: Date.now() }); + setStateHold(destination, type, false); } /** @@ -1136,11 +1181,39 @@ export async function deliverStateToDestination( } // Uninitialized, or its init flush is still running: the delivery joins - // the queue behind the older entries, so it never overtakes them. + // the queue behind the older entries, so it never overtakes them. While + // the destination waits (not flushing), a state delivery replaces an + // earlier queued one of its cell, in place: the cell's current value + // supersedes older ones, so a destination waiting for consent holds one + // entry per state type. A running flush delivers every entry in order. + // Other deliveries are bounded by `queueMax`: the oldest is dropped, one + // ERROR per cause. const flush = initFlushes.get(destination); if (!destination.config.init || flush) { - destination.queueOn = destination.queueOn || []; - destination.queueOn.push({ type, data }); + const queue = (destination.queueOn = destination.queueOn || []); + const earlier = + gated && !flush ? queue.findIndex((entry) => entry.type === type) : -1; + if (earlier !== -1) { + queue[earlier] = { type, data: resolveDeliveryData(collector, type) }; + } else if (!gated) { + const max = destination.config.queueMax ?? DEFAULT_QUEUE_MAX; + const lifecycle = queue.filter((entry) => !isStateDelivery(entry.type)); + if (lifecycle.length >= max) { + queue.splice(queue.indexOf(lifecycle[0]), 1); + const step = stepId('destination', destination.config.id || destId); + bumpDropped(collector.status, step, 'queue', 1); + reportCause( + collector, + collector.logger, + { step, code: 'on-overflow' }, + `destination.queueOn full (queueMax ${max}): oldest on() delivery dropped`, + { type: destination.type }, + ); + } + queue.push({ type, data }); + } else { + queue.push({ type, data }); + } if (gated) { setMark(collector, destination, type); // A state entry joining a running flush holds the destination for the diff --git a/packages/collector/src/outcome.ts b/packages/collector/src/outcome.ts new file mode 100644 index 000000000..02900bbe4 --- /dev/null +++ b/packages/collector/src/outcome.ts @@ -0,0 +1,175 @@ +import type { + Collector, + Destination, + FailureReason, + WalkerOS, +} from '@walkeros/core'; +import { + emitStep, + reportCause, + StepError, + stepId, + toFailureReason, +} from '@walkeros/core'; +import { journeyFields, observerStepError, stepState } from './observerEmit'; +import { pushBounded, resetOverflowFlag } from './buffers'; +import { DEFAULT_DLQ_MAX, bumpDropped, ensureDestStatus } from './report-error'; +import { recordStepOutcome, resolveBreakerConfig } from './breaker'; + +/** + * Codes the collector assigns at its own gates: the event never reached the + * transport, so the breaker records no transport failure for them. A probe + * the delivery pass took is settled by that pass. + */ +const GATE_CODES = [ + 'breaker-open', + 'blocked', + 'queue-full', + 'not-ready', + 'mapping', +]; + +/** Whether a failure reached the transport (and counts for the breaker). */ +export function isTransportFailure(reason: FailureReason): boolean { + return !GATE_CODES.includes(reason.code); +} + +/** + * A thrown value as a `StepError`: a `StepError` as it is, anything else + * wrapped with `code` (an unknown throw from a transformer is `transformer`). + */ +export function asStepError(err: unknown, code: string): StepError { + if (err instanceof StepError) return err; + return new StepError(err instanceof Error ? err.message : String(err), { + code, + cause: err, + }); +} + +export interface FailureOptions { + /** The caller already emitted the Observe `error` state of these events. */ + observed?: boolean; + /** The cause line's message; `Push failed` by default. */ + message?: string; + /** + * `false`: the caller writes the one line itself (a batch's partial + * failure summarizes its rows by cause in one line). + */ + report?: boolean; + /** `false`: breaker-neutral (a batch's partial-failure rows). */ + breaker?: boolean; +} + +/** + * The one outcome sink for events a destination failed: derives the reason, + * bumps the destination's and the global `failed`, writes the bounded DLQ + * (an eviction is an ERROR cause line), records the breaker outcome, emits + * Observe `error` per event and writes one cause line (rate limited, never + * the payload). Called with no events for a failure that lost none (an init + * that threw on a flush without events). Returns the reason. + */ +export function recordFailure( + collector: Collector.Instance, + destination: Destination.Instance, + id: string, + events: WalkerOS.Event[], + error: unknown, + options: FailureOptions = {}, +): FailureReason { + const reason = toFailureReason(error); + const step = stepId('destination', id); + const destStatus = ensureDestStatus(collector, id); + const now = Date.now(); + + if (events.length) { + destStatus.failed += events.length; + destStatus.lastAt = now; + collector.status.failed += events.length; + + // Bounded DLQ, oldest evicted: the evicted entries were reported failed + // already, only their replay material is lost. + const dlq = (destination.dlq = destination.dlq || []); + const max = destination.config.dlqMax ?? DEFAULT_DLQ_MAX; + let dropped = 0; + for (const event of events) + dropped += pushBounded(dlq, [event, error], { max }).dropped; + if (dropped > 0) { + const count = bumpDropped(collector.status, step, 'dlq', dropped); + reportCause( + collector, + collector.logger, + { step, code: 'dlq-overflow' }, + `destination.dlq full (dlqMax ${max}): oldest failed entries dropped`, + { type: destination.type, count }, + ); + } else if (dlq.length < max) { + resetOverflowFlag(dlq); + } + destStatus.dlqSize = dlq.length; + } + + // The breaker counts transport failures only. + const breakerConfig = resolveBreakerConfig(destination.config.breaker); + if (breakerConfig && options.breaker !== false && isTransportFailure(reason)) + recordStepOutcome( + collector.status.breakers, + stepId('destination', destination.config.id || id), + 'transport-failure', + breakerConfig.threshold, + breakerConfig.cooldown, + ); + + if (!options.observed) { + for (const event of events) { + const state = stepState( + collector, + step, + 'destination', + 'error', + typeof event.id === 'string' ? event.id : '', + now, + journeyFields(event, undefined, collector), + ); + state.error = observerStepError(error); + emitStep(collector, state); + } + } + + if (options.report === false) return reason; + + reportCause( + collector, + collector.logger, + { step, code: reason.code, status: reason.status }, + options.message ?? 'Push failed', + { + type: destination.type, + event: events[0]?.name, + retryable: reason.retryable, + error: reason.message, + ...(events.length > 1 ? { count: events.length } : {}), + }, + ); + + return reason; +} + +/** + * The one skip path: an event deliberately not sent to a destination (its + * consent denied under `consentDenied: 'skip'`, or a rule that `ignore`s + * it). Counted in `status.destinations[id].skipped`, one DEBUG line. The + * caller emits the Observe `skip` with its reason. + */ +export function recordSkip( + collector: Collector.Instance, + destination: Destination.Instance, + id: string, + event: WalkerOS.Event, + reason: 'consent' | 'ignored', +): void { + const destStatus = ensureDestStatus(collector, id); + destStatus.skipped = (destStatus.skipped ?? 0) + 1; + collector.logger + .scope(destination.type || 'unknown') + .debug('event skipped', { reason, event: event.name }); +} diff --git a/packages/collector/src/pending.ts b/packages/collector/src/pending.ts index b1d7592ed..4f96ab57e 100644 --- a/packages/collector/src/pending.ts +++ b/packages/collector/src/pending.ts @@ -1,5 +1,8 @@ import type { Collector } from '@walkeros/core'; -import { registerDestination } from './destination'; +import { + checkDestinationRegistration, + registerDestination, +} from './destination'; import { catchUpDestinationState, isRequireSatisfied } from './on'; import { flushSourceQueueOn, isSourceStarted } from './source'; @@ -67,8 +70,9 @@ export async function reconcilePending( delete collector.pending.destinations[id]; const instance = registerDestination(def); + checkDestinationRegistration(collector, id, instance, true); if (instance.config.queue !== false) { - instance.queuePush = [...collector.queue]; + instance.queuePush = collector.queue.map((event) => ({ event })); } collector.destinations[id] = instance; await catchUpDestinationState(collector, instance, id); diff --git a/packages/collector/src/push.ts b/packages/collector/src/push.ts index 280170364..1632254bc 100644 --- a/packages/collector/src/push.ts +++ b/packages/collector/src/push.ts @@ -7,22 +7,24 @@ import { getSpanId, InvalidEventError, processEventMapping, + reportCause, + StepError, + toFailureReason, tryCatchAsync, useHooks, } from '@walkeros/core'; -import { - pushBounded, - resetOverflowFlag, - seededQueueMax, - warnOverflowOnce, -} from './buffers'; -import { bumpDropped, errorMeta } from './report-error'; +import { pushBounded, seededQueueMax } from './buffers'; import { createEvent, enrichEvent } from './handle'; -import { pushToDestinations, createPushResult } from './destination'; +import { + chainFailureResult, + createPushResult, + mergePushResults, + pushToDestinations, +} from './destination'; import { emitCollectorDrop, journeyFields, - stepError, + observerStepError, stepState, } from './observerEmit'; import { runTransformerChain } from './transformer'; @@ -69,35 +71,42 @@ export function createPush( // by runCollector, so the pipeline (mapping, chains, enrichment) // runs exactly once, with post-run state. See prerun-hold.test.ts. if (!collector.allowed) { + // Bounded: when full, the newest event is refused (the held ones + // were promised already). That push answers `ok: false` with + // `queue-full`, counted failed, one cause line. const max = seededQueueMax(collector.config.queueMax); const held = pushBounded( collector.preRunQueue, { event, options }, - { max }, + { max, onOverflow: 'dropNewest' }, ); - if (held.dropped > 0) { - // Counted under its own step id, NOT `stepId('collector')`: that - // one's `queue` counter is the post-run replay buffer, and - // pre-run loss is a different problem with a different fix, so - // the two stay separately diagnosable. - const droppedCount = bumpDropped( - collector.status, - 'collector.preRun', - 'queue', - held.dropped, + if (!held.appended) { + const reason = toFailureReason( + new StepError( + `collector.preRunQueue full (queueMax ${max}): event refused before run`, + { code: 'queue-full' }, + ), ); - warnOverflowOnce( - collector.preRunQueue, + collector.status.failed++; + reportCause( + collector, collector.logger, - 'collector.preRunQueue overflow; oldest events dropped', - { buffer: 'queue', cap: max, droppedCount }, + { step: 'collector', code: reason.code }, + 'Push refused', + { + event: event.name, + retryable: reason.retryable, + error: reason.message, + }, ); - } else if (collector.preRunQueue.length < max) { - resetOverflowFlag(collector.preRunQueue); + return createPushResult({ + ok: false, + reason, + error: reason.message, + }); } collector.logger.debug('event held until run', { name: event.name, - dropped: held.dropped, }); return createPushResult({ ok: true }); } @@ -114,6 +123,9 @@ export function createPush( } = options; let respond = initialRespond; let partialEvent = event; + // A transformer of the source's next chain that failed in one copy + // of a fork while another copy goes on. + let preChainFailure: Elb.PushResult | undefined; // Build filtered destination set if include/exclude specified const filteredDests = @@ -133,9 +145,19 @@ export function createPush( collector, ); - // Check ignore flag + // A rule that ignores the event: deliberately not forwarded, a + // drop the result and observers see. if (processed.ignore) { - return createPushResult({ ok: true }); + const at = id ? `source.${id}` : 'collector.push'; + collector.logger.debug('Event ignored by source mapping', { + at, + mappingKey: processed.mappingKey, + }); + emitCollectorDrop(collector, processed.event, pipelineIngest, { + reason: 'ignored', + at, + }); + return createPushResult({ ok: true, dropped: true }); } // Check consent requirements @@ -185,6 +207,13 @@ export function createPush( chainPath, ); + // A transformer failed and no copy goes on: the push fails. + const chainFailure = chainResult.failure + ? chainFailureResult(chainResult.failure) + : undefined; + if (chainFailure && chainResult.copies.length === 0) + return chainFailure; + // Dropped or stopped: the event never reaches the collector. if (chainResult.copies.length === 0) { collector.logger.debug( @@ -248,11 +277,17 @@ export function createPush( sourceStatus.duration += Date.now() - pushStart; } - return forkResults[0] ?? createPushResult({ ok: true }); + // Every copy's outcome is reported, worst first. + const [first, ...rest] = forkResults; + return mergePushResults( + first, + chainFailure ? [...rest, chainFailure] : rest, + ); } partialEvent = chainResult.copies[0].event; pipelineIngest = chainResult.copies[0].ingest; + if (chainFailure) preChainFailure = chainFailure; } // Enrich into a full event (timing, source info, defaults) @@ -263,7 +298,7 @@ export function createPush( ); // Push to destinations with id and ingest - const result = await pushToDestinations( + const delivered = await pushToDestinations( collector, fullEvent, { @@ -288,7 +323,9 @@ export function createPush( sourceStatus.duration += Date.now() - pushStart; } - return result; + return preChainFailure + ? mergePushResults(delivered, [preChainFailure]) + : delivered; }, (err: unknown) => { if (err instanceof FatalError) throw err; @@ -306,9 +343,20 @@ export function createPush( const sourceStatus = collector.status.sources[sourceId]; sourceStatus.rejected = (sourceStatus.rejected ?? 0) + 1; } - collector.logger.warn('invalid event rejected', { - error: err.message, - }); + // WARN (the producer should act, the pipeline is fine), one line + // per source per window: a client sending garbage cannot flood + // the log. + reportCause( + collector, + collector.logger, + { + step: sourceId ? `source.${sourceId}` : 'collector', + code: 'invalid', + }, + 'invalid event rejected', + { error: err.message }, + 'warn', + ); return createPushResult({ ok: false, invalid: true, @@ -316,17 +364,26 @@ export function createPush( }); } collector.status.failed++; - // Identify the event by NAME only. The log context is serialized - // into stderr, the error ring and the managed-run jsonl sink, so it - // stays primitive and low-cardinality: the full event (user, - // consent, data) and the raw ingest payload would be a PII egress, - // and per-event values would make every failure look distinct to - // the ring's message dedup. - collector.logger.error('push failed', { - ...errorMeta(err), - event: event.name, + // A throwing rule `condition` (`mapping`) or any other fault: the + // push fails with its reason. The line names the event by NAME + // only, never the event or the ingest payload. + const reason = toFailureReason(err); + reportCause( + collector, + collector.logger, + { step: 'collector', code: reason.code, status: reason.status }, + options.id ? `push failed at source.${options.id}` : 'push failed', + { + event: event.name, + retryable: reason.retryable, + error: reason.message, + }, + ); + return createPushResult({ + ok: false, + reason, + error: reason.message, }); - return createPushResult({ ok: false }); }, )(); }, @@ -396,7 +453,7 @@ export function createPush( journey, ); errState.durationMs = finished - started; - errState.error = stepError(err); + errState.error = observerStepError(err); emitStep(collector, errState); throw err; } diff --git a/packages/collector/src/report-error.ts b/packages/collector/src/report-error.ts index 5806b6b09..ee53f4375 100644 --- a/packages/collector/src/report-error.ts +++ b/packages/collector/src/report-error.ts @@ -6,9 +6,8 @@ import type { StepKind, WalkerOS, } from '@walkeros/core'; -import { stepId } from '@walkeros/core'; -import { pushBounded, resetOverflowFlag, warnOverflowOnce } from './buffers'; -import { recordStepOutcome, resolveBreakerConfig } from './breaker'; +import { reportCause, stepId, toFailureReason } from '@walkeros/core'; +import { recordFailure } from './outcome'; /** * Maximum number of failed-push entries retained per destination DLQ before @@ -52,68 +51,6 @@ export function bumpDropped( return entry[buffer]!; } -/** - * Routes a single event/error pair to a destination's DLQ with the bounded - * write, overflow-warn-once, and full failure accounting (`destStatus.failed` - * + `collector.status.failed`). This mirrors the batch flush's `routeToDlq` - * for the single-event case: write to the DLQ AND bump the failed counters in - * one place. (The per-event in-band push path bumps `failed` later in the - * aggregation pass instead, so it does NOT call this; this helper is for the - * out-of-band `reportError(err, event)` seam that has no aggregation pass.) - */ -function routeEventToDlq( - collector: Collector.Instance, - destination: Destination.Instance, - destId: string, - event: WalkerOS.Event, - err: unknown, - logger: Logger.Instance, -): void { - const dlq = (destination.dlq = destination.dlq || []); - const dlqBound = { max: destination.config.dlqMax ?? DEFAULT_DLQ_MAX }; - const dlqResult = pushBounded(dlq, [event, err], dlqBound); - if (dlqResult.dropped > 0) { - const droppedCount = bumpDropped( - collector.status, - stepId('destination', destId), - 'dlq', - dlqResult.dropped, - ); - warnOverflowOnce( - dlq, - logger, - 'destination.dlq overflow; oldest entries dropped', - { - buffer: 'dlq', - destination: destId, - cap: dlqBound.max, - droppedCount, - }, - ); - } else if (dlq.length < dlqBound.max) { - resetOverflowFlag(dlq); - } - const destStatus = ensureDestStatus(collector, destId); - destStatus.failed++; - destStatus.dlqSize = dlq.length; - collector.status.failed++; - - // A connection-level error that DLQs a specific event is a transport - // failure: feed it to the circuit breaker so the gate picks it up. - // Presence-gated and keyed on the canonical stepId (matching the push gate). - const breakerConfig = resolveBreakerConfig(destination.config.breaker); - if (breakerConfig) { - const canonicalId = destination.config.id || destId; - recordStepOutcome( - collector.status.breakers, - stepId('destination', canonicalId), - 'transport-failure', - breakerConfig.threshold, - breakerConfig.cooldown, - ); - } -} - /** * Extract loggable metadata from an unknown error: the message plus, when * present, the error's `name` (skipped for the default 'Error', which adds no @@ -159,14 +96,11 @@ export function errorMeta(err: unknown): { * `logger.error`. Does NOT bump `failed` (no event lost at this instant; * counting it against `failed` would double-count the next push that hits * the broken writer and gets DLQ'd). - * - event-bearing (`reportError(err, event)`): route the event through the - * same DLQ + failure accounting an in-band push failure uses (for a - * destination). A step kind without a DLQ (or a destination not passed) - * still gets `failed`-counted and logged, never silently dropped. - * - * The with-event path deliberately funnels through the normal failure - * accounting so that when circuit-breaker accounting is later attached to that - * path, `reportError(err, event)` picks it up with no further wiring here. + * - event-bearing (`reportError(err, event)`): the event goes through the + * one outcome sink an in-band push failure uses (`recordFailure`: counters, + * DLQ, breaker, Observe, one cause line), for a destination. A step kind + * without a DLQ (or a destination not passed) still gets `failed`-counted + * and one cause line, never silently dropped. */ export function buildReportError( collector: Collector.Instance, @@ -180,16 +114,26 @@ export function buildReportError( try { if (event) { if (destination) { - routeEventToDlq(collector, destination, id, event, err, logger); + recordFailure(collector, destination, id, [event], err, { + message: 'report error', + }); } else { // No DLQ buffer for this step kind: still account + surface the // lost event so it is never silent. collector.status.failed++; + const reason = toFailureReason(err); + reportCause( + collector, + logger, + { step: key, code: reason.code, status: reason.status }, + 'report error', + { + event: event.name, + retryable: reason.retryable, + error: reason.message, + }, + ); } - logger.error('report error', { - ...errorMeta(err), - event: event.name, - }); return; } diff --git a/packages/collector/src/shutdown.ts b/packages/collector/src/shutdown.ts index 339d72865..c20ce5a25 100644 --- a/packages/collector/src/shutdown.ts +++ b/packages/collector/src/shutdown.ts @@ -1,4 +1,11 @@ -import type { Collector, Lifecycle, Logger } from '@walkeros/core'; +import type { + Collector, + Destination, + Elb, + Lifecycle, + Logger, +} from '@walkeros/core'; +import { createPushResult } from './destination'; const STEP_TIMEOUT = 5000; @@ -15,7 +22,7 @@ export async function destroyAllSteps( await destroyStepGroup(collector.sources, 'source', logger); // Phase 2: Destinations, flush pending batches first, then close connections. - await flushDestinationBatches(collector.destinations, logger); + await flushDestinationBatches(collector); await destroyStepGroup(collector.destinations, 'destination', logger); // Phase 3: Transformers (release caches) @@ -23,37 +30,78 @@ export async function destroyAllSteps( // Phase 4: Stores (release storage — last, so transformers can flush during their destroy) await destroyStepGroup(collector.stores, 'store', logger); + + // Last: the repeats the cause log counted and never printed. + collector.causeLog?.flush(); } -async function flushDestinationBatches( - destinations: Collector.Instance['destinations'], - rootLogger: Logger.Instance, -): Promise { - const promises = Object.entries(destinations).flatMap(([id, dest]) => { - const batches = dest.batches; - if (!batches) return []; - const logger = rootLogger.scope(dest.type || 'destination'); - return Object.values(batches).map(async (b) => { - let timer: ReturnType | undefined; - try { - await Promise.race([ - b.flush(), - new Promise((_, reject) => { - timer = setTimeout( - () => - reject(new Error(`destination '${id}' batch flush timed out`)), - STEP_TIMEOUT, - ); - }), - ]); - } catch (err) { - logger.error(`destination '${id}' batch flush failed: ${err}`); - } finally { - if (timer) clearTimeout(timer); +/** + * Flush every destination's pending batches now, through each batch's own + * flush (the one definition `walker shutdown` and `walker flush` share). A + * failed flush is failed through the outcome sink like any batch. Answers a + * destination with pending entries as `done`, or `failed` with the reason; + * a destination with nothing pending is not named. The collector keeps + * running. + */ +export async function flushDestinationBatches( + collector: Collector.Instance, +): Promise { + const done: Record = {}; + const failed: Record = {}; + + const flushes = Object.entries(collector.destinations).map( + async ([id, dest]) => { + const batches = Object.values(dest.batches || {}); + const pending = batches.reduce( + (sum, batch) => sum + batch.batched.entries.length, + 0, + ); + const ref: Destination.Ref = { type: dest.type || 'unknown' }; + const outcomes = await Promise.all( + batches.map(async (batch) => { + let timer: ReturnType | undefined; + try { + return await Promise.race([ + batch.flush(), + new Promise((_, reject) => { + timer = setTimeout( + () => + reject( + new Error(`destination '${id}' batch flush timed out`), + ), + STEP_TIMEOUT, + ); + }), + ]); + } catch (err) { + // The flush did not settle in time (it may still land later): + // named in the result, one ERROR for this destination. + const message = err instanceof Error ? err.message : String(err); + ref.reason = { code: 'timeout', message, retryable: true }; + collector.logger + .scope(dest.type || 'destination') + .error(`destination '${id}' batch flush failed: ${message}`); + return undefined; + } finally { + if (timer) clearTimeout(timer); + } + }), + ); + if (!pending) return; + for (const outcome of outcomes) { + if (outcome && outcome.reason && !ref.reason) + ref.reason = outcome.reason; } - }); + if (ref.reason) failed[id] = ref; + else done[id] = ref; + }, + ); + await Promise.all(flushes); + + return createPushResult({ + ...(Object.keys(done).length && { done }), + ...(Object.keys(failed).length && { failed }), }); - await Promise.allSettled(promises); } async function destroyStepGroup< diff --git a/packages/collector/src/source.ts b/packages/collector/src/source.ts index e4f6448e4..5db14db6f 100644 --- a/packages/collector/src/source.ts +++ b/packages/collector/src/source.ts @@ -21,14 +21,20 @@ import { storeCache, applyUpdate, createMappingRoot, + getMappingNoOpMessage, + getMappingNoOpPaths, stepId, } from '@walkeros/core'; -import { runTransformerChain } from './transformer'; +import { checkChainTargets, runTransformerChain } from './transformer'; import { compileStepState, runStepState, warnStateOff } from './state'; import { buildReportError, errorMeta } from './report-error'; import { isStateDelivery, shouldDeliver, setMark } from './on'; import { reconcilePending } from './pending'; -import { createPushResult } from './destination'; +import { + chainFailureResult, + createPushResult, + mergePushResults, +} from './destination'; import { emitCollectorDrop } from './observerEmit'; import { getCacheStore } from './cache'; import { @@ -133,6 +139,20 @@ export async function initSource( terminus, } = sourceDefinition; + // Registration checks, once at ERROR (the source still runs): mapping + // keys that do nothing at a source (behind the `__WALKEROS_VALIDATE__` + // build flag; CLI bundles check at build time), and the transformers its + // chains name. + if (typeof __WALKEROS_VALIDATE__ === 'undefined' || __WALKEROS_VALIDATE__) { + const noOps = getMappingNoOpPaths('source', config, 'config'); + if (noOps.length) + collector.logger + .scope('source') + .error(`Source ${sourceId}: ${getMappingNoOpMessage('source', noOps)}`); + } + checkChainTargets(collector, `source.${sourceId}.before`, before); + checkChainTargets(collector, `source.${sourceId}.next`, next); + // Compile declarative state entries once. For a source, all state entries // run before handing the event to `collector.push` (get enriches the event // the collector receives, set stashes from it), so they execute together in @@ -200,6 +220,9 @@ export async function initSource( let copies: Transformer.ChainCopy[] = [ { event: identified, ingest: scope.ingest }, ]; + // A transformer of the before chain that failed (with no copy left, or + // in one copy of a fork): the push fails. + let beforeFailure: Elb.PushResult | undefined; // Run source.before chain (consent-exempt, pre-source preprocessing). // The route resolves hop by hop with `{ ingest, event }`, the event being @@ -215,6 +238,10 @@ export async function initSource( scope.respond, chainPath, ); + if (beforeResult.failure) { + beforeFailure = chainFailureResult(beforeResult.failure); + if (beforeResult.copies.length === 0) return beforeFailure; + } if (beforeResult.copies.length === 0) { // Dropped or stopped: the event never reaches the collector. emitCollectorDrop(collector, identified, scope.ingest, { @@ -248,6 +275,8 @@ export async function initSource( cacheStore, cacheContext, // no per-step prefix — cache keys honor user-provided namespace only + undefined, + collector, ); if (cacheResult) { @@ -341,18 +370,26 @@ export async function initSource( // Push each event independently through the post-before pipeline. The // source's `next` route travels as the pre-collector chain and resolves - // per event, after the source's state, inside `collector.push`. - let pushResult: Elb.PushResult = { ok: true } as Elb.PushResult; + // per event, after the source's state, inside `collector.push`. Every + // copy's outcome is reported, worst first. + const results: Elb.PushResult[] = []; for (const copy of copies) { - pushResult = await terminalPush(copy.event, { - ...options, - id: sourceId, - ingest: copy.ingest, - respond: scope.respond, - mapping: config, - preChain: next, - }); + results.push( + await terminalPush(copy.event, { + ...options, + id: sourceId, + ingest: copy.ingest, + respond: scope.respond, + mapping: config, + preChain: next, + }), + ); } + const [first, ...rest] = results; + const pushResult = mergePushResults( + first ?? createPushResult({ ok: true }), + beforeFailure ? [...rest, beforeFailure] : rest, + ); // Wait for any deferred MISS update work to land on the source's // respond sender before returning control to the source. This diff --git a/packages/collector/src/state.ts b/packages/collector/src/state.ts index 094640259..0ec7026f1 100644 --- a/packages/collector/src/state.ts +++ b/packages/collector/src/state.ts @@ -38,7 +38,8 @@ export function runStepState( /** * A build without state ignores a step's `state` (e.g. on a destination - * added at runtime): warned, never silent. Stays in that lean bundle. + * added at runtime): an ERROR (a config value without effect), never + * silent. Stays in that lean bundle. */ export function warnStateOff( collector: Collector.Instance, @@ -49,5 +50,5 @@ export function warnStateOff( !(typeof __WALKEROS_STATE__ === 'undefined' || __WALKEROS_STATE__) && state !== undefined ) - collector.logger.warn(`state: not in this build, ${step} state ignored`); + collector.logger.error(`state: not in this build, ${step} state ignored`); } diff --git a/packages/collector/src/store.ts b/packages/collector/src/store.ts index d4e19bca9..a3ab380af 100644 --- a/packages/collector/src/store.ts +++ b/packages/collector/src/store.ts @@ -1,7 +1,7 @@ import type { Cache, Collector, Store } from '@walkeros/core'; import { emitStep, useHooks } from '@walkeros/core'; import { createCacheStore } from './cache-store'; -import { stepError, stepState } from './observerEmit'; +import { observerStepError, stepState } from './observerEmit'; import { wrapStoreWithCache } from './store-cache-wrapper'; import { buildReportError } from './report-error'; @@ -94,7 +94,7 @@ function applyStoreHooks( ); errState.durationMs = finished - started; errState.meta = { op: 'get', key }; - errState.error = stepError(err); + errState.error = observerStepError(err); emitStep(collector, errState); throw err; } @@ -139,7 +139,7 @@ function applyStoreHooks( ); errState.durationMs = finished - started; errState.meta = { op: 'set', key }; - errState.error = stepError(err); + errState.error = observerStepError(err); emitStep(collector, errState); throw err; } @@ -177,7 +177,7 @@ function applyStoreHooks( ); errState.durationMs = finished - started; errState.meta = { op: 'delete', key }; - errState.error = stepError(err); + errState.error = observerStepError(err); emitStep(collector, errState); throw err; } diff --git a/packages/collector/src/timeout.ts b/packages/collector/src/timeout.ts index 5288bce01..36e6190aa 100644 --- a/packages/collector/src/timeout.ts +++ b/packages/collector/src/timeout.ts @@ -1,3 +1,5 @@ +import { StepError } from '@walkeros/core'; + /** * Default per-destination delivery timeout in ms. Applied when a destination's * `config.timeout` is `0` or undefined. A hung delivery is converted into a @@ -17,13 +19,14 @@ export function resolveDestinationTimeout(timeout?: number): number { } /** - * Error thrown when a destination delivery does not settle within its timeout. - * The dedicated `name` lets DLQ consumers discriminate a timeout from a - * destination-thrown error without substring matching the message. + * Error thrown when a destination delivery does not settle within its timeout: + * a `StepError` with code `timeout` (retryable). The dedicated `name` lets DLQ + * consumers discriminate a timeout from a destination-thrown error without + * substring matching the message. */ -export class DestinationTimeoutError extends Error { +export class DestinationTimeoutError extends StepError { constructor(message: string) { - super(message); + super(message, { code: 'timeout' }); this.name = 'DestinationTimeoutError'; } } diff --git a/packages/collector/src/transformer.ts b/packages/collector/src/transformer.ts index 7dbb31ae4..cf478e86a 100644 --- a/packages/collector/src/transformer.ts +++ b/packages/collector/src/transformer.ts @@ -48,6 +48,7 @@ import { isObject, startChain, stepId, + tryCatch, tryCatchAsync, useHooks, compileCache, @@ -58,11 +59,16 @@ import { processEventMapping, getTransformerMapping, getTransformerMappingWarnings, + getRouteGraph, + reportCause, + StepError, + toFailureReason, } from '@walkeros/core'; -import { journeyFields, stepError, stepState } from './observerEmit'; +import { journeyFields, observerStepError, stepState } from './observerEmit'; import { getCacheStore } from './cache'; import { buildReportError, errorMeta } from './report-error'; import { compileStepState, runStepState, warnStateOff } from './state'; +import { asStepError } from './outcome'; /** * Extracts chain property from definition and merges into config. @@ -126,7 +132,7 @@ export async function initTransformers( 'Transformer', ); if (!validation.ok) { - collector.logger.warn( + collector.logger.error( `Transformer ${transformerId} invalid (${validation.code}): ${validation.reason}. Skipping.`, ); continue; @@ -189,10 +195,13 @@ export async function initTransformers( // Declarative mapping: `config.mapping` wins over the step level, like // `state`. Fields that do nothing here, or a mapping next to `code`, - // are warned about once per init. + // are reported once per init at ERROR (a config value without effect). + // Build flag: CLI bundles check this at build time and fold it out. const stepMapping = getTransformerMapping(transformerDef); - for (const warning of getTransformerMappingWarnings(transformerDef)) { - collector.logger.warn(`Transformer ${transformerId}: ${warning}`); + if (typeof __WALKEROS_VALIDATE__ === 'undefined' || __WALKEROS_VALIDATE__) { + for (const warning of getTransformerMappingWarnings(transformerDef)) { + collector.logger.error(`Transformer ${transformerId}: ${warning}`); + } } // Synthesize a passthrough instance when `code` is absent. @@ -427,12 +436,90 @@ export async function transformerPush( journey, ); errState.durationMs = finished - started; - errState.error = stepError(err); + errState.error = observerStepError(err, 'transformer'); emitStep(collector, errState); throw err; } } +/** Transformer ids already reported as missing, per collector. */ +const missingReported = new WeakMap>(); + +/** + * A chain names a transformer that does not exist: one ERROR per id per + * collector, at registration (`checkChainTargets`) or at the first event + * that meets it (a route swapped in at runtime), never per event. The copy + * goes on without it. + */ +function reportMissingTransformer( + collector: Collector.Instance, + id: string, + at: string, +): void { + let reported = missingReported.get(collector); + if (!reported) { + reported = new Set(); + missingReported.set(collector, reported); + } + if (reported.has(id)) return; + reported.add(id); + collector.logger.error( + `Transformer not found: ${id} (named in ${at}); the chain goes on without it`, + ); +} + +/** + * Registration check of a chain: every transformer a route can reach must + * exist. Reports each missing id once (see `reportMissingTransformer`). + */ +export function checkChainTargets( + collector: Collector.Instance, + at: string, + route: Transformer.Route | undefined, +): void { + if (route === undefined) return; + const transformers = collector.transformers || {}; + // A route that does not compile fails where it runs, with its own error. + const nodes = tryCatch(getRouteGraph, () => [])(route, transformers); + for (const node of nodes) { + for (const id of node.targets) { + if (!transformers[id]) reportMissingTransformer(collector, id, at); + } + } +} + +/** + * Report a transformer that failed (threw, or could not start) and hand the + * failure to the chain result: counted in `status.failed`, one cause line + * under the transformer's step. An unknown throw is code `transformer`. + */ +function transformerFailure( + collector: Collector.Instance, + transformerId: string, + transformerType: string | undefined, + err: unknown, + message: string, +): Transformer.ChainFailure { + const reason = toFailureReason(asStepError(err, 'transformer')); + collector.status.failed++; + reportCause( + collector, + collector.logger, + { + step: stepId('transformer', transformerId), + code: reason.code, + status: reason.status, + }, + message, + { + type: transformerType, + retryable: reason.retryable, + error: reason.message, + }, + ); + return { by: transformerId, reason }; +} + /** * Clone an ingest for an independent copy (a fork). The top level and * `_meta` are shallow-copied and `_meta.path` is duplicated, so the path @@ -589,10 +676,12 @@ async function runForks( let lastRespond = respond; let allStopped = results.length > 0; let droppedBy: string | undefined; + let failure: Transformer.ChainFailure | undefined; results.forEach((result, index) => { if (copies[index].respond !== undefined && result.respond) lastRespond = result.respond; if (!result.stopped) allStopped = false; + if (!failure && result.failure) failure = result.failure; // A dropped or stopped copy (route stop or cache.stop halt) ends here. if (result.stopped || result.copies.length === 0) { if (droppedBy === undefined) droppedBy = result.droppedBy; @@ -607,9 +696,14 @@ async function runForks( respond: lastRespond, ...(allStopped ? { stopped: true as const } : {}), ...(droppedBy !== undefined ? { droppedBy } : {}), + ...(failure ? { failure } : {}), }; } - return finished(survivors, lastRespond); + // The copies that finished go on; one that failed still fails the push. + return { + ...finished(survivors, lastRespond), + ...(failure ? { failure } : {}), + }; } /** Walks one copy to its end: loops `advanceChain`, runs each member. */ @@ -712,7 +806,11 @@ async function runMember( }); if (!transformer) { - collector.logger.warn(`Transformer not found: ${transformerName}`); + reportMissingTransformer( + collector, + transformerName, + chainContext ?? 'a chain', + ); return goOn(); } @@ -740,26 +838,39 @@ async function runMember( ingest._meta.hops++; ingest._meta.path.push(transformerName); - // Initialize transformer if needed. The wrap surfaces a thrown init - // (misconfiguration, missing env, etc.) with full cause via the scoped - // logger and counts it on `status.failed`. A non-throw `false` return - // from transformerInit itself flows through the same early return below. + // Initialize transformer if needed. A thrown init (misconfiguration, + // missing env) and an init that returned `false` (bundled code is not + // typechecked; runtime guard) both fail the copy: counted on + // `status.failed`, one cause line, and the push answers `ok: false`. + let initError: unknown; const isInitialized = await tryCatchAsync( transformerInit, (err: unknown): boolean => { if (err instanceof FatalError) throw err; - collector.status.failed++; - collector.logger - .scope(`transformer:${transformer.type || 'unknown'}`) - .error('transformer init failed', { - transformer: transformerName, - ...errorMeta(err), - }); + initError = err; return false; }, )(collector, transformer, transformerName); - if (!isInitialized) return halt({ copies: [], respond: currentRespond }); + if (!isInitialized) { + const err = + initError ?? + new StepError( + 'init returned false: throw a StepError that names the missing setting', + { code: 'not-ready' }, + ); + return halt({ + copies: [], + respond: currentRespond, + failure: transformerFailure( + collector, + transformerName, + transformer.type, + err, + 'transformer init failed', + ), + }); + } // Path-specific mock check (takes precedence) if ( @@ -792,6 +903,8 @@ async function runMember( compiledTCache, tCacheStore, createMappingRoot(ingest, processedEvent), + undefined, + collector, ); if (cacheResult?.status === 'HIT' && cacheResult.value) { @@ -876,13 +989,18 @@ async function runMember( ); } - // Run the transformer + // Run the transformer. A throw is a failure, not a drop: the copy ends + // and the push answers `ok: false` naming the transformer. + let failure: Transformer.ChainFailure | undefined; const result = await tryCatchAsync(transformerPush, (err) => { - collector.status.failed++; - collector.logger - .scope(`transformer:${transformer.type || 'unknown'}`) - .error('Push failed', errorMeta(err)); - return false as const; // Stop chain on error + failure = transformerFailure( + collector, + transformerName, + transformer.type, + err, + 'Push failed', + ); + return undefined; })( collector, transformer, @@ -892,7 +1010,9 @@ async function runMember( currentRespond, ); - // Returned false, or threw and was converted to false above. + if (failure) return halt({ copies: [], respond: currentRespond, failure }); + + // Returned false: a deliberate drop. if (result === false) { return halt({ copies: [], diff --git a/packages/collector/src/types/collector.ts b/packages/collector/src/types/collector.ts index afd5bdced..b6751da20 100644 --- a/packages/collector/src/types/collector.ts +++ b/packages/collector/src/types/collector.ts @@ -24,6 +24,7 @@ export type CommandTypes = | 'Custom' | 'Destination' | 'Elb' + | 'Flush' | 'Globals' | 'Hook' | 'Init' diff --git a/packages/core/src/__tests__/causeLog.test.ts b/packages/core/src/__tests__/causeLog.test.ts new file mode 100644 index 000000000..ffb18511f --- /dev/null +++ b/packages/core/src/__tests__/causeLog.test.ts @@ -0,0 +1,148 @@ +import { createCauseLog, reportCause } from '../causeLog'; + +type Line = { + level: 'error' | 'warn'; + message: string; + context: Record; +}; + +function recorder() { + const lines: Line[] = []; + const logger = { + error(message: string, context: Record): void { + lines.push({ level: 'error', message, context }); + }, + warn(message: string, context: Record): void { + lines.push({ level: 'warn', message, context }); + }, + }; + return { lines, logger }; +} + +describe('createCauseLog', () => { + it('logs the first occurrence, counts repeats in the window, prints suppressed after it', () => { + let now = 0; + const { lines, logger } = recorder(); + const log = createCauseLog(logger, { windowMs: 60_000, now: () => now }); + const key = { step: 'destination.ga4', code: 'transport', status: 503 }; + log.report(key, 'Push failed', { event: 'page view' }); + log.report(key, 'Push failed', { event: 'page view' }); + log.report(key, 'Push failed', { event: 'page view' }); + expect(lines).toHaveLength(1); + expect(lines[0].level).toBe('error'); + expect(lines[0].message).toContain('status.destinations.ga4.failed'); + now = 61_000; + log.report(key, 'Push failed', { event: 'page view' }); + expect(lines).toHaveLength(2); + expect(lines[1].context.suppressed).toBe(2); + }); + + it('logs every occurrence of a key beyond the 100-key cap, never drops a line', () => { + const { lines, logger } = recorder(); + const log = createCauseLog(logger, { windowMs: 60_000, now: () => 0 }); + for (let i = 0; i < 100; i++) + log.report( + { step: `destination.d${i}`, code: 'transport' }, + 'Push failed', + {}, + ); + expect(lines).toHaveLength(100); + log.report( + { step: 'destination.d100', code: 'transport' }, + 'Push failed', + {}, + ); + log.report( + { step: 'destination.d100', code: 'transport' }, + 'Push failed', + {}, + ); + expect(lines).toHaveLength(102); + expect(lines[100].context.step).toBe('destination.d100'); + expect(lines[101].context.step).toBe('destination.d100'); + expect(lines[101].context.suppressed).toBeUndefined(); + expect(log.size()).toBe(100); + }); + + it('hands every line, the flushed summary included, to the logger with structured fields', () => { + const { lines, logger } = recorder(); + const log = createCauseLog(logger, { windowMs: 60_000, now: () => 0 }); + const key = { step: 'destination.ga4', code: 'transport', status: 503 }; + log.report(key, 'Push failed', { event: 'page view', retryable: true }); + log.report(key, 'Push failed', { event: 'page view', retryable: true }); + log.flush(); + log.flush(); + expect(lines).toHaveLength(2); + expect(lines[0].context).toEqual({ + step: 'destination.ga4', + code: 'transport', + status: 503, + event: 'page view', + retryable: true, + }); + expect(lines[1].context).toMatchObject({ + step: 'destination.ga4', + code: 'transport', + status: 503, + suppressed: 1, + }); + }); + + it('reduces an event to its name and drops every payload key', () => { + const { lines, logger } = recorder(); + const log = createCauseLog(logger, { windowMs: 60_000, now: () => 0 }); + const fullEvent = { + name: 'order complete', + data: { email: 'a@b.c' }, + user: { id: 'user-1' }, + globals: { pagegroup: 'checkout' }, + }; + log.report({ step: 'destination.d', code: 'mapping' }, 'mapping failed', { + event: fullEvent, + data: { email: 'a@b.c' }, + ingest: { ip: '127.0.0.1' }, + }); + expect(lines[0].context.event).toBe('order complete'); + const text = JSON.stringify(lines); + expect(text).not.toContain('a@b.c'); + expect(text).not.toContain('user-1'); + expect(text).not.toContain('checkout'); + expect(text).not.toContain('127.0.0.1'); + }); + + it('writes a WARN cause through warn, rate limited the same way', () => { + const { lines, logger } = recorder(); + const log = createCauseLog(logger, { windowMs: 60_000, now: () => 0 }); + const key = { step: 'source.express', code: 'invalid' }; + log.report(key, 'invalid event rejected', {}, 'warn'); + log.report(key, 'invalid event rejected', {}, 'warn'); + expect(lines).toHaveLength(1); + expect(lines[0].level).toBe('warn'); + expect(lines[0].message).toContain('status.sources.express.rejected'); + }); +}); + +describe('reportCause', () => { + it('writes one filtered line through the logger when the collector has no cause log', () => { + const { lines, logger } = recorder(); + reportCause( + undefined, + logger, + { step: 'store.s', code: 'transport', status: 403 }, + 'store read failed', + { error: new Error('HTTP 403'), data: { email: 'a@b.c' } }, + ); + expect(lines).toEqual([ + { + level: 'error', + message: 'store read failed', + context: { + step: 'store.s', + code: 'transport', + status: 403, + error: 'HTTP 403', + }, + }, + ]); + }); +}); diff --git a/packages/core/src/__tests__/mapping-error-visibility.test.ts b/packages/core/src/__tests__/mapping-error-visibility.test.ts index db0bb7890..c573b42bb 100644 --- a/packages/core/src/__tests__/mapping-error-visibility.test.ts +++ b/packages/core/src/__tests__/mapping-error-visibility.test.ts @@ -3,13 +3,11 @@ import { createMockLogger } from '../mockLogger'; import { getMappingValue } from '../mapping'; /** - * Mapping error visibility: throws inside user-supplied callbacks - * (condition, fn, validate) MUST log via the scoped logger but MUST - * NOT increment `collector.status.failed` (Category B - user code). - * - * The outer `processMappingValue` wrap in `getMappingValue` (Category A) - * MUST log AND increment `status.failed` for internal walkerOS pipeline - * failures. + * Mapping error visibility: a value whose user code throws (condition, fn, + * validate), whose resolution throws, or that has no operator is dropped and + * reported once through the cause log, with the event's name and never the + * event. It never increments `collector.status.failed`: the event is still + * delivered with its other values. */ function createTestCollector(): Collector.Instance { @@ -51,8 +49,8 @@ function createTestCollector(): Collector.Instance { } as unknown as Collector.Instance; } -describe('mapping condition throws (Category B)', () => { - test('logs via scoped logger and does NOT increment status.failed', async () => { +describe('mapping condition throws', () => { + test('reports the value and does NOT increment status.failed', async () => { const collector = createTestCollector(); const event: WalkerOS.DeepPartialEvent = { name: 'page view' }; @@ -73,15 +71,17 @@ describe('mapping condition throws (Category B)', () => { // Find the error log somewhere in the logger tree (root or scoped). const errorCall = findLoggerError(collector, 'mapping condition failed'); - expect(errorCall).toBeDefined(); - expect(errorCall?.[1]).toEqual( - expect.objectContaining({ error: expect.any(Error) }), - ); + expect(errorCall?.[1]).toEqual({ + step: 'mapping', + code: 'mapping', + event: 'page view', + error: 'condition boom', + }); }); }); -describe('mapping fn throws (Category B)', () => { - test('logs via scoped logger and does NOT increment status.failed', async () => { +describe('mapping fn throws', () => { + test('reports the value and does NOT increment status.failed', async () => { const collector = createTestCollector(); const event: WalkerOS.DeepPartialEvent = { name: 'page view' }; @@ -98,15 +98,17 @@ describe('mapping fn throws (Category B)', () => { expect(collector.status.failed).toBe(0); const errorCall = findLoggerError(collector, 'mapping fn failed'); - expect(errorCall).toBeDefined(); - expect(errorCall?.[1]).toEqual( - expect.objectContaining({ error: expect.any(Error) }), - ); + expect(errorCall?.[1]).toEqual({ + step: 'mapping', + code: 'mapping', + event: 'page view', + error: 'fn boom', + }); }); }); -describe('mapping validate throws (Category B)', () => { - test('logs via scoped logger and does NOT increment status.failed', async () => { +describe('mapping validate throws', () => { + test('reports the value and does NOT increment status.failed', async () => { const collector = createTestCollector(); const event: WalkerOS.DeepPartialEvent = { name: 'page view', @@ -127,15 +129,17 @@ describe('mapping validate throws (Category B)', () => { expect(collector.status.failed).toBe(0); const errorCall = findLoggerError(collector, 'mapping validate failed'); - expect(errorCall).toBeDefined(); - expect(errorCall?.[1]).toEqual( - expect.objectContaining({ error: expect.any(Error) }), - ); + expect(errorCall?.[1]).toEqual({ + step: 'mapping', + code: 'mapping', + event: 'page view', + error: 'validate boom', + }); }); }); -describe('processMappingValue outer wrap (Category A)', () => { - test('logs and increments status.failed when an internal throw escapes', async () => { +describe('processMappingValue outer wrap', () => { + test('reports the dropped value when an internal throw escapes, the event goes on', async () => { const collector = createTestCollector(); const event: WalkerOS.DeepPartialEvent = { name: 'page view' }; @@ -164,14 +168,48 @@ describe('processMappingValue outer wrap (Category A)', () => { // Outer wrap catches; downstream undefined. expect(result).toBeUndefined(); - // Internal pipeline failures count. - expect(collector.status.failed).toBe(1); + // A dropped value is not a failed event. + expect(collector.status.failed).toBe(0); - const errorCall = findLoggerError(collector, 'mapping processing failed'); - expect(errorCall).toBeDefined(); - expect(errorCall?.[1]).toEqual( - expect.objectContaining({ error: expect.any(Error) }), + const errorCall = findLoggerError(collector, 'mapping value failed'); + expect(errorCall?.[1]).toEqual({ + step: 'mapping', + code: 'mapping', + event: 'page view', + error: 'internal boom', + }); + }); +}); + +describe('a value object without an operator (E14)', () => { + test('drops that value instead of resolving to the whole input, keeps the others', async () => { + const collector = createTestCollector(); + const event: WalkerOS.DeepPartialEvent = { + name: 'order complete', + data: { total: 42, currency: 'EUR' }, + user: { email: 'person@example.com' }, + }; + + // A plain object of paths, the shape a hand-written config arrives in. + const order: Record = { total: 'data.total' }; + const mapping: Mapping.Value = { + map: { order, currency: 'data.currency' }, + }; + + const result = await getMappingValue(event, mapping, { collector, event }); + + expect(result).toEqual({ currency: 'EUR' }); + expect(collector.status.failed).toBe(0); + const errorCall = findLoggerError( + collector, + 'mapping value has no operator (total): wrap the fields in map, or use key, value, fn, loop or set', ); + expect(errorCall?.[1]).toEqual({ + step: 'mapping', + code: 'mapping', + event: 'order complete', + }); + expect(JSON.stringify(errorCall)).not.toContain('person@example.com'); }); }); diff --git a/packages/core/src/__tests__/mapping-no-ops.test.ts b/packages/core/src/__tests__/mapping-no-ops.test.ts new file mode 100644 index 000000000..33a11fd8f --- /dev/null +++ b/packages/core/src/__tests__/mapping-no-ops.test.ts @@ -0,0 +1,54 @@ +import type { MappingPosition } from '../mapping-no-ops'; +import { getMappingNoOpMessage, getMappingNoOpPaths } from '../mapping-no-ops'; + +describe('getMappingNoOpPaths', () => { + const mapping = { + data: { map: {} }, + include: ['data'], + consent: { marketing: true }, + policy: {}, + mapping: { + page: { + view: { data: 'data', extend: {}, name: 'pv', ignore: false }, + click: [{ silent: true }, { batch: 10, condition: () => true }], + }, + }, + }; + + it.each<[MappingPosition, string[]]>([ + [ + 'transformer', + [ + 'config.consent', + 'config.include', + 'config.data', + 'config.mapping.page.view.extend', + 'config.mapping.page.view.data', + 'config.mapping.page.click[0].silent', + 'config.mapping.page.click[1].batch', + ], + ], + [ + 'source', + [ + 'config.data', + 'config.include', + 'config.mapping.page.view.data', + 'config.mapping.page.view.extend', + 'config.mapping.page.click[0].silent', + 'config.mapping.page.click[1].batch', + ], + ], + ['destination', ['config.mapping.page.view.extend']], + ])('lists what does nothing at the %s position', (position, paths) => { + expect(getMappingNoOpPaths(position, mapping, 'config')).toEqual(paths); + }); + + it('names the paths and what applies at the position', () => { + expect( + getMappingNoOpMessage('source', ['config.mapping.page.view.data']), + ).toBe( + "`config.mapping.page.view.data` does nothing at the source position; only `consent`, `policy` and a rule's `condition`, `policy`, `name` and `ignore` apply.", + ); + }); +}); diff --git a/packages/core/src/__tests__/mapping.test.ts b/packages/core/src/__tests__/mapping.test.ts index 149b543cc..74049aa7a 100644 --- a/packages/core/src/__tests__/mapping.test.ts +++ b/packages/core/src/__tests__/mapping.test.ts @@ -88,6 +88,45 @@ describe('getMappingEvent', () => { }); }); + test('tries entity action, entity *, * action, * * in that order (E22)', async () => { + const R: Mapping.Rule = { name: 'r' }; + expect( + await getMappingEvent( + { name: 'product click' }, + { product: { view: {} }, '*': { click: R } }, + ), + ).toStrictEqual({ eventMapping: R, mappingKey: '* click' }); + + const failing = jest.fn(() => false); + const S: Mapping.Rule = { name: 's' }; + const T: Mapping.Rule = { name: 't' }; + expect( + await getMappingEvent( + { name: 'product click' }, + { + product: { click: { condition: failing }, '*': S }, + '*': { '*': T }, + }, + ), + ).toStrictEqual({ eventMapping: S, mappingKey: 'product *' }); + expect(failing).toHaveBeenCalledTimes(1); + + // A tier whose async conditions all fail falls through as well, and + // every condition runs once. + const gated = jest.fn(async () => false); + const actionWildcard: Mapping.Rule = { name: 'action' }; + expect( + await getMappingEvent( + { name: 'product click' }, + { + product: { click: { condition: gated }, '*': { condition: gated } }, + '*': { click: actionWildcard, '*': T }, + }, + ), + ).toStrictEqual({ eventMapping: actionWildcard, mappingKey: '* click' }); + expect(gated).toHaveBeenCalledTimes(2); + }); + test('an unmatched * * rule runs its condition once', async () => { const condition = jest.fn(() => false); const mapping: Mapping.Rules = { '*': { '*': { condition } } }; @@ -217,7 +256,7 @@ describe('getMappingEvent', () => { ]; test.each(throwing)( - 'a %s throwing condition is logged and counts as no match', + 'a %s throwing condition fails the event instead of falling through', async (_, condition) => { const collector = createMockCollector(); const fallback: Mapping.Rule = { name: 'fallback' }; @@ -225,12 +264,18 @@ describe('getMappingEvent', () => { order: { complete: [{ condition, name: 'broken' }, fallback] }, }; - const result = await getMappingEvent(event, mapping, collector); - expect(result.eventMapping).toBe(fallback); - expect(collector.logger.error).toHaveBeenCalledWith( - 'mapping rule condition failed', - { event, mappingKey: 'order complete', error: boom }, + const failure = await getMappingEvent(event, mapping, collector).then( + () => undefined, + (err: unknown) => err, ); + expect(failure).toBeInstanceOf(core.StepError); + expect(failure).toMatchObject({ + code: 'mapping', + retryable: false, + message: 'mapping rule condition failed at "order complete": boom', + }); + // The caller reports the failed event; the lookup logs nothing. + expect(collector.logger.error).not.toHaveBeenCalled(); }, ); diff --git a/packages/core/src/__tests__/state.test.ts b/packages/core/src/__tests__/state.test.ts index e0f480006..294609655 100644 --- a/packages/core/src/__tests__/state.test.ts +++ b/packages/core/src/__tests__/state.test.ts @@ -149,7 +149,7 @@ describe('applyState', () => { expect(store._data.size).toBe(0); }); - test('key that does not resolve warns and skips the entry', async () => { + test('key that does not resolve is reported at ERROR and skips the entry', async () => { const store = createMockStore(); const getStore = makeGetStore({ sessions: store }); const collector = createMockCollector({ stores: { sessions: store } }); @@ -175,9 +175,13 @@ describe('applyState', () => { ); expect(store._data.size).toBe(0); - expect(collector.logger.warn).toHaveBeenCalledWith( - '[state] key did not resolve', - { mode: 'set', store: 'sessions', key: 'event.user.session' }, + expect(collector.logger.error).toHaveBeenCalledWith( + '[state] set key did not resolve, entry skipped', + { + step: 'store.sessions', + code: 'mapping', + mappingKey: 'event.user.session', + }, ); }); @@ -662,7 +666,7 @@ describe('applyState resolves against { event, ingest }', () => { expect(await store.get('s1')).toBe('g1'); }); - test('a bare path no longer resolves and warns', async () => { + test('a bare path no longer resolves and is reported', async () => { const { store, getStore, collector } = setup(); await applyState( @@ -681,9 +685,9 @@ describe('applyState resolves against { event, ingest }', () => { ); expect(store._data.size).toBe(0); - expect(collector.logger.warn).toHaveBeenCalledWith( - '[state] key did not resolve', - { mode: 'set', store: 'registry', key: 'user.session' }, + expect(collector.logger.error).toHaveBeenCalledWith( + '[state] set key did not resolve, entry skipped', + { step: 'store.registry', code: 'mapping', mappingKey: 'user.session' }, ); }); diff --git a/packages/core/src/__tests__/stepError.test.ts b/packages/core/src/__tests__/stepError.test.ts new file mode 100644 index 000000000..ef65cdfc3 --- /dev/null +++ b/packages/core/src/__tests__/stepError.test.ts @@ -0,0 +1,91 @@ +import { StepError, toFailureReason } from '../stepError'; + +describe('StepError', () => { + it('carries code, status, retryable and the response body', () => { + const err = new StepError('Config settings apiKey missing', { + code: 'config', + }); + expect(err).toBeInstanceOf(Error); + expect(err.name).toBe('StepError'); + expect(err.code).toBe('config'); + expect(err.retryable).toBe(false); + const http = new StepError('HTTP 400', { + code: 'transport', + status: 400, + body: { error: 'bad' }, + }); + expect(http.body).toEqual({ error: 'bad' }); + expect(http.message).toBe('HTTP 400'); + }); + + it.each([ + 'transport', + 'timeout', + 'breaker-open', + 'blocked', + 'queue-full', + 'vendor-not-ready', + 'vendor-timeout', + ])('%s defaults to retryable', (code) => { + expect(new StepError('x', { code }).retryable).toBe(true); + }); + + it.each([ + 'config', + 'not-ready', + 'mapping', + 'transformer', + 'missing-input', + 'vendor-not-loaded', + 'a-code-of-its-own', + ])('%s defaults to not retryable', (code) => { + expect(new StepError('x', { code }).retryable).toBe(false); + }); + + it('lets the status, then an explicit option, override the code default', () => { + expect( + new StepError('x', { code: 'transport', status: 400 }).retryable, + ).toBe(false); + expect( + new StepError('x', { code: 'transport', status: 429 }).retryable, + ).toBe(true); + expect( + new StepError('x', { code: 'transport', status: 503 }).retryable, + ).toBe(true); + expect( + new StepError('x', { code: 'transport', status: 503, retryable: false }) + .retryable, + ).toBe(false); + }); + + it('maps any thrown value to a reason without payload or body', () => { + expect(toFailureReason(new Error('boom'))).toEqual({ + code: 'transport', + message: 'boom', + retryable: true, + }); + expect(toFailureReason(new StepError('late', { code: 'timeout' }))).toEqual( + { code: 'timeout', message: 'late', retryable: true }, + ); + expect( + toFailureReason( + new StepError('HTTP 400', { + code: 'transport', + status: 400, + body: { email: 'a@b.c' }, + }), + ), + ).toEqual({ + code: 'transport', + message: 'HTTP 400', + retryable: false, + status: 400, + }); + }); + + it('caps the message at 300 characters', () => { + expect(toFailureReason(new Error('a'.repeat(500))).message).toHaveLength( + 300, + ); + }); +}); diff --git a/packages/core/src/cache.ts b/packages/core/src/cache.ts index 2565196fb..9d4b87498 100644 --- a/packages/core/src/cache.ts +++ b/packages/core/src/cache.ts @@ -8,6 +8,9 @@ import { getMappingValue } from './mapping'; import { wrapCacheEnvelope, readCacheEnvelope } from './cache-envelope'; import { isStoreValue } from './store/codec'; import { isThenable } from './is'; +import { FatalError } from './fatalError'; +import { reportCause } from './causeLog'; +import { toFailureReason } from './stepError'; interface CompiledCacheRule { match: CompiledMatcher; @@ -56,11 +59,18 @@ export function compileCache(cache: Cache): CompiledCache { }; } +/** + * Look a cache rule's key up in its store. A read that throws (a refused or + * failed request, not a miss) is a MISS for that event, reported once per + * cause under the store's step through the collector's cause log. Without a + * collector there is nowhere to report, and the throw propagates. + */ export async function checkCache( compiled: CompiledCache, store: Store.Instance, context: Record, namespace?: string, + collector?: Collector.Instance, ): Promise { const rule = compiled.rules.find((r) => r.match(context)); if (!rule) return null; @@ -79,7 +89,26 @@ export async function checkCache( // Promise`). Always await so a Promise return from an // async store (Redis, fs, the cache wrapper) never lands in the HIT // path as the cached "value". - const decoded = readCacheEnvelope(await store.get(namespacedKey)); + let stored: StoreValue | undefined; + try { + stored = await store.get(namespacedKey); + } catch (err) { + if (!collector || err instanceof FatalError) throw err; + const reason = toFailureReason(err); + reportCause( + collector, + collector.logger, + { + step: `store.${compiled.storeId ?? '__cache'}`, + code: reason.code, + status: reason.status, + }, + 'cache read failed, treated as a miss', + { error: reason.message, retryable: reason.retryable }, + ); + return { status: 'MISS', key: namespacedKey, rule }; + } + const decoded = readCacheEnvelope(stored); if (decoded === undefined) return { status: 'MISS', key: namespacedKey, rule }; diff --git a/packages/core/src/causeLog.ts b/packages/core/src/causeLog.ts new file mode 100644 index 000000000..364db9e50 --- /dev/null +++ b/packages/core/src/causeLog.ts @@ -0,0 +1,230 @@ +import type { Collector } from './types'; +import { capMessage } from './stepError'; +import { isObject } from './is'; + +/** What a failure line is about: the step, the cause code and the HTTP status. */ +export interface CauseKey { + step: string; + code: string; + status?: number; +} + +/** Where cause lines go; `Logger.Instance` fits, and so does a test recorder. */ +export interface CauseLogger { + error(message: string, context: Record): void; + warn(message: string, context: Record): void; +} + +/** + * One line per cause: the first occurrence of a key is logged at once, + * repeats within the window are counted and printed as `suppressed: ` + * with the next line of that key or at `flush`. + */ +export interface CauseLog { + report( + key: CauseKey, + message: string, + context: Record, + level?: 'error' | 'warn', + ): void; + /** Print the pending `suppressed` counts (server shutdown). */ + flush(): void; + /** Number of keys held. */ + size(): number; +} + +export interface CauseLogOptions { + windowMs?: number; + maxKeys?: number; + now?: () => number; +} + +interface CauseEntry { + start: number; + suppressed: number; + message: string; + context: Record; + level: 'error' | 'warn'; +} + +/** + * Context keys a cause line may carry besides the key's own fields. Never + * event payloads, ingest values, response bodies, settings or `Error.cause`. + */ +const STRING_KEYS = ['type', 'mappingKey'] as const; +const NUMBER_KEYS = ['count', 'suppressed'] as const; + +/** + * The counter a cause's occurrences are counted in, named by its first line: + * from the step id (`destination.`: its `failed`; rejected input at + * `source.`: its `rejected`; otherwise `status.failed`), except for the + * causes that are counted as drops or not at all. + */ +function counterOf({ step, code }: CauseKey): string | undefined { + const [kind, ...rest] = step.split('.'); + const id = rest.join('.'); + // Evicted entries: the DLQ's were counted failed already, the queue's + // were never delivered; both are counted with the step's drops. + if (code === 'dlq-overflow') return `status.dropped["${step}"].dlq`; + if (code === 'on-overflow') return `status.dropped["${step}"].queue`; + // A failed on(consent) holds the destination; its events are reported + // when they fail. + if (code === 'consent-not-applied') return undefined; + // A failed store read or a dropped mapping value degrades an event that is + // still delivered: nothing counts it. + if (kind === 'store' || kind === 'mapping') return undefined; + if (kind === 'destination' && id) return `status.destinations.${id}.failed`; + if (code === 'invalid') + return kind === 'source' && id + ? `status.sources.${id}.rejected` + : undefined; + return 'status.failed'; +} + +/** An error-ish value as a capped message. */ +function errorText(value: unknown): string | undefined { + if (value === undefined) return undefined; + if (typeof value === 'string') return capMessage(value); + if (value instanceof Error) return capMessage(value.message); + if (isObject(value) && typeof value.message === 'string') + return capMessage(value.message); + return capMessage(String(value)); +} + +/** An `event` value reduced to its name. */ +function eventName(value: unknown): string | undefined { + if (typeof value === 'string') return value; + if (isObject(value) && typeof value.name === 'string') return value.name; + return undefined; +} + +/** + * The context a cause line carries: the key's fields plus the allowed keys + * of the given context. Everything else is dropped. + */ +export function causeContext( + key: CauseKey, + context: Record, +): Record { + const line: Record = { step: key.step, code: key.code }; + if (key.status !== undefined) line.status = key.status; + for (const name of STRING_KEYS) { + if (typeof context[name] === 'string') line[name] = context[name]; + } + for (const name of NUMBER_KEYS) { + if (typeof context[name] === 'number') line[name] = context[name]; + } + if (typeof context.retryable === 'boolean') + line.retryable = context.retryable; + const event = eventName(context.event); + if (event !== undefined) line.event = event; + const error = errorText(context.error); + if (error !== undefined) line.error = error; + return line; +} + +/** The first line of a cause names the counter its repeats are counted in. */ +function firstLine(key: CauseKey, message: string): string { + const counter = counterOf(key); + return counter + ? `${message} (repeats within a minute are counted in ${counter})` + : message; +} + +/** + * Creates the cause log of one collector. Rate limit per (step, code, + * status): one line per window, repeats counted. At most `maxKeys` keys are + * held; a key beyond the cap is logged on every occurrence, never dropped + * and never suppressed. No timers: windows are checked on report. + */ +export function createCauseLog( + logger: CauseLogger, + options: CauseLogOptions = {}, +): CauseLog { + const { windowMs = 60_000, maxKeys = 100, now = Date.now } = options; + const entries = new Map(); + + const write = ( + level: 'error' | 'warn', + message: string, + context: Record, + ): void => { + if (level === 'warn') logger.warn(message, context); + else logger.error(message, context); + }; + + return { + report(key, message, context, level = 'error') { + const id = `${key.step}\u0000${key.code}\u0000${key.status ?? ''}`; + const line = causeContext(key, context); + const entry = entries.get(id); + const at = now(); + + if (!entry) { + if (entries.size < maxKeys) { + entries.set(id, { + start: at, + suppressed: 0, + message, + context: line, + level, + }); + } + write(level, firstLine(key, message), line); + return; + } + + if (at - entry.start < windowMs) { + entry.suppressed++; + return; + } + + const suppressed = entry.suppressed; + entry.start = at; + entry.suppressed = 0; + entry.message = message; + entry.context = line; + entry.level = level; + write( + level, + firstLine(key, message), + suppressed > 0 ? { ...line, suppressed } : line, + ); + }, + + flush() { + for (const entry of entries.values()) { + if (entry.suppressed === 0) continue; + const suppressed = entry.suppressed; + entry.suppressed = 0; + write(entry.level, entry.message, { ...entry.context, suppressed }); + } + }, + + size() { + return entries.size; + }, + }; +} + +/** + * The one entry for every failure line: through the collector's cause log, + * or, without one (a hand-built collector), one filtered line through + * `logger`. + */ +export function reportCause( + collector: Collector.Instance | undefined, + logger: CauseLogger, + key: CauseKey, + message: string, + context: Record, + level: 'error' | 'warn' = 'error', +): void { + if (collector?.causeLog) { + collector.causeLog.report(key, message, context, level); + return; + } + const line = causeContext(key, context); + if (level === 'warn') logger.warn(message, line); + else logger.error(message, line); +} diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 9bf25c27e..c9604fef4 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -36,6 +36,20 @@ export * from './trim'; export * from './tryCatch'; export * from './fatalError'; export * from './invalidEventError'; +export { StepError, toFailureReason } from './stepError'; +export type { + FailureReason, + QueueReason, + QueuedReason, + StepErrorOptions, +} from './stepError'; +export { createCauseLog, reportCause } from './causeLog'; +export type { + CauseKey, + CauseLog, + CauseLogOptions, + CauseLogger, +} from './causeLog'; export * from './useHooks'; export * from './telemetry'; export * from './telemetryResolver'; @@ -101,6 +115,7 @@ export { export * from './state'; export * from './step-entry'; export * from './transformer-mapping'; +export * from './mapping-no-ops'; export * from './examples/formatOut'; export { REF_VAR_FULL, diff --git a/packages/core/src/logger.ts b/packages/core/src/logger.ts index 39b094c33..3381e6e16 100644 --- a/packages/core/src/logger.ts +++ b/packages/core/src/logger.ts @@ -10,15 +10,37 @@ import type { import { Level } from './types/logger'; /** - * Normalize an Error object into ErrorContext + * An error's `cause` reduced to its name and code: a cause can carry a + * request, a response body or settings, none of which belong in a log line. + */ +function normalizeCause( + cause: unknown, +): { name?: string; code?: string | number } | undefined { + if (typeof cause !== 'object' || cause === null) return undefined; + const reduced: { name?: string; code?: string | number } = {}; + if ('name' in cause && typeof cause.name === 'string') + reduced.name = cause.name; + if ( + 'code' in cause && + (typeof cause.code === 'string' || typeof cause.code === 'number') + ) + reduced.code = cause.code; + return reduced; +} + +/** + * Normalize an Error object into ErrorContext. Never copies a response body + * (`StepError.body`) and keeps only the cause's name and code. */ function normalizeError(err: Error): ErrorContext { - return { + const context: ErrorContext = { message: err.message, name: err.name, stack: err.stack, - cause: (err as Error & { cause?: unknown }).cause, }; + const cause = normalizeCause(err.cause); + if (cause) context.cause = cause; + return context; } /** @@ -173,31 +195,11 @@ function createLoggerInternal(config: InternalConfig): Instance { }; /** - * Log and throw - combines logging and throwing + * Throw only: the collector's catch writes the one line for the failure, + * so logging here as well would print every failure twice. */ - const logAndThrow = (message: string | Error, context?: unknown): never => { - // Always log errors regardless of level - const normalized = normalizeParams(message, context); - - if (handler) { - handler( - Level.ERROR, - normalized.message, - normalized.context, - scope, - defaultHandler, - ); - } else { - defaultHandler( - Level.ERROR, - normalized.message, - normalized.context, - scope, - ); - } - - // Throw with the message - throw new Error(normalized.message); + const throwOnly = (message: string | Error, context?: unknown): never => { + throw new Error(normalizeParams(message, context).message); }; return { @@ -205,7 +207,7 @@ function createLoggerInternal(config: InternalConfig): Instance { warn: (message, context) => log(Level.WARN, message, context), info: (message, context) => log(Level.INFO, message, context), debug: (message, context) => log(Level.DEBUG, message, context), - throw: logAndThrow, + throw: throwOnly, json: (data: unknown) => { if (jsonHandler) { jsonHandler(data); diff --git a/packages/core/src/mapping-no-ops.ts b/packages/core/src/mapping-no-ops.ts new file mode 100644 index 000000000..130411ae0 --- /dev/null +++ b/packages/core/src/mapping-no-ops.ts @@ -0,0 +1,109 @@ +import type { Mapping } from './types'; +import { isArray, isObject } from './is'; + +/** A position a mapping runs at. */ +export type MappingPosition = 'source' | 'transformer' | 'destination'; + +/** + * Mapping fields that do nothing at a position, one table for the three. + * - transformer: the mapping only mutates the event; config `policy` and a + * rule's `condition`, `policy`, `name` and `ignore` apply. + * - source: only the edited event goes on; config `consent`, `policy` and a + * rule's `condition`, `policy`, `name` and `ignore` apply. A rule's + * `consent` is not listed here. + * - destination: a rule's `extend` is merged only by a package onto its own + * shipped rules (`@walkeros/transformer-ga4`'s `settings.mapping`). + */ +export const MAPPING_NO_OPS: Record< + MappingPosition, + { + config: ReadonlyArray; + rule: ReadonlyArray; + } +> = { + transformer: { + config: ['consent', 'include', 'data'], + rule: [ + 'consent', + 'include', + 'remove', + 'batch', + 'settings', + 'extend', + 'data', + 'silent', + ], + }, + source: { + config: ['data', 'include'], + rule: [ + 'data', + 'include', + 'remove', + 'batch', + 'settings', + 'extend', + 'silent', + ], + }, + destination: { + config: [], + rule: ['extend'], + }, +}; + +/** What applies at a position, for the message that names a no-op. */ +const APPLIES: Record = { + transformer: + "only `policy` and a rule's `condition`, `policy`, `name` and `ignore` apply", + source: + "only `consent`, `policy` and a rule's `condition`, `policy`, `name` and `ignore` apply", + destination: + "a rule's `extend` is merged only by a package onto its own shipped rules; write the fields on the rule itself", +}; + +/** + * Dotted paths of the fields in `mapping` that do nothing at `position`, + * rooted at `field`. Rules in array form are indexed. + */ +export function getMappingNoOpPaths( + position: MappingPosition, + mapping: unknown, + field: string, +): string[] { + if (!isObject(mapping)) return []; + const { config, rule: ruleKeys } = MAPPING_NO_OPS[position]; + const paths: string[] = []; + for (const key of config) { + if (mapping[key] !== undefined) paths.push(`${field}.${key}`); + } + + const rules = mapping.mapping; + if (!isObject(rules)) return paths; + for (const [entity, actions] of Object.entries(rules)) { + if (!isObject(actions)) continue; + for (const [action, value] of Object.entries(actions)) { + const list: unknown[] = isArray(value) ? value : [value]; + list.forEach((rule, index) => { + if (!isObject(rule)) return; + const at = `${field}.mapping.${entity}.${action}${ + isArray(value) ? `[${index}]` : '' + }`; + for (const key of ruleKeys) { + if (rule[key] !== undefined) paths.push(`${at}.${key}`); + } + }); + } + } + return paths; +} + +/** The message naming no-op paths at a position and what applies there. */ +export function getMappingNoOpMessage( + position: MappingPosition, + paths: string[], +): string { + return `${paths.map((path) => `\`${path}\``).join(', ')} ${ + paths.length === 1 ? 'does' : 'do' + } nothing at the ${position} position; ${APPLIES[position]}.`; +} diff --git a/packages/core/src/mapping.ts b/packages/core/src/mapping.ts index 4d5d7767a..b0f5b4472 100644 --- a/packages/core/src/mapping.ts +++ b/packages/core/src/mapping.ts @@ -4,7 +4,10 @@ import { isArray, isDefined, isObject, isThenable } from './is'; import { assign } from './assign'; import { flattenIncludeSections } from './include'; import { resolveMappingValue } from './mappingValue'; +import type { ValueTrace } from './mappingValue'; import { FatalError } from './fatalError'; +import { StepError } from './stepError'; +import { getErrorMessage } from './getErrorMessage'; /** * Gets the mapping for an event. @@ -22,21 +25,15 @@ export async function getMappingEvent( const [entity, action] = (event.name || '').split(' '); if (!mapping || !entity || !action) return {}; - let eventMapping: Mapping.Rule | undefined; - let mappingKey = ''; - let entityKey = entity; - let actionKey = action; - - // A condition that throws or rejects: logged and no match. A FatalError, - // and any throw without a collector (nothing to log to), propagates. - const onConditionError = (err: unknown, key: string): false => { - if (err instanceof FatalError || !collector) throw err; - collector.logger.error('mapping rule condition failed', { - event, - mappingKey: key, - error: err, - }); - return false; + // A condition that throws or rejects fails the event: falling through to + // the next rule would release an event the throwing rule meant to stop or + // strip. A FatalError propagates as is. + const onConditionError = (err: unknown, key: string): never => { + if (err instanceof FatalError) throw err; + throw new StepError( + `mapping rule condition failed at "${key}": ${getErrorMessage(err)}`, + { code: 'mapping', retryable: false, cause: err }, + ); }; // Synchronous unless the condition returns a promise. @@ -102,35 +99,44 @@ export async function getMappingEvent( ): Promise => (await pending) ? rule : resolveEventMapping(key, list, index + 1); - if (!mapping[entityKey]) entityKey = '*'; - const entityMapping = mapping[entityKey]; - - if (entityMapping) { - if (!entityMapping[actionKey]) actionKey = '*'; - const found = resolveEventMapping( - `${entityKey} ${actionKey}`, - entityMapping[actionKey], - ); - // Only an async condition costs an await. A thenable check, not - // `instanceof Promise`: pages may replace the global Promise. - eventMapping = isThenable(found) ? await found : found; - } - - // The * * fallback, unless the lookup above already tried exactly those - // rules (each condition runs once). - if ( - !eventMapping && - !(entityMapping && entityKey === '*' && actionKey === '*') - ) { - entityKey = '*'; - actionKey = '*'; - const found = resolveEventMapping('* *', mapping[entityKey]?.[actionKey]); - eventMapping = isThenable(found) ? await found : found; + // The tiers in the order the mapping schema states: exact, entity + // wildcard, action wildcard, global wildcard. A tier whose rules all fail + // their conditions falls through to the next; a tier met twice (an event + // named with a literal `*`) is tried once, so each condition runs once. + const tiers: Array<[string, string]> = []; + for (const tier of [ + [entity, action], + [entity, '*'], + ['*', action], + ['*', '*'], + ] as const) { + if (!tiers.some(([e, a]) => e === tier[0] && a === tier[1])) + tiers.push([tier[0], tier[1]]); } - if (eventMapping) mappingKey = `${entityKey} ${actionKey}`; + const resolveTier = ( + index: number, + ): Mapping.Result | Promise => { + for (let tier = index; tier < tiers.length; tier++) { + const [entityKey, actionKey] = tiers[tier]; + const rules = mapping[entityKey]?.[actionKey]; + if (!rules) continue; + const mappingKey = `${entityKey} ${actionKey}`; + const found = resolveEventMapping(mappingKey, rules); + // Only an async condition costs an await. A thenable check, not + // `instanceof Promise`: pages may replace the global Promise. + if (isThenable(found)) + return Promise.resolve(found).then( + (eventMapping): Mapping.Result | Promise => + eventMapping ? { eventMapping, mappingKey } : resolveTier(tier + 1), + ); + if (found) return { eventMapping: found, mappingKey }; + } + return { eventMapping: undefined, mappingKey: '' }; + }; - return { eventMapping, mappingKey }; + const result = resolveTier(0); + return isThenable(result) ? await result : result; } /** @@ -145,6 +151,19 @@ export async function getMappingValue( value: WalkerOS.DeepPartialEvent | unknown | undefined, data: Mapping.Data = {}, context: Mapping.ValueContext, +): Promise { + return valueAt(value, data, context); +} + +/** + * `getMappingValue` with the matched rule's mapping key, so a value that + * fails is reported with the rule it belongs to. + */ +async function valueAt( + value: WalkerOS.DeepPartialEvent | unknown | undefined, + data: Mapping.Data, + context: Mapping.ValueContext, + trace?: ValueTrace, ): Promise { if (!isDefined(value)) return; @@ -160,7 +179,12 @@ export async function getMappingValue( const event = (context?.event ?? (isObject(value) ? value : {})) as WalkerOS.DeepPartialEvent; - return resolveMappingValue(value, data, { ...context, consent, event }); + return resolveMappingValue( + value, + data, + { ...context, consent, event }, + trace, + ); } /** @@ -216,14 +240,19 @@ export async function processEventMapping< collector, ); + // A value that fails is reported with the rule it belongs to. + const trace: ValueTrace | undefined = mappingKey ? { mappingKey } : undefined; + // Step 2.5: Apply event-level policy (modifies event) if (eventMapping?.policy) { await Promise.all( Object.entries(eventMapping.policy).map(async ([key, mapping]) => { - const value = await getMappingValue(event, mapping, { - collector, + const value = await valueAt( event, - }); + mapping, + { collector, event }, + trace, + ); event = setByPath(event, key, value); }), ); @@ -232,7 +261,7 @@ export async function processEventMapping< // Step 3: Transform global data let data = config.data && - (await getMappingValue(event, config.data, { collector, event })); + (await valueAt(event, config.data, { collector, event }, trace)); const silent = Boolean(eventMapping?.silent); @@ -256,10 +285,7 @@ export async function processEventMapping< if (eventMapping.data) { const dataEvent = eventMapping.data && - (await getMappingValue(event, eventMapping.data, { - collector, - event, - })); + (await valueAt(event, eventMapping.data, { collector, event }, trace)); data = isObject(data) && isObject(dataEvent) // Only merge objects ? assign(data, dataEvent) diff --git a/packages/core/src/mappingValue.ts b/packages/core/src/mappingValue.ts index 0ca2bce0a..e1f71bd53 100644 --- a/packages/core/src/mappingValue.ts +++ b/packages/core/src/mappingValue.ts @@ -5,11 +5,56 @@ import { castToProperty } from './property'; import { tryCatchAsync } from './tryCatch'; import { FatalError } from './fatalError'; import { getGrantedConsent } from './consent'; +import { reportCause } from './causeLog'; // Package-internal value resolution shared by `mapping.ts` and `state.ts`. // Not re-exported from the package index: `getMappingValue` is the public // entry point. +/** Where a value is resolved, for the report of a value that failed. */ +export interface ValueTrace { + /** The matched rule's mapping key, when a rule matched. */ + mappingKey?: string; +} + +/** The keys a value config is made of; an object with none of them has no operator. */ +const VALUE_KEYS = [ + 'condition', + 'consent', + 'fn', + 'key', + 'loop', + 'map', + 'set', + 'validate', + 'value', +] as const satisfies ReadonlyArray; + +/** + * Report a value dropped by throwing user code or a malformed config: one + * line per cause through the collector's cause log, with the event's name + * and the mapping key, never the event. The event itself is still delivered. + */ +function reportValue( + context: Mapping.Context, + trace: ValueTrace | undefined, + message: string, + err?: unknown, +): void { + const name = context.event.name; + reportCause( + context.collector, + context.logger, + { step: 'mapping', code: 'mapping' }, + message, + { + ...(typeof name === 'string' ? { event: name } : {}), + ...(trace?.mappingKey ? { mappingKey: trace.mappingKey } : {}), + ...(err !== undefined ? { error: err } : {}), + }, + ); +} + /** * Like getMappingValue, but also runs for an undefined source and takes * consent and event only from the context. @@ -23,6 +68,7 @@ export async function resolveMappingValue( value: unknown, data: Mapping.Data = {}, context: Mapping.ValueContext, + trace?: ValueTrace, ): Promise { if (!context.collector) { // The type requires a collector. This guard protects untyped callers @@ -47,17 +93,19 @@ export async function resolveMappingValue( processMappingValue, (err: unknown): undefined => { if (err instanceof FatalError) throw err; - context.collector.status.failed++; - baseContext.logger.error('mapping processing failed', { - event, - error: err, - }); + // The value is dropped, the event goes on: not a failed event. + reportValue(baseContext, trace, 'mapping value failed', err); return undefined; }, - )(value, mapping, { - ...baseContext, + )( + value, mapping, - }); + { + ...baseContext, + mapping, + }, + trace, + ); if (isDefined(result)) return result; } return; @@ -67,6 +115,7 @@ async function processMappingValue( value: WalkerOS.DeepPartialEvent | unknown, mapping: Mapping.Value, context: Mapping.Context, + trace: ValueTrace | undefined, ): Promise { const mappings = isArray(mapping) ? mapping : [mapping]; @@ -81,6 +130,18 @@ async function processMappingValue( if (!Object.keys(mapping).length) return; + // An object without any operator (`{ total: 'data.total' }` where + // `{ map: { total: 'data.total' } }` was meant) would resolve to the + // whole input. Drop it and say how to fix it. + if (!VALUE_KEYS.some((name) => name in mapping)) { + reportValue( + { ...context, mapping: mappingItem }, + trace, + `mapping value has no operator (${Object.keys(mapping).join(', ')}): wrap the fields in map, or use key, value, fn, loop or set`, + ); + return; + } + const { condition, consent, @@ -100,10 +161,7 @@ async function processMappingValue( condition && !(await tryCatchAsync(condition, (err: unknown): boolean => { if (err instanceof FatalError) throw err; - cbContext.logger.error('mapping condition failed', { - event: cbContext.event, - error: err, - }); + reportValue(cbContext, trace, 'mapping condition failed', err); return false; // Preserve "skip this rule" semantic on throw. })(value, cbContext)) ) @@ -121,7 +179,7 @@ async function processMappingValue( const data = scope === 'this' ? [value] - : await resolveMappingValue(value, scope, cbContext); + : await resolveMappingValue(value, scope, cbContext, trace); // Each item is mapped with the parent's consent: a `consent` key on // the item is plain data and never grants anything. Undefined items @@ -131,7 +189,7 @@ async function processMappingValue( await Promise.all( data.map((item) => isDefined(item) - ? resolveMappingValue(item, itemMapping, cbContext) + ? resolveMappingValue(item, itemMapping, cbContext, trace) : undefined, ), ) @@ -145,6 +203,7 @@ async function processMappingValue( value, mapValue, cbContext, + trace, ); if (isDefined(result)) mappedObj[mapKey] = result; return mappedObj; @@ -153,17 +212,14 @@ async function processMappingValue( ); } else if (set) { mappingValue = await Promise.all( - set.map((item) => processMappingValue(value, item, cbContext)), + set.map((item) => processMappingValue(value, item, cbContext, trace)), ); } else if (key) { mappingValue = getByPath(value, key, staticValue); } else if (fn) { mappingValue = await tryCatchAsync(fn, (err: unknown): undefined => { if (err instanceof FatalError) throw err; - cbContext.logger.error('mapping fn failed', { - event: cbContext.event, - error: err, - }); + reportValue(cbContext, trace, 'mapping fn failed', err); return undefined; // No transform on error. })(value, cbContext); } @@ -172,10 +228,7 @@ async function processMappingValue( validate && !(await tryCatchAsync(validate, (err: unknown): boolean => { if (err instanceof FatalError) throw err; - cbContext.logger.error('mapping validate failed', { - event: cbContext.event, - error: err, - }); + reportValue(cbContext, trace, 'mapping validate failed', err); return false; // Preserve "validation failed" semantic on throw. })(mappingValue, cbContext)) ) diff --git a/packages/core/src/schemas/collector.ts b/packages/core/src/schemas/collector.ts index 4161af548..e441c0614 100644 --- a/packages/core/src/schemas/collector.ts +++ b/packages/core/src/schemas/collector.ts @@ -120,6 +120,12 @@ export const ConfigSchema = z .describe( 'Maximum events retained in collector.queue (late-registration replay). FIFO drop on overflow. Default 1000.', ), + consentDenied: z + .enum(['queue', 'skip']) + .optional() + .describe( + "What a destination does with a consent-denied event: 'queue' (default) keeps it until consent is granted, 'skip' skips it (counted in status.destinations[id].skipped).", + ), name: z .string() .optional() diff --git a/packages/core/src/state.ts b/packages/core/src/state.ts index 60fe52e15..6fcf761dc 100644 --- a/packages/core/src/state.ts +++ b/packages/core/src/state.ts @@ -9,6 +9,9 @@ import { deepMerge } from './deepMerge'; import { isArray, isDefined, isObject, isString } from './is'; import { isPropertyType } from './property'; import { tryCatchAsync } from './tryCatch'; +import { reportCause } from './causeLog'; +import type { CauseKey } from './causeLog'; +import { toFailureReason } from './stepError'; /** * Resolve a store by id. An `undefined` id falls back to the default @@ -63,6 +66,46 @@ function isEmptyResult(value: unknown): boolean { return false; } +/** + * Report a state problem through the collector's cause log. A partial + * collector without a logger (tests) has nowhere to write. + */ +function reportState( + collector: Collector.Instance, + key: CauseKey, + message: string, + context: Record, +): void { + if (collector.causeLog || collector.logger) + reportCause(collector, collector.logger, key, message, context); +} + +/** + * Read a store key. A read that throws (a refused or failed request, not a + * miss) is reported once per cause under the store's step and answered as + * failed; the event is never failed by a read. + */ +async function readStore( + store: Store.Instance, + storeId: string, + key: string, + collector: Collector.Instance, +): Promise<{ ok: true; value: Store.StoreValue | undefined } | { ok: false }> { + try { + return { ok: true, value: await store.get(key) }; + } catch (err) { + if (err instanceof FatalError) throw err; + const reason = toFailureReason(err); + reportState( + collector, + { step: `store.${storeId}`, code: reason.code, status: reason.status }, + 'store read failed', + { error: reason.message, retryable: reason.retryable }, + ); + return { ok: false }; + } +} + /** * Apply declarative store operations against an event, in array order, * sequentially. @@ -85,8 +128,11 @@ function isEmptyResult(value: unknown): boolean { * Each entry is fail-open: a store or resolution error is logged and the * event is left unmutated, the chain continues. An unknown store logs an * error and a `file: true` store is skipped with a warning. A key that does - * not resolve warns; a store miss is silent. Only `FatalError` rethrows (via - * `getMappingValue`). + * not resolve is reported at ERROR (the enrichment is lost); a store miss is + * silent. A read that throws is a miss for that event (a `get` applies its + * fallbacks) and is reported once per cause; a merge whose read throws skips + * its write, since a blind overwrite would drop the stored fields. Only + * `FatalError` rethrows (via `getMappingValue`). */ export async function applyState( states: State[], @@ -126,13 +172,15 @@ export async function applyState( event: result, consent: eventConsent(result), }; + const storeId = entry.store ?? '__cache'; const rawKey = await getMappingValue(root, entry.key, context); if (!isString(rawKey)) { - collector.logger?.warn?.('[state] key did not resolve', { - mode: entry.mode, - store: entry.store, - key: entry.key, - }); + reportState( + collector, + { step: `store.${storeId}`, code: 'mapping' }, + `[state] ${entry.mode} key did not resolve, entry skipped`, + isString(entry.key) ? { mappingKey: entry.key } : {}, + ); return; } // namespace state keys on the shared default store to avoid cache collisions @@ -157,7 +205,9 @@ export async function applyState( return; } // Read only when a merge is possible. Not atomic: read, merge, write. - const stored: Store.StoreValue | undefined = await store.get(key); + const read = await readStore(store, storeId, key, collector); + if (!read.ok) return; + const stored = read.value; if (isDefined(stored) && !isObject(stored)) collector.logger?.warn?.( '[state] stored value is not an object, replaced', @@ -180,7 +230,8 @@ export async function applyState( ); return; } - const fetched = await store.get(key); + const read = await readStore(store, storeId, key, collector); + const fetched = read.ok ? read.value : undefined; if (!isDefined(entry.mapping)) { if (!isDefined(fetched)) return; // miss: leave event unchanged if (target.side === 'ingest') { diff --git a/packages/core/src/stepError.ts b/packages/core/src/stepError.ts new file mode 100644 index 000000000..6570d189f --- /dev/null +++ b/packages/core/src/stepError.ts @@ -0,0 +1,111 @@ +/** + * Options of a {@link StepError}: the failure code, and what decides whether + * a retry can help (`status`, an explicit `retryable`). + */ +export interface StepErrorOptions { + code: string; + status?: number; + retryable?: boolean; + /** The response body: kept on the error, never in its message or a log line. */ + body?: unknown; + cause?: unknown; +} + +/** Why a push failed, as a push result and the logs carry it. Never a body. */ +export type FailureReason = { + code: string; + message: string; + retryable: boolean; + status?: number; +}; + +/** Why an event waits instead of being delivered now. */ +export type QueueReason = 'consent' | 'batch' | 'held'; + +export type QueuedReason = { code: QueueReason; message: string }; + +/** Codes a retry can help with, unless a status or an option says otherwise. */ +const RETRYABLE_CODES = [ + 'transport', + 'timeout', + 'breaker-open', + 'blocked', + 'queue-full', + 'vendor-not-ready', + 'vendor-timeout', +]; + +/** Longest message a reason or a log line carries. */ +const MAX_MESSAGE_LENGTH = 300; + +/** + * Precedence: an explicit `retryable`, then the HTTP status (429 and 5xx + * retryable, other 4xx not), then the code's default. A code outside the + * retryable list is not retryable. + */ +function resolveRetryable(options: StepErrorOptions): boolean { + if (options.retryable !== undefined) return options.retryable; + const { status } = options; + if (status !== undefined) { + if (status === 429 || (status >= 500 && status < 600)) return true; + if (status >= 400 && status < 500) return false; + } + return RETRYABLE_CODES.includes(options.code); +} + +/** + * The error a step throws when it cannot do its work: a missing setting + * (`config`), a refused request (`transport` with its `status`), a vendor + * that is not there. The collector reads `code` and `retryable` instead of + * the message. + */ +export class StepError extends Error { + code: string; + status?: number; + retryable: boolean; + body?: unknown; + + constructor(message: string, options: StepErrorOptions) { + super( + message, + options.cause !== undefined ? { cause: options.cause } : undefined, + ); + this.name = 'StepError'; + this.code = options.code; + if (options.status !== undefined) this.status = options.status; + this.retryable = resolveRetryable(options); + if (options.body !== undefined) this.body = options.body; + + // Preserve the prototype chain where the class is downleveled. + Object.setPrototypeOf(this, new.target.prototype); + } +} + +/** A message capped to what a reason or a log line may carry. */ +export function capMessage(message: string): string { + return message.length > MAX_MESSAGE_LENGTH + ? message.slice(0, MAX_MESSAGE_LENGTH) + : message; +} + +/** + * The reason of any thrown value: a `StepError` keeps its code, status and + * `retryable`; anything else is a `transport` failure. The message is capped + * at 300 characters; the body never leaves the error. + */ +export function toFailureReason(err: unknown): FailureReason { + if (err instanceof StepError) { + const reason: FailureReason = { + code: err.code, + message: capMessage(err.message), + retryable: err.retryable, + }; + if (err.status !== undefined) reason.status = err.status; + return reason; + } + return { + code: 'transport', + message: capMessage(err instanceof Error ? err.message : String(err)), + retryable: true, + }; +} diff --git a/packages/core/src/transformer-mapping.ts b/packages/core/src/transformer-mapping.ts index e0720dae1..2bf875289 100644 --- a/packages/core/src/transformer-mapping.ts +++ b/packages/core/src/transformer-mapping.ts @@ -1,5 +1,4 @@ -import type { Mapping } from './types'; -import { isArray, isObject } from './is'; +import { getMappingNoOpMessage, getMappingNoOpPaths } from './mapping-no-ops'; /** * A transformer entry as far as its declarative mapping is concerned: a @@ -13,29 +12,6 @@ export interface TransformerMappingEntry { config?: { mapping?: M }; } -/** - * Mapping fields that do nothing at the transformer position. A - * transformer's mapping only mutates the event: config-level `policy` and a - * rule's `condition`, `policy`, `name` and `ignore` apply. Everything that - * shapes a vendor payload or gates a destination is a no-op there. - */ -const CONFIG_NO_OPS = [ - 'consent', - 'include', - 'data', -] as const satisfies ReadonlyArray; - -const RULE_NO_OPS = [ - 'consent', - 'include', - 'remove', - 'batch', - 'settings', - 'extend', - 'data', - 'silent', -] as const satisfies ReadonlyArray; - /** * The mapping a transformer step declares. A `config.mapping` wins over the * step-level `mapping`, as `config.state` wins over `state`. @@ -46,37 +22,6 @@ export function getTransformerMapping( return entry.config?.mapping ?? entry.mapping; } -/** - * Dotted paths of the fields in `mapping` that do nothing at the - * transformer position, rooted at `field`. Rules in array form are indexed. - */ -function getNoOpPaths(mapping: unknown, field: string): string[] { - if (!isObject(mapping)) return []; - const paths: string[] = []; - for (const key of CONFIG_NO_OPS) { - if (mapping[key] !== undefined) paths.push(`${field}.${key}`); - } - - const rules = mapping.mapping; - if (!isObject(rules)) return paths; - for (const [entity, actions] of Object.entries(rules)) { - if (!isObject(actions)) continue; - for (const [action, value] of Object.entries(actions)) { - const list: unknown[] = isArray(value) ? value : [value]; - list.forEach((rule, index) => { - if (!isObject(rule)) return; - const at = `${field}.mapping.${entity}.${action}${ - isArray(value) ? `[${index}]` : '' - }`; - for (const key of RULE_NO_OPS) { - if (rule[key] !== undefined) paths.push(`${at}.${key}`); - } - }); - } - } - return paths; -} - /** * Warnings for a transformer's declarative mapping, one definition for the * collector's init warning and `validateFlowStructure`. They never reject @@ -113,13 +58,8 @@ export function getTransformerMappingWarnings( ); } - const paths = getNoOpPaths(mapping, field); - if (paths.length > 0) { - warnings.push( - `${paths.map((path) => `\`${path}\``).join(', ')} ${ - paths.length === 1 ? 'does' : 'do' - } nothing at the transformer position; only \`policy\` and a rule's \`condition\`, \`policy\`, \`name\` and \`ignore\` apply.`, - ); - } + const paths = getMappingNoOpPaths('transformer', mapping, field); + if (paths.length > 0) + warnings.push(getMappingNoOpMessage('transformer', paths)); return warnings; } diff --git a/packages/core/src/types/collector.ts b/packages/core/src/types/collector.ts index 8037693e5..219866ce4 100644 --- a/packages/core/src/types/collector.ts +++ b/packages/core/src/types/collector.ts @@ -13,6 +13,7 @@ import type { import type { Ingest } from './ingest'; import type { Observe } from '../observeConnect'; import type { ObserverFn } from './observer'; +import type { CauseLog } from '../causeLog'; /** Identifies which kind of step a stepId belongs to. */ export type StepKind = @@ -97,6 +98,14 @@ export interface Config { * destination backfill. Overflow drops oldest (FIFO). Default 1000. */ queueMax?: number; + /** + * What a destination does with an event whose required consent is not + * granted: `'queue'` (default) keeps it in the destination's `queuePush` + * until consent is granted; `'skip'` skips it (counted in + * `status.destinations[id].skipped`, a DEBUG line), for a server collector + * that never receives a later grant for an event's own consent. + */ + consentDenied?: 'queue' | 'skip'; /** Flow name; keys this flow's entry in event.source.release and the observer flowId. */ name?: string; /** Config release id stamped into event.source.release for this flow. */ @@ -255,6 +264,11 @@ export interface SourceStatus { export interface DestinationStatus { count: number; failed: number; + /** + * Events deliberately not sent: a consent-denied event under + * `consentDenied: 'skip'`, or an event a mapping rule `ignore`s. + */ + skipped?: number; lastAt?: number; duration: number; /** Current size of the destination's queuePush buffer (point-in-time). */ @@ -293,11 +307,13 @@ export type CommandType = | 'context' | 'destination' | 'elb' + | 'flush' | 'globals' | 'hook' | 'init' | 'link' | 'run' + | 'shutdown' | 'user' | 'walker' | string; @@ -409,6 +425,12 @@ export interface Instance { */ observeLevel?: () => 'off' | 'standard' | 'trace'; logger: Logger.Instance; + /** + * The collector's one cause log: every failure line goes through it, one + * line per (step, cause code, status) per minute. Set by `startFlow`; + * optional only so hand-built test collectors keep compiling. + */ + causeLog?: CauseLog; on: On.OnConfig; queue: WalkerOS.Events; /** diff --git a/packages/core/src/types/destination.ts b/packages/core/src/types/destination.ts index 1d40a6397..bfb70b8ce 100644 --- a/packages/core/src/types/destination.ts +++ b/packages/core/src/types/destination.ts @@ -10,6 +10,7 @@ import type { } from '.'; import type { DestroyFn, SetupFn } from './lifecycle'; import type { Ingest } from './ingest'; +import type { FailureReason, QueuedReason } from '../stepError'; import type { ConfigTypes, EntryTypes, TypedStepEntry } from './util'; /** @@ -97,9 +98,21 @@ export type Credentials = T['credentials']; */ export type TypesOf = I extends Instance ? T : never; +/** + * An event waiting in a destination's `queuePush`, with why it waits: + * `consent` (its required consent is not granted) or `held` (a state + * delivery to the destination is running or failed). An entry seeded from + * the collector's replay buffer has no reason yet. The entry object keeps its + * identity across drain and requeue; the event inside is copied. + */ +export interface Queued { + event: WalkerOS.Event; + reason?: 'consent' | 'held'; +} + export interface Instance { config: Config; - queuePush?: WalkerOS.Events; + queuePush?: Queued[]; queueOn?: Array<{ type: On.Types; data?: unknown }>; dlq?: DLQ; batches?: BatchRegistry>; @@ -411,13 +424,24 @@ export interface BatchRegistry { batchFn: () => void; /** * Force a flush of the current batch immediately. Used by shutdown - * drain and by tests for deterministic flushing without timer - * advancement. Resolves after the underlying `pushBatch` settles. + * drain, `walker flush` and by tests for deterministic flushing without + * timer advancement. Resolves after the underlying `pushBatch` settles, + * with what that flush did (nothing when the batch was empty). */ - flush: () => Promise; + flush: () => Promise; }; } +/** + * What one batch flush did: entries delivered and failed, and the reason of + * a failure when any entry failed. + */ +export interface BatchFlushOutcome { + delivered: number; + failed: number; + reason?: FailureReason; +} + export type Data = | WalkerOS.Property | undefined @@ -427,10 +451,15 @@ export interface Ref { type: string; // Destination type ("gtag", "meta", "bigquery") data?: unknown; // Response from push() error?: unknown; // Error if failed + /** + * Why the destination did not deliver now: a `FailureReason` under + * `failed`, a `QueuedReason` under `queued`. + */ + reason?: FailureReason | QueuedReason; } export type Push = { - queuePush?: WalkerOS.Events; + queuePush?: Queued[]; error?: unknown; }; diff --git a/packages/core/src/types/elb.ts b/packages/core/src/types/elb.ts index 8c045e2f3..ee6150320 100644 --- a/packages/core/src/types/elb.ts +++ b/packages/core/src/types/elb.ts @@ -1,4 +1,5 @@ import type { Destination, Hooks, On, WalkerOS } from '.'; +import type { FailureReason } from '../stepError'; // Event signatures only export interface EventFn> { @@ -67,6 +68,12 @@ export interface PushResult { invalid?: boolean; /** Human-readable failure reason when ok is false. */ error?: string; + /** + * Why the push failed when `ok` is false without a `failed` entry: a full + * pre-run queue (`queue-full`), a transformer that threw (`transformer`) + * or is not ready, a rule `condition` that threw (`mapping`). + */ + reason?: FailureReason; } // Simplified Layer type for core collector diff --git a/packages/core/src/types/journey.ts b/packages/core/src/types/journey.ts index 5b0b266de..39590628a 100644 --- a/packages/core/src/types/journey.ts +++ b/packages/core/src/types/journey.ts @@ -1,5 +1,5 @@ import type { Simulation } from '.'; -import type { FlowStatePhase, FlowStepType } from './telemetry'; +import type { FlowState, FlowStatePhase, FlowStepType } from './telemetry'; /** * Journey types describe the assembled, cross-runtime life of ONE EVENT as @@ -128,15 +128,9 @@ export interface JourneyBranch { /** Wall-clock duration of the branch, when measured. */ durationMs?: number; /** Skip discriminator when the branch was skipped. */ - skipReason?: - | 'consent' - | 'cache_hit' - | 'sampled_out' - | 'disabled' - | 'dropped' - | 'unknown'; + skipReason?: FlowState['skipReason']; /** Error info when the branch failed. */ - error?: { name?: string; message: string }; + error?: FlowState['error']; /** Outbound payload of the branch, when captured. */ out?: unknown; /** Vendor calls recorded on the branch's out record, when captured. */ @@ -174,15 +168,9 @@ export interface JourneyHop { /** Outbound event/payload for the hop, when captured (trace level). */ out?: unknown; /** Error info when the hop's terminal phase is error. */ - error?: { name?: string; message: string }; + error?: FlowState['error']; /** Skip discriminator when the hop's terminal phase is skip. */ - skipReason?: - | 'consent' - | 'cache_hit' - | 'sampled_out' - | 'disabled' - | 'dropped' - | 'unknown'; + skipReason?: FlowState['skipReason']; /** Matched mapping rule, when one matched. */ mappingKey?: string; /** Matched contract rule, when one matched. */ diff --git a/packages/core/src/types/logger.ts b/packages/core/src/types/logger.ts index e523028e3..6b263bb31 100644 --- a/packages/core/src/types/logger.ts +++ b/packages/core/src/types/logger.ts @@ -37,8 +37,8 @@ export type LogFn = ( ) => void; /** - * Throw function signature - logs error then throws - * Returns never as it always throws + * Throw function signature: throws an Error with the message, never logs + * (the catching collector logs the failure once). Returns never. */ export type ThrowFn = (message: string | Error, context?: unknown) => never; @@ -90,8 +90,8 @@ export interface Instance { debug: LogFn; /** - * Log an error message and throw an Error - * Combines logging and throwing in one call + * Throw an Error with the message. Does not log: the collector's catch + * writes the one line for the failure. */ throw: ThrowFn; diff --git a/packages/core/src/types/telemetry.ts b/packages/core/src/types/telemetry.ts index e0d0d9462..580e2691d 100644 --- a/packages/core/src/types/telemetry.ts +++ b/packages/core/src/types/telemetry.ts @@ -68,8 +68,17 @@ export interface FlowState { inEvent?: unknown; /** Outbound walker event/payload for this hop. Only populated when level === 'trace' or includeOut === true. */ outEvent?: unknown; - /** Error info when phase === 'error'. */ - error?: { name?: string; message: string }; + /** + * Error info when phase === 'error'. A delivery failure also carries its + * failure `code`, whether a retry can help and the HTTP `status`. + */ + error?: { + name?: string; + message: string; + code?: string; + retryable?: boolean; + status?: number; + }; /** The mapping rule that matched, when one matched. */ mappingKey?: string; /** Contract rule that matched, if any. */ @@ -89,6 +98,8 @@ export interface FlowState { | 'sampled_out' | 'disabled' | 'dropped' + | 'held' + | 'ignored' | 'unknown'; /** Free-form metadata: store key/value, cached: true, etc. */ meta?: Record; diff --git a/packages/core/src/types/transformer.ts b/packages/core/src/types/transformer.ts index 5f9160321..051b57647 100644 --- a/packages/core/src/types/transformer.ts +++ b/packages/core/src/types/transformer.ts @@ -1,6 +1,7 @@ /* eslint-disable @typescript-eslint/no-explicit-any */ import type { Collector, Logger, WalkerOS, Context as BaseContext } from '.'; import type { DestroyFn } from './lifecycle'; +import type { FailureReason } from '../stepError'; import type { Ingest } from './ingest'; import type { Config as MappingConfig } from './mapping'; import type { MatchExpression } from './matcher'; @@ -213,8 +214,10 @@ export interface ChainCopy { * with the ingest it finished with. The next position goes on with each * pair. * - * - dropped (a transformer returned `false` or threw): `copies` empty, - * `droppedBy` names the transformer. + * - dropped (a transformer returned `false`): `copies` empty, `droppedBy` + * names the transformer. + * - failed (a transformer threw, or its `init` threw or returned `false`): + * `failure` names it and carries the reason; not a drop. * - route `stop`: `copies` empty, `stopped: true`; `droppedBy` names the * transformer whose route stopped it, absent for the start route. * - `cache.stop: true` HIT: `copies` holds the cached copy, `stopped: true`. @@ -227,6 +230,18 @@ export interface ChainResult { stopped?: true; /** Transformer that dropped or stopped the chain. */ droppedBy?: string; + /** + * A transformer that threw or could not start: the failed copy ends (not + * a drop), and the position answers `ok: false` with this reason. Other + * copies of a fork still go on in `copies`. + */ + failure?: ChainFailure; +} + +/** A transformer in a chain that failed: its id and why. */ +export interface ChainFailure { + by: string; + reason: FailureReason; } /** diff --git a/packages/web/destinations/snowplow/src/index.test.ts b/packages/web/destinations/snowplow/src/index.test.ts index b1664949d..8605c0da8 100644 --- a/packages/web/destinations/snowplow/src/index.test.ts +++ b/packages/web/destinations/snowplow/src/index.test.ts @@ -143,10 +143,15 @@ describe('destination snowplow', () => { await elb(getEvent()); - // logger.throw logs error and throws, collector catches and logs + // logger.throw only throws; the collector catches and writes the one line + expect(consoleErrorSpy).toHaveBeenCalledTimes(1); expect(consoleErrorSpy).toHaveBeenCalledWith( - expect.stringContaining('[snowplow]'), - 'Config settings collectorUrl missing', + 'ERROR', + expect.stringContaining('destination init failed'), + expect.objectContaining({ + type: 'snowplow', + error: 'Config settings collectorUrl missing', + }), ); consoleErrorSpy.mockRestore(); }); @@ -2943,9 +2948,14 @@ describe('destination snowplow', () => { await elb(getEvent()); + // The destination throws; the collector writes the one line for it. expect(consoleErrorSpy).toHaveBeenCalledWith( - expect.stringContaining('[snowplow]'), - 'tracker.newTracker is required for browser-tracker mode', + 'ERROR', + expect.stringContaining('destination init failed'), + expect.objectContaining({ + type: 'snowplow', + error: 'tracker.newTracker is required for browser-tracker mode', + }), ); consoleErrorSpy.mockRestore(); }); @@ -2968,9 +2978,15 @@ describe('destination snowplow', () => { await elb(getEvent()); + // The destination throws; the collector writes the one line for it. expect(consoleErrorSpy).toHaveBeenCalledWith( - expect.stringContaining('[snowplow]'), - 'tracker.trackSelfDescribingEvent is required for browser-tracker mode', + 'ERROR', + expect.stringContaining('destination init failed'), + expect.objectContaining({ + type: 'snowplow', + error: + 'tracker.trackSelfDescribingEvent is required for browser-tracker mode', + }), ); consoleErrorSpy.mockRestore(); }); From 8ba06c45c9af3f2cb81566f1bbd1d299f20b5fb1 Mon Sep 17 00:00:00 2001 From: alexanderkirtzel Date: Wed, 21 Oct 2026 00:42:32 +0000 Subject: [PATCH 02/41] delivery order --- .../components/demos/PromotionPlayground.tsx | 25 ++++++- .../src/__tests__/delivery-order.test.ts | 68 +++++++++++++++++++ packages/collector/src/destination.ts | 55 +++++++++++++-- .../core/src/__tests__/collector.test.ts | 6 +- 4 files changed, 145 insertions(+), 9 deletions(-) create mode 100644 packages/collector/src/__tests__/delivery-order.test.ts diff --git a/apps/explorer/src/components/demos/PromotionPlayground.tsx b/apps/explorer/src/components/demos/PromotionPlayground.tsx index b572dffe6..5a16b175f 100644 --- a/apps/explorer/src/components/demos/PromotionPlayground.tsx +++ b/apps/explorer/src/components/demos/PromotionPlayground.tsx @@ -216,12 +216,31 @@ export function PromotionPlayground({ }, consent: { functional: true, marketing: true }, user: { session: 'playground' }, - // A failure the collector logs (a destination push, a mapping) shows - // in the Result box, besides the console. + // Every event the mapped destination fails (its init, its push, its + // mapping) shows in the Result box with the failure's message: the + // collector reports each failed event to its observers, while its log + // writes one line per cause and minute. + observers: [ + (state) => { + if ( + active && + state.phase === 'error' && + state.stepId === `destination.${DESTINATION_ID}` && + state.error + ) + setResultError(`${DESTINATION_ID}: ${state.error.message}`); + }, + ], + // Any other failure the collector logs (a mapping value) shows in the + // Result box too, besides the console. logger: { handler: (level, message, context, scope, original) => { original(level, message, context, scope); - if (active && level === Level.ERROR) + if ( + active && + level === Level.ERROR && + context.step !== `destination.${DESTINATION_ID}` + ) setResultError(logged(message, context, scope)); }, }, diff --git a/packages/collector/src/__tests__/delivery-order.test.ts b/packages/collector/src/__tests__/delivery-order.test.ts new file mode 100644 index 000000000..9af653148 --- /dev/null +++ b/packages/collector/src/__tests__/delivery-order.test.ts @@ -0,0 +1,68 @@ +import type { Destination, WalkerOS } from '@walkeros/core'; +import { startFlow } from '..'; + +describe('delivery order across a destination init', () => { + it('delivers events pushed without awaiting in push order to a destination without init', async () => { + const names: string[] = []; + const destination: Destination.Instance = { + type: 'recorder', + config: {}, + push: (event: WalkerOS.Event) => { + names.push(event.name); + }, + }; + const { elb } = await startFlow({ + destinations: { recorder: { code: destination } }, + }); + + await Promise.all([elb('a one'), elb('b two'), elb('c three')]); + + expect(names).toEqual(['a one', 'b two', 'c three']); + }); + + it('keeps push order behind an async init too', async () => { + const names: string[] = []; + const destination: Destination.Instance = { + type: 'recorder', + config: {}, + init: async () => { + // Several turns, so later pushes arrive while it runs. + for (let turn = 0; turn < 10; turn++) await Promise.resolve(); + }, + push: (event: WalkerOS.Event) => { + names.push(event.name); + }, + }; + const { elb } = await startFlow({ + destinations: { recorder: { code: destination } }, + }); + + await Promise.all([elb('a one'), elb('b two'), elb('c three')]); + + expect(names).toEqual(['a one', 'b two', 'c three']); + }); + + it('fails the events queued behind an init that fails, never leaves them waiting', async () => { + const destination: Destination.Instance = { + type: 'broken', + config: {}, + init: async () => { + for (let turn = 0; turn < 10; turn++) await Promise.resolve(); + throw new Error('init failed'); + }, + push: jest.fn(), + }; + const { elb, collector } = await startFlow({ + logger: { handler: () => undefined }, + destinations: { broken: { code: destination } }, + }); + + await Promise.all([elb('a one'), elb('b two'), elb('c three')]); + + expect(collector.status.destinations.broken.failed).toBe(3); + expect(collector.destinations.broken.queuePush ?? []).toHaveLength(0); + expect( + (collector.destinations.broken.dlq ?? []).map(([event]) => event.name), + ).toEqual(['a one', 'b two', 'c three']); + }); +}); diff --git a/packages/collector/src/destination.ts b/packages/collector/src/destination.ts index a26ecee9e..465f98c88 100644 --- a/packages/collector/src/destination.ts +++ b/packages/collector/src/destination.ts @@ -42,6 +42,7 @@ import { BATCH_WAIT, CONSENT_WAIT, HELD_WAIT, + INIT_WAIT, breakerState, heldState, } from './destination-outcome-state'; @@ -606,6 +607,19 @@ async function deliverToDestinations( const requeueHeld = (entries: Destination.Queued[]) => enqueue(entries, 'held'); + // The entries concurrent pushes queued behind a running init (held, + // not among `keep`), taken out of the queue in push order. + const takeQueuedBehindInit = (keep: Set) => { + const queuePush = destination.queuePush || []; + const behind = queuePush.filter( + (queued) => queued.reason === 'held' && !keep.has(queued), + ); + destination.queuePush = queuePush.filter( + (queued) => !behind.includes(queued), + ); + return behind; + }; + // Fail refused held entries through the one outcome sink. const refuseHeld = (refused: Destination.Queued[]) => recordFailure( @@ -808,9 +822,16 @@ async function deliverToDestinations( true, ); } catch (err) { - recordFailure(collector, destination, id, [], err, { - message: 'destination init failed', - }); + // Events queued behind this init by concurrent pushes fail with + // it, rather than waiting for a next push that may never come. + recordFailure( + collector, + destination, + id, + takeQueuedBehindInit(new Set()).map((entry) => entry.event), + err, + { message: 'destination init failed' }, + ); } // The flush held the destination: there are no events of this call // to requeue, and any parked by a concurrent push stay queued. @@ -866,6 +887,21 @@ async function deliverToDestinations( // a concurrent push while the init flush was running. const parked = new Set(destination.queuePush); + // An init of this destination (with its queue flush) is running in + // another pass: the admitted events queue behind it, and that pass + // delivers them right after its own, so the destination receives its + // events in push order. A destination without an `init` runs a no-op + // one through the same path, so it is covered too. + if (initsRunning.has(destination)) { + releaseProbeSlot(); + const refused = requeueHeld(allowedEvents); + if (refused.length) { + const failed = refuseHeld(refused); + return { id, destination, failed }; + } + return { id, destination, queued: INIT_WAIT }; + } + // Initialize the destination if needed. An init that throws, or // returns false (runtime guard: `not-ready`), fails the admitted // events through the outcome sink; init is re-attempted on the next @@ -878,11 +914,15 @@ async function deliverToDestinations( code: 'not-ready', }); } catch (err) { + // The events queued behind this init by concurrent pushes fail with + // it, in push order after this pass's own. const failed = recordFailure( collector, destination, id, - allowedEvents.map((entry) => entry.event), + [...allowedEvents, ...takeQueuedBehindInit(parked)].map( + (entry) => entry.event, + ), err, { message: 'destination init failed' }, ); @@ -1552,6 +1592,13 @@ export async function destinationInit( // flushed instead of starting another init(). const running = initsRunning.get(destination); if (running) return running; + // Nothing to run and nothing queued: initialized at once, without a + // turn, so this destination's events keep their place among its + // siblings' (a destination without `init` delivered at once before). + if (!destination.init && !destination.queueOn?.length) { + destination.config = { ...destination.config, init: true }; + return true; + } const started = initAndFlush( collector, destination, diff --git a/packages/server/core/src/__tests__/collector.test.ts b/packages/server/core/src/__tests__/collector.test.ts index 4f088c9b0..58589504b 100644 --- a/packages/server/core/src/__tests__/collector.test.ts +++ b/packages/server/core/src/__tests__/collector.test.ts @@ -87,8 +87,9 @@ describe('Server Collector', () => { expect(mockDestinationPush).toHaveBeenCalledTimes(1); expect(mockDestinationPush).toHaveBeenCalledWith( mockEvent, + // A destination without `init` is marked initialized at its first push. expect.objectContaining({ - config: mockDestination.config, + config: { ...mockDestination.config, init: true }, }), ); expect(result.ok).toBe(true); @@ -118,8 +119,9 @@ describe('Server Collector', () => { user: { id: 'us3r1d' }, consent: { test: true }, }), + // A destination without `init` is marked initialized at its first push. expect.objectContaining({ - config: mockDestination.config, + config: { ...mockDestination.config, init: true }, }), ); }); From 4100ede14efcaa683a3a743bf319f1d6cc9ab2dc Mon Sep 17 00:00:00 2001 From: alexanderkirtzel Date: Wed, 21 Oct 2026 00:42:34 +0000 Subject: [PATCH 03/41] standard events --- .changeset/core-standard-event-value.md | 9 + .changeset/transformer-ga4-order-value.md | 7 + apps/demos/flows/index.html | 2 +- apps/demos/flows/pentest/load-test.js | 2 +- apps/demos/react/src/components/Order.tsx | 2 +- .../demos/__fixtures__/shop-attributes.json | 4 +- apps/demos/storybook/src/demos/shop/data.ts | 4 +- apps/demos/tagging/index.html | 4 +- apps/demos/tagging/index.local.html | 4 +- .../components/molecules/code-box.stories.tsx | 2 +- .../src/__tests__/mapping-test.test.ts | 2 +- .../src/__tests__/web-basic.test.ts | 6 +- apps/quickstart/src/mapping-test.ts | 8 +- apps/quickstart/src/web-basic.ts | 2 +- apps/walkerjs/examples/index.html | 2 +- apps/walkerjs/src/__tests__/walkerjs.test.ts | 6 +- packages/cli/examples/event.json | 2 +- packages/cli/examples/flow-complete.json | 156 +++++++----------- packages/cli/examples/flow-complete.md | 14 +- .../cli/examples/flow-order-complete.json | 2 +- packages/cli/examples/flow.json | 2 +- .../f18-unknown-contract-ref.json | 8 +- .../f26-var-contract.json | 8 +- .../f4-dangling-extend.json | 4 +- .../f5-contract-valid.json | 4 +- .../f7-validate-step.json | 8 +- .../examples/aws-packed-flow.test.ts | 4 +- .../examples/flow-complete-simulate.test.ts | 30 +--- .../src/__tests__/unit/validate/flow.test.ts | 8 +- .../push/__tests__/simulate-isolation.test.ts | 6 +- .../src/examples/flow-complete.manifest.ts | 9 +- .../src/__tests__/boundary-error.test.ts | 2 +- .../src/__tests__/mapping-failures.test.ts | 4 +- .../src/destination-outcome-state.ts | 10 ++ .../core/src/__tests__/eventGenerator.test.ts | 38 ++++- packages/core/src/__tests__/include.test.ts | 14 +- .../mapping-error-visibility.test.ts | 4 +- packages/core/src/__tests__/mapping.test.ts | 20 +-- .../core/src/__tests__/mergeMapping.test.ts | 12 +- .../schemas/validate-flow-config.test.ts | 12 +- .../core/src/__tests__/step-examples.test.ts | 2 +- packages/core/src/eventGenerator.ts | 45 ++++- packages/core/src/mappingValue.ts | 4 +- .../mcp/src/__tests__/tools/examples.test.ts | 6 +- .../amplitude/src/examples/step.ts | 15 +- .../destinations/api/src/examples/step.ts | 9 +- .../aws/src/firehose/examples/step.ts | 9 +- .../destinations/aws/src/sns/examples/step.ts | 4 +- .../bing/src/__tests__/index.test.ts | 13 +- .../destinations/bing/src/examples/step.ts | 9 +- .../clickhouse/src/examples/step.ts | 2 +- .../criteo/src/__tests__/hash.test.ts | 4 +- .../criteo/src/__tests__/index.test.ts | 8 +- .../destinations/criteo/src/examples/step.ts | 2 +- .../customerio/src/examples/step.ts | 21 ++- .../datamanager/src/__tests__/index.test.ts | 2 +- .../datamanager/src/examples/mapping.ts | 2 +- .../datamanager/src/examples/step.ts | 9 +- .../destinations/file/src/examples/step.ts | 5 +- .../gcp/src/bigquery/examples/step.ts | 2 +- packages/server/destinations/gcp/src/hints.ts | 2 +- .../gcp/src/pubsub/examples/step.ts | 4 +- .../destinations/hubspot/src/examples/step.ts | 12 +- .../destinations/kafka/src/examples/step.ts | 15 +- .../destinations/klaviyo/src/examples/step.ts | 17 +- .../linkedin/src/__tests__/index.test.ts | 40 +++-- .../linkedin/src/examples/step.ts | 13 +- .../destinations/meta/src/examples/step.ts | 13 +- .../mixpanel/src/examples/step.ts | 21 ++- .../mparticle/src/__tests__/push.test.ts | 4 +- .../mparticle/src/examples/step.ts | 9 +- .../pinterest/src/examples/step.ts | 4 +- .../piwikpro/src/__tests__/hits.test.ts | 37 +++-- .../piwikpro/src/__tests__/push.test.ts | 31 +++- .../piwikpro/src/examples/step.ts | 27 ++- .../piwikpro/src/schemas/mapping.ts | 2 +- .../destinations/posthog/src/examples/step.ts | 10 +- .../destinations/reddit/src/examples/step.ts | 15 +- .../destinations/redis/src/examples/step.ts | 10 +- .../rudderstack/src/examples/step.ts | 17 +- .../destinations/segment/src/examples/step.ts | 14 +- .../slack/src/__tests__/message.test.ts | 13 +- .../destinations/slack/src/examples/step.ts | 24 ++- .../snapchat/src/__tests__/index.test.ts | 13 +- .../snapchat/src/examples/step.ts | 9 +- .../destinations/sqlite/src/examples/step.ts | 16 +- .../tiktok/src/__tests__/index.test.ts | 6 +- .../destinations/tiktok/src/examples/step.ts | 9 +- .../twitter/src/__tests__/index.test.ts | 52 +++--- .../destinations/twitter/src/examples/step.ts | 13 +- .../sources/aws/src/sqs/examples/step.ts | 2 +- .../sources/fetch/src/examples/inputs.ts | 2 +- .../gcp/src/cloudfunction/examples/step.ts | 4 +- .../gcp/src/pubsub/pull/examples/step.ts | 2 +- .../gcp/src/pubsub/push/examples/step.ts | 2 +- .../ga4/src/__tests__/map.test.ts | 4 +- .../ga4/src/__tests__/transformer.test.ts | 6 +- packages/transformers/ga4/src/defaults.ts | 6 +- .../transformers/ga4/src/examples/step.ts | 4 +- packages/transformers/validate/README.md | 2 +- .../validate/src/__tests__/validate.test.ts | 2 +- .../amplitude/src/examples/step.ts | 17 +- .../web/destinations/api/src/examples/step.ts | 2 +- .../web/destinations/api/src/index.test.ts | 23 ++- .../destinations/clarity/src/examples/step.ts | 9 +- packages/web/destinations/d8a/README.md | 2 +- .../web/destinations/d8a/src/examples/step.ts | 4 +- .../fullstory/src/examples/step.ts | 12 +- .../gtag/src/__tests__/ads.test.ts | 2 +- .../gtag/src/__tests__/index.test.ts | 6 +- .../gtag/src/__tests__/userData.test.ts | 8 +- .../destinations/gtag/src/examples/step.ts | 18 +- .../destinations/heap/src/examples/step.ts | 13 +- .../destinations/hotjar/src/examples/step.ts | 7 +- .../linkedin/src/examples/step.ts | 13 +- .../destinations/matomo/src/examples/step.ts | 7 +- .../web/destinations/matomo/src/index.test.ts | 13 +- .../destinations/meta/src/examples/step.ts | 7 +- .../mixpanel/src/examples/step.ts | 20 ++- .../optimizely/src/examples/step.ts | 14 +- .../destinations/piano/src/examples/step.ts | 2 +- .../pinterest/src/examples/step.ts | 15 +- .../piwikpro/src/examples/step.ts | 21 ++- .../web/destinations/piwikpro/src/hints.ts | 2 +- .../destinations/piwikpro/src/index.test.ts | 22 ++- .../piwikpro/src/schemas/mapping.ts | 2 +- .../destinations/piwikpro/src/types/index.ts | 2 +- .../plausible/src/examples/step.ts | 2 +- .../destinations/posthog/src/examples/step.ts | 17 +- .../destinations/segment/src/examples/step.ts | 17 +- .../snowplow/src/examples/step.ts | 4 +- .../destinations/snowplow/src/index.test.ts | 8 +- .../destinations/tiktok/src/examples/step.ts | 10 +- .../sources/dataLayer/src/examples/mapping.ts | 2 +- skills/walkeros-create-destination/SKILL.md | 2 +- .../examples/events.ts | 2 +- .../examples/mapping.ts | 2 +- skills/walkeros-create-source/SKILL.md | 4 +- .../examples/mapping.ts | 2 +- .../examples/outputs.ts | 2 +- skills/walkeros-debugging/SKILL.md | 2 +- .../walkeros-mapping-configuration/SKILL.md | 10 +- skills/walkeros-understanding-flow/SKILL.md | 2 +- .../walkeros-understanding-mapping/SKILL.md | 2 +- skills/walkeros-using-cli/SKILL.md | 2 +- .../walkeros-using-cli/flow-configuration.md | 2 +- skills/walkeros-using-step-examples/SKILL.md | 10 +- .../walkeros-using-transformer-ga4/SKILL.md | 6 +- website/docs/apps/cli.mdx | 6 +- website/docs/destinations/server/aws.mdx | 2 +- website/docs/destinations/server/bing.mdx | 2 +- .../docs/destinations/server/clickhouse.mdx | 2 +- .../docs/destinations/server/datamanager.mdx | 2 +- website/docs/destinations/server/linkedin.mdx | 2 +- website/docs/destinations/server/piwikpro.mdx | 6 +- website/docs/destinations/server/reddit.mdx | 2 +- website/docs/destinations/server/slack.mdx | 2 +- website/docs/destinations/server/snapchat.mdx | 2 +- website/docs/destinations/server/twitter.mdx | 2 +- website/docs/destinations/web/d8a.mdx | 2 +- website/docs/destinations/web/linkedin.mdx | 2 +- website/docs/destinations/web/piwikpro.mdx | 4 +- .../getting-started/flow/step-examples.mdx | 8 +- .../docs/getting-started/flow/validate.mdx | 6 +- .../docs/getting-started/quickstart/react.mdx | 6 +- .../getting-started/quickstart/walkerjs.mdx | 4 +- website/docs/mapping/rule.mdx | 2 +- .../web/browser/tagging/javascript.mdx | 2 +- website/docs/sources/web/dataLayer/index.mdx | 4 +- website/docs/transformers/file.mdx | 2 - website/docs/transformers/ga4.mdx | 8 +- website/scripts/flow-snippets.test.mjs | 11 +- 172 files changed, 905 insertions(+), 687 deletions(-) create mode 100644 .changeset/core-standard-event-value.md create mode 100644 .changeset/transformer-ga4-order-value.md diff --git a/.changeset/core-standard-event-value.md b/.changeset/core-standard-event-value.md new file mode 100644 index 000000000..c6f65137e --- /dev/null +++ b/.changeset/core-standard-event-value.md @@ -0,0 +1,9 @@ +--- +'@walkeros/core': patch +--- + +`getEvent` returns the same event on every run: a fixed timestamp and an id +derived from the event name. It accepts only standard event names, so a typo no +longer compiles; `createEvent` builds any other event and uses the same fixed +default timestamp. `order complete` carries the order amount as `data.value` +instead of `data.total`, and the package examples read `data.value`. diff --git a/.changeset/transformer-ga4-order-value.md b/.changeset/transformer-ga4-order-value.md new file mode 100644 index 000000000..4210f131f --- /dev/null +++ b/.changeset/transformer-ga4-order-value.md @@ -0,0 +1,7 @@ +--- +'@walkeros/transformer-ga4': patch +--- + +GA4 `purchase` and `refund` hits now decode the order amount into `data.value` +instead of `data.total`, matching the standard `order complete` event. A mapping +that reads `data.total` from these events must read `data.value` now. diff --git a/apps/demos/flows/index.html b/apps/demos/flows/index.html index d25c3c5e3..9c4a6f3dd 100644 --- a/apps/demos/flows/index.html +++ b/apps/demos/flows/index.html @@ -101,7 +101,7 @@

Quick Events

- diff --git a/apps/demos/flows/pentest/load-test.js b/apps/demos/flows/pentest/load-test.js index 1c6478c18..ac036e504 100644 --- a/apps/demos/flows/pentest/load-test.js +++ b/apps/demos/flows/pentest/load-test.js @@ -39,7 +39,7 @@ const events = [ name: 'product view', data: { id: 'SKU-001', name: 'walkerOS Pro', price: 299 }, }, - { name: 'order complete', data: { orderId: 'ORD-12345', total: 299.99 } }, + { name: 'order complete', data: { orderId: 'ORD-12345', value: 299.99 } }, ]; export default function () { diff --git a/apps/demos/react/src/components/Order.tsx b/apps/demos/react/src/components/Order.tsx index 36c290a2e..c9bd23d33 100644 --- a/apps/demos/react/src/components/Order.tsx +++ b/apps/demos/react/src/components/Order.tsx @@ -84,7 +84,7 @@ function Order({ orderId }: OrderProps) {

Total

€{orderDetails.total.toFixed(2)} diff --git a/apps/demos/storybook/src/demos/__fixtures__/shop-attributes.json b/apps/demos/storybook/src/demos/__fixtures__/shop-attributes.json index 982709e24..c0358a33d 100644 --- a/apps/demos/storybook/src/demos/__fixtures__/shop-attributes.json +++ b/apps/demos/storybook/src/demos/__fixtures__/shop-attributes.json @@ -2,14 +2,14 @@ "data-elb-checkout=shipping:5;currency:EUR", "data-elb-checkout=subtotal:249;currency:EUR", "data-elb-checkout=taxes:5.52;currency:EUR", - "data-elb-checkout=total:259.52;currency:EUR", + "data-elb-checkout=value:259.52;currency:EUR", "data-elb-order=id:0rd3r1d", "data-elb-order=shipping-city:20354 Hamburg", "data-elb-order=shipping-country:Germany", "data-elb-order=shipping:5", "data-elb-order=subtotal:249;currency:EUR", "data-elb-order=taxes:5.52", - "data-elb-order=total:259.52", + "data-elb-order=value:259.52", "data-elb-product=availability:in_stock", "data-elb-product=category:Travel/Bags", "data-elb-product=color:black", diff --git a/apps/demos/storybook/src/demos/shop/data.ts b/apps/demos/storybook/src/demos/shop/data.ts index 13d09a5e5..431e81bfb 100644 --- a/apps/demos/storybook/src/demos/shop/data.ts +++ b/apps/demos/storybook/src/demos/shop/data.ts @@ -176,7 +176,7 @@ export const checkoutSummary: SummaryLine[] = [ { label: 'Taxes', property: 'taxes', amount: 5.52, currency: 'EUR' }, { label: 'Total', - property: 'total', + property: 'value', amount: 259.52, currency: 'EUR', emphasis: true, @@ -207,7 +207,7 @@ export const order: OrderContent = { { label: 'Subtotal', property: 'subtotal', amount: 249, currency: 'EUR' }, { label: 'Shipping', property: 'shipping', amount: 5 }, { label: 'Taxes', property: 'taxes', amount: 5.52 }, - { label: 'Total', property: 'total', amount: 259.52, emphasis: true }, + { label: 'Total', property: 'value', amount: 259.52, emphasis: true }, ], address: { lines: ['elbwalker GmbH', 'Gerhofstraße 1-3'], diff --git a/apps/demos/tagging/index.html b/apps/demos/tagging/index.html index 7b04db5df..335d6bc2e 100644 --- a/apps/demos/tagging/index.html +++ b/apps/demos/tagging/index.html @@ -1163,7 +1163,7 @@

>
Total
€259.52 @@ -1304,7 +1304,7 @@

class="flex items-center justify-between border-t border-gray-200 text-gray-900 pt-6" >
Total
-
+
€259.52

diff --git a/apps/demos/tagging/index.local.html b/apps/demos/tagging/index.local.html index 415ad9f45..0c5d8ba27 100644 --- a/apps/demos/tagging/index.local.html +++ b/apps/demos/tagging/index.local.html @@ -1182,7 +1182,7 @@

>
Total
€259.52 @@ -1323,7 +1323,7 @@

class="flex items-center justify-between border-t border-gray-200 text-gray-900 pt-6" >
Total
-
+
€259.52
diff --git a/apps/explorer/src/components/molecules/code-box.stories.tsx b/apps/explorer/src/components/molecules/code-box.stories.tsx index a0f31112c..b7f88ebba 100644 --- a/apps/explorer/src/components/molecules/code-box.stories.tsx +++ b/apps/explorer/src/components/molecules/code-box.stories.tsx @@ -231,7 +231,7 @@ export const DynamicFlowContext: Story = { data: { map: { value: { - key: 'data.total', + key: 'data.value', fn: '$code:(value) => Number(value)', }, }, diff --git a/apps/quickstart/src/__tests__/mapping-test.test.ts b/apps/quickstart/src/__tests__/mapping-test.test.ts index 663597c89..248949d3d 100644 --- a/apps/quickstart/src/__tests__/mapping-test.test.ts +++ b/apps/quickstart/src/__tests__/mapping-test.test.ts @@ -24,7 +24,7 @@ describe('walkerOS Mapping Example', () => { data: { item_id: 'P123', quantity: 2 }, rule: 'add_to_cart', }, - // order complete -> purchase, tax computed as total * 0.1 + // order complete -> purchase, tax computed as value * 0.1 { name: 'purchase', data: { diff --git a/apps/quickstart/src/__tests__/web-basic.test.ts b/apps/quickstart/src/__tests__/web-basic.test.ts index 5588526f1..a22d16adf 100644 --- a/apps/quickstart/src/__tests__/web-basic.test.ts +++ b/apps/quickstart/src/__tests__/web-basic.test.ts @@ -93,7 +93,7 @@ describe('walkerOS Web Basic Example', () => { data: { map: { transaction_id: 'data.id', - value: 'data.total', + value: 'data.value', currency: { value: 'EUR' }, }, }, @@ -159,8 +159,8 @@ describe('walkerOS Web Basic Example', () => { 'purchase', expect.objectContaining({ transaction_id: orderEvent.data.id, - value: orderEvent.data.total, - data_total: 555, + value: orderEvent.data.value, + data_value: 555, currency: 'EUR', }), ); diff --git a/apps/quickstart/src/mapping-test.ts b/apps/quickstart/src/mapping-test.ts index 7270f273d..4a7dd5cde 100644 --- a/apps/quickstart/src/mapping-test.ts +++ b/apps/quickstart/src/mapping-test.ts @@ -85,14 +85,14 @@ export async function testMapping(): Promise { data: { map: { transaction_id: 'data.id', - value: 'data.total', + value: 'data.value', currency: { value: 'EUR' }, // Custom function tax: { fn: (value) => { const event = value as WalkerOS.Event; - const total = (event.data?.total as number) || 0; - return total * 0.1; + const orderValue = (event.data?.value as number) || 0; + return orderValue * 0.1; }, }, }, @@ -118,7 +118,7 @@ export async function testMapping(): Promise { await elb({ name: 'product add', data: { id: 'P123', quantity: 2 } }); // Order complete with calculated tax - await elb({ name: 'order complete', data: { id: 'ORDER-789', total: 1000 } }); + await elb({ name: 'order complete', data: { id: 'ORDER-789', value: 1000 } }); const { collector } = await startFlow({ sources: { diff --git a/apps/quickstart/src/web-basic.ts b/apps/quickstart/src/web-basic.ts index e3b913d91..88bb632a9 100644 --- a/apps/quickstart/src/web-basic.ts +++ b/apps/quickstart/src/web-basic.ts @@ -95,7 +95,7 @@ export async function setupWebBasic(): Promise<{ data: { map: { transaction_id: 'data.id', - value: 'data.total', + value: 'data.value', currency: { key: 'data.currency', value: 'EUR' }, tax: 'data.tax', shipping: 'data.shipping', diff --git a/apps/walkerjs/examples/index.html b/apps/walkerjs/examples/index.html index 0e336edfa..adb11b9b9 100644 --- a/apps/walkerjs/examples/index.html +++ b/apps/walkerjs/examples/index.html @@ -32,7 +32,7 @@

Cotton Tee

- +

dataLayer pushes


diff --git a/apps/walkerjs/src/__tests__/walkerjs.test.ts b/apps/walkerjs/src/__tests__/walkerjs.test.ts
index f14aa664e..ea08207d4 100644
--- a/apps/walkerjs/src/__tests__/walkerjs.test.ts
+++ b/apps/walkerjs/src/__tests__/walkerjs.test.ts
@@ -123,7 +123,7 @@ describe('the entry', () => {
   it('delivers calls queued before load, before the page view', async () => {
     Reflect.set(window, 'elbLayer', [
       ['walker user', { id: 'u1' }],
-      ['order complete', { total: 1 }],
+      ['order complete', { value: 1 }],
     ]);
     load();
     await settle();
@@ -137,12 +137,12 @@ describe('the entry', () => {
     setReadyState('loading');
     load();
     const elb: unknown = Reflect.get(window, 'elb');
-    if (typeof elb === 'function') elb('order complete', { total: 2 });
+    if (typeof elb === 'function') elb('order complete', { value: 2 });
     setReadyState('complete');
     document.dispatchEvent(new Event('DOMContentLoaded'));
     await settle();
     expect(named('order complete')).toEqual([
-      expect.objectContaining({ data: { total: 2 } }),
+      expect.objectContaining({ data: { value: 2 } }),
     ]);
   });
 
diff --git a/packages/cli/examples/event.json b/packages/cli/examples/event.json
index 6352f25e8..50a9d6934 100644
--- a/packages/cli/examples/event.json
+++ b/packages/cli/examples/event.json
@@ -2,7 +2,7 @@
   "name": "order complete",
   "data": {
     "id": "ORD-2024-12345",
-    "total": 299.99,
+    "value": 299.99,
     "currency": "USD",
     "tax": 24.99,
     "shipping": 15.0
diff --git a/packages/cli/examples/flow-complete.json b/packages/cli/examples/flow-complete.json
index 25a5a396a..f22464e11 100644
--- a/packages/cli/examples/flow-complete.json
+++ b/packages/cli/examples/flow-complete.json
@@ -140,13 +140,13 @@
             "properties": {
               "data": {
                 "type": "object",
-                "required": ["id", "total", "currency"],
+                "required": ["id", "value", "currency"],
                 "properties": {
                   "id": {
                     "type": "string",
                     "minLength": 1
                   },
-                  "total": {
+                  "value": {
                     "type": "number",
                     "minimum": 0
                   },
@@ -736,7 +736,7 @@
                 "action": "complete",
                 "data": {
                   "id": "ORD-100",
-                  "total": 149.8,
+                  "value": 149.8,
                   "currency": "EUR",
                   "coupon": "WELCOME10"
                 },
@@ -790,7 +790,7 @@
                       "action": "complete",
                       "data": {
                         "id": "ORD-100",
-                        "total": 149.8,
+                        "value": 149.8,
                         "currency": "EUR",
                         "coupon": "WELCOME10"
                       },
@@ -966,12 +966,12 @@
               "order": {
                 "complete": {
                   "name": "purchase",
-                  "condition": "$code:(event) => event.data && event.data.total > 0",
+                  "condition": "$code:(event) => event.data && event.data.value > 0",
                   "data": {
                     "map": {
                       "transaction_id": "data.id",
                       "value": {
-                        "fn": "$code:(event) => Math.round(Number(event.data.total) * 100) / 100"
+                        "fn": "$code:(event) => Math.round(Number(event.data.value) * 100) / 100"
                       },
                       "currency": [
                         "data.currency",
@@ -1013,12 +1013,12 @@
               "public": true,
               "mapping": {
                 "name": "purchase",
-                "condition": "$code:(event) => event.data && event.data.total > 0",
+                "condition": "$code:(event) => event.data && event.data.value > 0",
                 "data": {
                   "map": {
                     "transaction_id": "data.id",
                     "value": {
-                      "fn": "$code:(event) => Math.round(Number(event.data.total) * 100) / 100"
+                      "fn": "$code:(event) => Math.round(Number(event.data.value) * 100) / 100"
                     },
                     "currency": [
                       "data.currency",
@@ -1056,7 +1056,7 @@
                 "action": "complete",
                 "data": {
                   "id": "ORD-100",
-                  "total": 149.8,
+                  "value": 149.8,
                   "currency": "EUR",
                   "coupon": "WELCOME10"
                 },
@@ -1150,12 +1150,12 @@
               "description": "The coupon condition leaves the parameter out when the order has none.",
               "mapping": {
                 "name": "purchase",
-                "condition": "$code:(event) => event.data && event.data.total > 0",
+                "condition": "$code:(event) => event.data && event.data.value > 0",
                 "data": {
                   "map": {
                     "transaction_id": "data.id",
                     "value": {
-                      "fn": "$code:(event) => Math.round(Number(event.data.total) * 100) / 100"
+                      "fn": "$code:(event) => Math.round(Number(event.data.value) * 100) / 100"
                     },
                     "currency": [
                       "data.currency",
@@ -1193,7 +1193,7 @@
                 "action": "complete",
                 "data": {
                   "id": "ORD-101",
-                  "total": 129.9,
+                  "value": 129.9,
                   "currency": "EUR"
                 },
                 "id": "ev-order-complete",
@@ -1626,7 +1626,7 @@
                 "action": "complete",
                 "data": {
                   "id": "ORD-100",
-                  "total": 149.8,
+                  "value": 149.8,
                   "currency": "EUR",
                   "coupon": "WELCOME10"
                 },
@@ -1677,7 +1677,7 @@
                 [
                   "sendWeb",
                   "http://localhost:8080/collect",
-                  "[{\"name\":\"order complete\",\"data\":{\"id\":\"ORD-100\",\"total\":149.8,\"currency\":\"EUR\",\"coupon\":\"WELCOME10\"},\"context\":{},\"globals\":{\"site\":\"shop\",\"language\":\"en\",\"pageGroup\":\"checkout\"},\"custom\":{},\"user\":{\"device\":\"d3v1c3\",\"session\":\"s3ss10n\",\"id\":\"cust-42\",\"email\":\"jane@example.com\"},\"nested\":[{\"entity\":\"product\",\"data\":{\"id\":\"SKU-1\",\"name\":\"Trail Runner\",\"price\":129.9,\"quantity\":1}},{\"entity\":\"product\",\"data\":{\"id\":\"SKU-2\",\"name\":\"Running Socks\",\"price\":9.95,\"quantity\":2}}],\"consent\":{\"functional\":true,\"marketing\":true},\"id\":\"ev-order-complete\",\"trigger\":\"load\",\"entity\":\"order\",\"action\":\"complete\",\"timestamp\":1700000000000,\"source\":{\"type\":\"browser\",\"platform\":\"web\",\"url\":\"https://www.example.com/checkout/thanks\",\"count\":1,\"release\":{\"web\":\"4.6.1\"}}}]",
+                  "[{\"name\":\"order complete\",\"data\":{\"id\":\"ORD-100\",\"value\":149.8,\"currency\":\"EUR\",\"coupon\":\"WELCOME10\"},\"context\":{},\"globals\":{\"site\":\"shop\",\"language\":\"en\",\"pageGroup\":\"checkout\"},\"custom\":{},\"user\":{\"device\":\"d3v1c3\",\"session\":\"s3ss10n\",\"id\":\"cust-42\",\"email\":\"jane@example.com\"},\"nested\":[{\"entity\":\"product\",\"data\":{\"id\":\"SKU-1\",\"name\":\"Trail Runner\",\"price\":129.9,\"quantity\":1}},{\"entity\":\"product\",\"data\":{\"id\":\"SKU-2\",\"name\":\"Running Socks\",\"price\":9.95,\"quantity\":2}}],\"consent\":{\"functional\":true,\"marketing\":true},\"id\":\"ev-order-complete\",\"trigger\":\"load\",\"entity\":\"order\",\"action\":\"complete\",\"timestamp\":1700000000000,\"source\":{\"type\":\"browser\",\"platform\":\"web\",\"url\":\"https://www.example.com/checkout/thanks\",\"count\":1,\"release\":{\"web\":\"4.6.1\"}}}]",
                   {
                     "transport": "beacon"
                   }
@@ -1693,7 +1693,7 @@
                 "action": "complete",
                 "data": {
                   "id": "ORD-100",
-                  "total": 149.8,
+                  "value": 149.8,
                   "currency": "EUR",
                   "coupon": "WELCOME10"
                 },
@@ -1743,7 +1743,7 @@
                 [
                   "sendWeb",
                   "http://localhost:8080/collect",
-                  "[{\"name\":\"order complete\",\"data\":{\"id\":\"ORD-100\",\"total\":149.8,\"currency\":\"EUR\",\"coupon\":\"WELCOME10\"},\"context\":{},\"globals\":{\"site\":\"shop\",\"language\":\"en\",\"pageGroup\":\"checkout\"},\"custom\":{},\"user\":{\"device\":\"d3v1c3\",\"session\":\"s3ss10n\",\"id\":\"cust-42\"},\"nested\":[{\"entity\":\"product\",\"data\":{\"id\":\"SKU-1\",\"name\":\"Trail Runner\",\"price\":129.9,\"quantity\":1}},{\"entity\":\"product\",\"data\":{\"id\":\"SKU-2\",\"name\":\"Running Socks\",\"price\":9.95,\"quantity\":2}}],\"consent\":{\"functional\":true},\"id\":\"ev-order-complete\",\"trigger\":\"load\",\"entity\":\"order\",\"action\":\"complete\",\"timestamp\":1700000000000,\"source\":{\"type\":\"browser\",\"platform\":\"web\",\"url\":\"https://www.example.com/checkout/thanks\",\"count\":1,\"release\":{\"web\":\"4.6.1\"}}}]",
+                  "[{\"name\":\"order complete\",\"data\":{\"id\":\"ORD-100\",\"value\":149.8,\"currency\":\"EUR\",\"coupon\":\"WELCOME10\"},\"context\":{},\"globals\":{\"site\":\"shop\",\"language\":\"en\",\"pageGroup\":\"checkout\"},\"custom\":{},\"user\":{\"device\":\"d3v1c3\",\"session\":\"s3ss10n\",\"id\":\"cust-42\"},\"nested\":[{\"entity\":\"product\",\"data\":{\"id\":\"SKU-1\",\"name\":\"Trail Runner\",\"price\":129.9,\"quantity\":1}},{\"entity\":\"product\",\"data\":{\"id\":\"SKU-2\",\"name\":\"Running Socks\",\"price\":9.95,\"quantity\":2}}],\"consent\":{\"functional\":true},\"id\":\"ev-order-complete\",\"trigger\":\"load\",\"entity\":\"order\",\"action\":\"complete\",\"timestamp\":1700000000000,\"source\":{\"type\":\"browser\",\"platform\":\"web\",\"url\":\"https://www.example.com/checkout/thanks\",\"count\":1,\"release\":{\"web\":\"4.6.1\"}}}]",
                   {
                     "transport": "beacon"
                   }
@@ -1966,7 +1966,7 @@
                   "action": "complete",
                   "data": {
                     "id": "ORD-100",
-                    "total": 149.8,
+                    "value": 149.8,
                     "currency": "EUR",
                     "coupon": "WELCOME10"
                   },
@@ -2023,7 +2023,7 @@
                     "action": "complete",
                     "data": {
                       "id": "ORD-100",
-                      "total": 149.8,
+                      "value": 149.8,
                       "currency": "EUR",
                       "coupon": "WELCOME10"
                     },
@@ -2117,7 +2117,7 @@
                       "data": {
                         "id": "SUB-77",
                         "currency": "EUR",
-                        "total": 59.9
+                        "value": 59.9
                       },
                       "user": {
                         "device": "cid-9",
@@ -2407,42 +2407,6 @@
           },
           "next": {
             "stop": true
-          },
-          "examples": {
-            "walkerJs": {
-              "title": "Serve walker.js first party",
-              "description": "GET /walker.js is answered from the assets store with the configured headers; the step returns false and its next stops, so a script request never becomes an event. Proven over HTTP in the flow test.",
-              "public": true,
-              "in": {
-                "name": "page view",
-                "entity": "page",
-                "action": "view",
-                "data": {
-                  "domain": "www.example.com",
-                  "title": "Trail Runner",
-                  "id": "/product/trail-runner"
-                },
-                "id": "ev-page-view",
-                "trigger": "load",
-                "timestamp": 1700000000000,
-                "consent": {
-                  "functional": true,
-                  "marketing": true
-                },
-                "globals": {
-                  "language": "en",
-                  "pageGroup": "product"
-                },
-                "user": {
-                  "device": "d3v1c3",
-                  "session": "s3ss10n"
-                },
-                "source": {
-                  "type": "express",
-                  "platform": "server"
-                }
-              }
-            }
           }
         },
         "dedup": {
@@ -2467,7 +2431,7 @@
                 "action": "complete",
                 "data": {
                   "id": "ORD-100",
-                  "total": 149.8,
+                  "value": 149.8,
                   "currency": "EUR",
                   "coupon": "WELCOME10"
                 },
@@ -2524,7 +2488,7 @@
                       "action": "complete",
                       "data": {
                         "id": "ORD-100",
-                        "total": 149.8,
+                        "value": 149.8,
                         "currency": "EUR",
                         "coupon": "WELCOME10"
                       },
@@ -2597,7 +2561,7 @@
                 "action": "complete",
                 "data": {
                   "id": "ORD-100",
-                  "total": 149.8,
+                  "value": 149.8,
                   "currency": "EUR",
                   "coupon": "WELCOME10"
                 },
@@ -2654,7 +2618,7 @@
                       "action": "complete",
                       "data": {
                         "id": "ORD-100",
-                        "total": 149.8,
+                        "value": 149.8,
                         "currency": "EUR",
                         "coupon": "WELCOME10"
                       },
@@ -2721,7 +2685,7 @@
                 "action": "complete",
                 "data": {
                   "id": "ORD-100",
-                  "total": 149.8,
+                  "value": 149.8,
                   "currency": "EUR",
                   "coupon": "WELCOME10"
                 },
@@ -2778,7 +2742,7 @@
                       "action": "complete",
                       "data": {
                         "id": "ORD-100",
-                        "total": 149.8,
+                        "value": 149.8,
                         "currency": "EUR",
                         "coupon": "WELCOME10"
                       },
@@ -2840,7 +2804,7 @@
                 "action": "complete",
                 "data": {
                   "id": "ORD-100",
-                  "total": 149.8,
+                  "value": 149.8,
                   "currency": "EUR",
                   "coupon": "WELCOME10"
                 },
@@ -2897,7 +2861,7 @@
                       "action": "complete",
                       "data": {
                         "id": "ORD-100",
-                        "total": 149.8,
+                        "value": 149.8,
                         "currency": "EUR",
                         "coupon": "WELCOME10"
                       },
@@ -2991,7 +2955,7 @@
                 "action": "complete",
                 "data": {
                   "id": "ORD-100",
-                  "total": 149.8,
+                  "value": 149.8,
                   "currency": "EUR",
                   "coupon": "WELCOME10"
                 },
@@ -3051,7 +3015,7 @@
                       "action": "complete",
                       "data": {
                         "id": "ORD-100",
-                        "total": 149.8,
+                        "value": 149.8,
                         "currency": "EUR",
                         "coupon": "WELCOME10"
                       },
@@ -3134,7 +3098,7 @@
                 "action": "complete",
                 "data": {
                   "id": "ORD-100",
-                  "total": 149.8,
+                  "value": 149.8,
                   "currency": "EUR",
                   "coupon": "WELCOME10"
                 },
@@ -3194,7 +3158,7 @@
                       "action": "complete",
                       "data": {
                         "id": "ORD-100",
-                        "total": 149.8,
+                        "value": 149.8,
                         "currency": "EUR",
                         "coupon": "WELCOME10"
                       },
@@ -3365,7 +3329,7 @@
                 "action": "complete",
                 "data": {
                   "id": "ORD-100",
-                  "total": 149.8,
+                  "value": 149.8,
                   "currency": "EUR",
                   "coupon": "WELCOME10"
                 },
@@ -3425,7 +3389,7 @@
                       "action": "complete",
                       "data": {
                         "id": "ORD-100",
-                        "total": 149.8,
+                        "value": 149.8,
                         "currency": "EUR",
                         "coupon": "WELCOME10"
                       },
@@ -3500,7 +3464,7 @@
                 "action": "complete",
                 "data": {
                   "id": "ORD-100",
-                  "total": 149.8,
+                  "value": 149.8,
                   "currency": "EUR",
                   "coupon": "WELCOME10"
                 },
@@ -3561,7 +3525,7 @@
                       "action": "complete",
                       "data": {
                         "id": "ORD-100",
-                        "total": 149.8,
+                        "value": 149.8,
                         "currency": "EUR",
                         "coupon": "WELCOME10"
                       },
@@ -3626,7 +3590,7 @@
                 "action": "complete",
                 "data": {
                   "id": "ORD-100",
-                  "total": 149.8,
+                  "value": 149.8,
                   "currency": "EUR",
                   "coupon": "WELCOME10"
                 },
@@ -3685,7 +3649,7 @@
                       "action": "complete",
                       "data": {
                         "id": "ORD-100",
-                        "total": 149.8,
+                        "value": 149.8,
                         "currency": "EUR",
                         "coupon": "WELCOME10"
                       },
@@ -3775,7 +3739,7 @@
                 "action": "complete",
                 "data": {
                   "id": "ORD-100",
-                  "total": 149.8,
+                  "value": 149.8,
                   "currency": "EUR",
                   "coupon": "WELCOME10"
                 },
@@ -3837,7 +3801,7 @@
                       "action": "complete",
                       "data": {
                         "id": "ORD-100",
-                        "total": 149.8,
+                        "value": 149.8,
                         "currency": "EUR",
                         "coupon": "WELCOME10"
                       },
@@ -3942,7 +3906,7 @@
                 "action": "complete",
                 "data": {
                   "id": "ORD-100",
-                  "total": 149.8,
+                  "value": 149.8,
                   "currency": "EUR",
                   "coupon": "WELCOME10"
                 },
@@ -4005,7 +3969,7 @@
                 "action": "complete",
                 "data": {
                   "id": "ORD-100",
-                  "total": 149.8,
+                  "value": 149.8,
                   "currency": "EUR",
                   "coupon": "WELCOME10"
                 },
@@ -4086,7 +4050,7 @@
                 "action": "complete",
                 "data": {
                   "id": "ORD-100",
-                  "total": 149.8,
+                  "value": 149.8,
                   "currency": "EUR",
                   "coupon": "WELCOME10"
                 },
@@ -4149,7 +4113,7 @@
                       "action": "complete",
                       "data": {
                         "id": "ORD-100",
-                        "total": 149.8,
+                        "value": 149.8,
                         "currency": "EUR",
                         "coupon": "WELCOME10"
                       },
@@ -4254,7 +4218,7 @@
                 "action": "complete",
                 "data": {
                   "id": "ORD-100",
-                  "total": 149.8,
+                  "value": 149.8,
                   "currency": "EUR",
                   "coupon": "WELCOME10"
                 },
@@ -4311,7 +4275,7 @@
                 [
                   "PubSub.topic.publishMessage",
                   {
-                    "data": "{\"name\":\"order complete\",\"data\":{\"id\":\"ORD-100\",\"total\":149.8,\"currency\":\"EUR\",\"coupon\":\"WELCOME10\"},\"context\":{},\"globals\":{\"language\":\"en\",\"pageGroup\":\"checkout\"},\"custom\":{},\"user\":{\"device\":\"d3v1c3\",\"session\":\"s3ss10n\",\"id\":\"cust-42\",\"email\":\"4cdea64501d6109d0a25750a7ce1d5c87e5fc927c61b674f191b73e28b582c93\",\"hash\":\"cea3ca84340feb4a\",\"botScore\":null,\"botCategory\":\"unknown\",\"ltv\":420,\"segment\":\"loyal\"},\"nested\":[{\"entity\":\"product\",\"data\":{\"id\":\"SKU-1\",\"name\":\"Trail Runner\",\"price\":129.9,\"quantity\":1}},{\"entity\":\"product\",\"data\":{\"id\":\"SKU-2\",\"name\":\"Running Socks\",\"price\":9.95,\"quantity\":2}}],\"consent\":{\"functional\":true,\"marketing\":true},\"id\":\"ev-order-complete\",\"trigger\":\"load\",\"entity\":\"order\",\"action\":\"complete\",\"timestamp\":1700000000000,\"timing\":0,\"source\":{\"type\":\"express\",\"platform\":\"server\",\"url\":\"https://www.example.com/checkout/thanks\",\"valid\":true,\"count\":1,\"release\":{\"server\":\"4.6.1\"}}}"
+                    "data": "{\"name\":\"order complete\",\"data\":{\"id\":\"ORD-100\",\"value\":149.8,\"currency\":\"EUR\",\"coupon\":\"WELCOME10\"},\"context\":{},\"globals\":{\"language\":\"en\",\"pageGroup\":\"checkout\"},\"custom\":{},\"user\":{\"device\":\"d3v1c3\",\"session\":\"s3ss10n\",\"id\":\"cust-42\",\"email\":\"4cdea64501d6109d0a25750a7ce1d5c87e5fc927c61b674f191b73e28b582c93\",\"hash\":\"cea3ca84340feb4a\",\"botScore\":null,\"botCategory\":\"unknown\",\"ltv\":420,\"segment\":\"loyal\"},\"nested\":[{\"entity\":\"product\",\"data\":{\"id\":\"SKU-1\",\"name\":\"Trail Runner\",\"price\":129.9,\"quantity\":1}},{\"entity\":\"product\",\"data\":{\"id\":\"SKU-2\",\"name\":\"Running Socks\",\"price\":9.95,\"quantity\":2}}],\"consent\":{\"functional\":true,\"marketing\":true},\"id\":\"ev-order-complete\",\"trigger\":\"load\",\"entity\":\"order\",\"action\":\"complete\",\"timestamp\":1700000000000,\"timing\":0,\"source\":{\"type\":\"express\",\"platform\":\"server\",\"url\":\"https://www.example.com/checkout/thanks\",\"valid\":true,\"count\":1,\"release\":{\"server\":\"4.6.1\"}}}"
                   }
                 ]
               ]
@@ -4361,7 +4325,7 @@
                   },
                   "data": {
                     "map": {
-                      "value": "data.total",
+                      "value": "data.value",
                       "currency": [
                         "data.currency",
                         {
@@ -4411,7 +4375,7 @@
                 },
                 "data": {
                   "map": {
-                    "value": "data.total",
+                    "value": "data.value",
                     "currency": [
                       "data.currency",
                       {
@@ -4428,7 +4392,7 @@
                 "action": "complete",
                 "data": {
                   "id": "ORD-100",
-                  "total": 149.8,
+                  "value": 149.8,
                   "currency": "EUR",
                   "coupon": "WELCOME10"
                 },
@@ -4570,7 +4534,7 @@
                   "name": "ecommerceOrder",
                   "settings": {
                     "goalId": "$env.PIWIK_ORDER_GOAL:6a0f4c1e-3b2d-4e8f-a7c9-1d2e3f4a5b6c",
-                    "goalValue": "data.total"
+                    "goalValue": "data.value"
                   },
                   "data": {
                     "set": [
@@ -4593,7 +4557,7 @@
                       {
                         "map": {
                           "orderId": "data.id",
-                          "grandTotal": "data.total"
+                          "grandTotal": "data.value"
                         }
                       }
                     ]
@@ -4616,7 +4580,7 @@
                 "name": "ecommerceOrder",
                 "settings": {
                   "goalId": "$env.PIWIK_ORDER_GOAL:6a0f4c1e-3b2d-4e8f-a7c9-1d2e3f4a5b6c",
-                  "goalValue": "data.total"
+                  "goalValue": "data.value"
                 },
                 "data": {
                   "set": [
@@ -4639,7 +4603,7 @@
                     {
                       "map": {
                         "orderId": "data.id",
-                        "grandTotal": "data.total"
+                        "grandTotal": "data.value"
                       }
                     }
                   ]
@@ -4651,7 +4615,7 @@
                 "action": "complete",
                 "data": {
                   "id": "ORD-100",
-                  "total": 149.8,
+                  "value": 149.8,
                   "currency": "EUR",
                   "coupon": "WELCOME10"
                 },
@@ -4743,7 +4707,7 @@
                   "data": {
                     "map": {
                       "transactionId": "data.id",
-                      "conversionValue": "data.total",
+                      "conversionValue": "data.value",
                       "currency": [
                         "data.currency",
                         {
@@ -4777,7 +4741,7 @@
                 "data": {
                   "map": {
                     "transactionId": "data.id",
-                    "conversionValue": "data.total",
+                    "conversionValue": "data.value",
                     "currency": [
                       "data.currency",
                       {
@@ -4794,7 +4758,7 @@
                 "action": "complete",
                 "data": {
                   "id": "ORD-100",
-                  "total": 149.8,
+                  "value": 149.8,
                   "currency": "EUR",
                   "coupon": "WELCOME10",
                   "session": {
@@ -4913,7 +4877,7 @@
               "public": true,
               "in": {
                 "id": "msg-1",
-                "dataString": "{\"name\":\"order complete\",\"entity\":\"order\",\"action\":\"complete\",\"data\":{\"id\":\"ORD-100\",\"total\":149.8,\"currency\":\"EUR\",\"coupon\":\"WELCOME10\"},\"id\":\"ev-order-complete\",\"trigger\":\"load\",\"timestamp\":1700000000000,\"consent\":{\"functional\":true,\"marketing\":true},\"globals\":{\"language\":\"en\",\"pageGroup\":\"checkout\"},\"user\":{\"device\":\"d3v1c3\",\"session\":\"s3ss10n\",\"id\":\"cust-42\",\"email\":\"31b4ae339b671c9e81adbe27e74b38f09a4756300de6b81998c83a3964cbb849\",\"hash\":\"5f1c0a9e7d3b2c41\",\"botScore\":0,\"botCategory\":\"human\",\"ltv\":420,\"segment\":\"loyal\"},\"source\":{\"type\":\"express\",\"platform\":\"server\",\"url\":\"https://www.example.com/checkout/thanks\",\"valid\":true},\"nested\":[{\"entity\":\"product\",\"data\":{\"id\":\"SKU-1\",\"name\":\"Trail Runner\",\"price\":129.9,\"quantity\":1}},{\"entity\":\"product\",\"data\":{\"id\":\"SKU-2\",\"name\":\"Running Socks\",\"price\":9.95,\"quantity\":2}}]}",
+                "dataString": "{\"name\":\"order complete\",\"entity\":\"order\",\"action\":\"complete\",\"data\":{\"id\":\"ORD-100\",\"value\":149.8,\"currency\":\"EUR\",\"coupon\":\"WELCOME10\"},\"id\":\"ev-order-complete\",\"trigger\":\"load\",\"timestamp\":1700000000000,\"consent\":{\"functional\":true,\"marketing\":true},\"globals\":{\"language\":\"en\",\"pageGroup\":\"checkout\"},\"user\":{\"device\":\"d3v1c3\",\"session\":\"s3ss10n\",\"id\":\"cust-42\",\"email\":\"31b4ae339b671c9e81adbe27e74b38f09a4756300de6b81998c83a3964cbb849\",\"hash\":\"5f1c0a9e7d3b2c41\",\"botScore\":0,\"botCategory\":\"human\",\"ltv\":420,\"segment\":\"loyal\"},\"source\":{\"type\":\"express\",\"platform\":\"server\",\"url\":\"https://www.example.com/checkout/thanks\",\"valid\":true},\"nested\":[{\"entity\":\"product\",\"data\":{\"id\":\"SKU-1\",\"name\":\"Trail Runner\",\"price\":129.9,\"quantity\":1}},{\"entity\":\"product\",\"data\":{\"id\":\"SKU-2\",\"name\":\"Running Socks\",\"price\":9.95,\"quantity\":2}}]}",
                 "attributes": {}
               },
               "out": [
@@ -4925,7 +4889,7 @@
                     "action": "complete",
                     "data": {
                       "id": "ORD-100",
-                      "total": 149.8,
+                      "value": 149.8,
                       "currency": "EUR",
                       "coupon": "WELCOME10"
                     },
@@ -5017,7 +4981,7 @@
                 "action": "complete",
                 "data": {
                   "id": "ORD-100",
-                  "total": 149.8,
+                  "value": 149.8,
                   "currency": "EUR",
                   "coupon": "WELCOME10"
                 },
@@ -5075,7 +5039,7 @@
                   [
                     {
                       "name": "order complete",
-                      "data": "{\"id\":\"ORD-100\",\"total\":149.8,\"currency\":\"EUR\",\"coupon\":\"WELCOME10\"}",
+                      "data": "{\"id\":\"ORD-100\",\"value\":149.8,\"currency\":\"EUR\",\"coupon\":\"WELCOME10\"}",
                       "context": null,
                       "globals": "{\"language\":\"en\",\"pageGroup\":\"checkout\"}",
                       "custom": null,
diff --git a/packages/cli/examples/flow-complete.md b/packages/cli/examples/flow-complete.md
index 4a4016b2a..f1d17dc95 100644
--- a/packages/cli/examples/flow-complete.md
+++ b/packages/cli/examples/flow-complete.md
@@ -256,7 +256,7 @@ curl -I http://localhost:8080/walker.js
 
 ```bash
 walkeros validate packages/cli/examples/flow-complete.json -f web --path destinations.collect
-walkeros push packages/cli/examples/flow-complete.json -f web -e '{"name":"order complete","data":{"id":"ORD-100","total":149.8,"currency":"EUR"}}' --simulate destination.ga4
+walkeros push packages/cli/examples/flow-complete.json -f web -e '{"name":"order complete","data":{"id":"ORD-100","value":149.8,"currency":"EUR"}}' --simulate destination.ga4
 ```
 
 - The second line prints `mapping: none (skipped before mapping)`: the event has
@@ -294,8 +294,8 @@ walkeros push packages/cli/examples/flow-complete.json -f web -e '{"name":"order
     key reads a path from the event.
   - `value-fn` at
     `/flows/web/destinations/ga4/config/mapping/order/complete/data/map/value/fn`:
-    fn computes a value from the event, here the total rounded to cents. fn and
-    key are alternatives: next to a key, fn never runs.
+    fn computes a value from the event, here the order value rounded to cents.
+    fn and key are alternatives: next to a key, fn never runs.
   - `value-loop` at
     `/flows/web/destinations/ga4/config/mapping/order/complete/data/map/items/loop`:
     loop turns nested products into GA4 items.
@@ -314,7 +314,7 @@ walkeros push packages/cli/examples/flow-complete.json -f web -e '{"name":"order
     value is a constant.
   - `rule-condition` at
     `/flows/web/destinations/ga4/config/mapping/order/complete/condition`: A
-    rule condition skips the rule, here for orders without a total.
+    rule condition skips the rule, here for orders without a value.
   - `rule-settings` at
     `/flows/server/destinations/piwikpro/config/mapping/order/complete/settings`:
     Rule settings are per-event package options: the Piwik PRO goal for orders.
@@ -332,7 +332,7 @@ walkeros push packages/cli/examples/flow-complete.json -f web -e '{"name":"order
 - **CLI:** the GA4 purchase, with consent and one product:
 
 ```bash
-walkeros push packages/cli/examples/flow-complete.json -f web -e '{"name":"order complete","data":{"id":"ORD-100","total":149.8,"currency":"EUR"},"nested":[{"entity":"product","data":{"id":"SKU-1","name":"Linen shirt","price":74.9,"quantity":2}}],"consent":{"functional":true}}' --simulate destination.ga4
+walkeros push packages/cli/examples/flow-complete.json -f web -e '{"name":"order complete","data":{"id":"ORD-100","value":149.8,"currency":"EUR"},"nested":[{"entity":"product","data":{"id":"SKU-1","name":"Linen shirt","price":74.9,"quantity":2}}],"consent":{"functional":true}}' --simulate destination.ga4
 ```
 
 - Output:
@@ -636,7 +636,7 @@ walkeros push packages/cli/examples/flow-complete.json -f server -e '{"name":"pr
 
 ```bash
 walkeros validate packages/cli/examples/flow-complete.json -t contract
-walkeros push packages/cli/examples/flow-complete.json -f server -e '{"name":"order complete","data":{"id":"ORD-100","total":149.8,"currency":"EUR"},"globals":{"language":"en"},"consent":{"functional":true}}' --simulate transformer.validate
+walkeros push packages/cli/examples/flow-complete.json -f server -e '{"name":"order complete","data":{"id":"ORD-100","value":149.8,"currency":"EUR"},"globals":{"language":"en"},"consent":{"functional":true}}' --simulate transformer.validate
 ```
 
 - **MCP:** `flow_validate`, `flow_simulate`.
@@ -760,7 +760,7 @@ walkeros push packages/cli/examples/flow-complete.json -f server -e '{"name":"pa
   (`user.ltv: 420`, `user.segment: "loyal"`):
 
 ```bash
-walkeros push packages/cli/examples/flow-complete.json -f server -e '{"name":"order complete","data":{"id":"ORD-100","total":149.8,"currency":"EUR"},"user":{"id":"cust-42"}}' --simulate transformer.loadUser
+walkeros push packages/cli/examples/flow-complete.json -f server -e '{"name":"order complete","data":{"id":"ORD-100","value":149.8,"currency":"EUR"},"user":{"id":"cust-42"}}' --simulate transformer.loadUser
 ```
 
 - Build the web bundle straight into the shared folder (web builds ignore root
diff --git a/packages/cli/examples/flow-order-complete.json b/packages/cli/examples/flow-order-complete.json
index 2abe02df7..285a92e67 100644
--- a/packages/cli/examples/flow-order-complete.json
+++ b/packages/cli/examples/flow-order-complete.json
@@ -20,7 +20,7 @@
                   "data": {
                     "map": {
                       "transaction_id": "data.id",
-                      "value": "data.total",
+                      "value": "data.value",
                       "currency": "data.currency",
                       "tax": "data.tax",
                       "shipping": "data.shipping",
diff --git a/packages/cli/examples/flow.json b/packages/cli/examples/flow.json
index 50ee8cd29..bfe62bfcf 100644
--- a/packages/cli/examples/flow.json
+++ b/packages/cli/examples/flow.json
@@ -21,7 +21,7 @@
                   "data": {
                     "map": {
                       "transaction_id": "data.id",
-                      "value": "data.total",
+                      "value": "data.value",
                       "currency": "data.currency",
                       "tax": "data.tax",
                       "shipping": "data.shipping",
diff --git a/packages/cli/src/__tests__/fixtures/validate-adversarial/f18-unknown-contract-ref.json b/packages/cli/src/__tests__/fixtures/validate-adversarial/f18-unknown-contract-ref.json
index c8e59be1e..8ca988944 100644
--- a/packages/cli/src/__tests__/fixtures/validate-adversarial/f18-unknown-contract-ref.json
+++ b/packages/cli/src/__tests__/fixtures/validate-adversarial/f18-unknown-contract-ref.json
@@ -9,9 +9,9 @@
             "properties": {
               "data": {
                 "type": "object",
-                "required": ["total"],
+                "required": ["value"],
                 "properties": {
-                  "total": {
+                  "value": {
                     "type": "number"
                   }
                 }
@@ -55,7 +55,7 @@
                 "entity": "order",
                 "action": "complete",
                 "data": {
-                  "total": 9.99
+                  "value": 9.99
                 },
                 "user": {},
                 "source": {
@@ -71,7 +71,7 @@
                       "entity": "order",
                       "action": "complete",
                       "data": {
-                        "total": 9.99
+                        "value": 9.99
                       },
                       "user": {},
                       "source": {
diff --git a/packages/cli/src/__tests__/fixtures/validate-adversarial/f26-var-contract.json b/packages/cli/src/__tests__/fixtures/validate-adversarial/f26-var-contract.json
index b75fabb04..13759ed59 100644
--- a/packages/cli/src/__tests__/fixtures/validate-adversarial/f26-var-contract.json
+++ b/packages/cli/src/__tests__/fixtures/validate-adversarial/f26-var-contract.json
@@ -9,9 +9,9 @@
             "properties": {
               "data": {
                 "type": "object",
-                "required": ["total"],
+                "required": ["value"],
                 "properties": {
-                  "total": {
+                  "value": {
                     "type": "number"
                   }
                 }
@@ -55,7 +55,7 @@
                 "entity": "order",
                 "action": "complete",
                 "data": {
-                  "total": 9.99
+                  "value": 9.99
                 },
                 "user": {},
                 "source": {
@@ -71,7 +71,7 @@
                       "entity": "order",
                       "action": "complete",
                       "data": {
-                        "total": 9.99
+                        "value": 9.99
                       },
                       "user": {},
                       "source": {
diff --git a/packages/cli/src/__tests__/fixtures/validate-adversarial/f4-dangling-extend.json b/packages/cli/src/__tests__/fixtures/validate-adversarial/f4-dangling-extend.json
index e656c60a6..d04275f83 100644
--- a/packages/cli/src/__tests__/fixtures/validate-adversarial/f4-dangling-extend.json
+++ b/packages/cli/src/__tests__/fixtures/validate-adversarial/f4-dangling-extend.json
@@ -9,9 +9,9 @@
             "properties": {
               "data": {
                 "type": "object",
-                "required": ["total"],
+                "required": ["value"],
                 "properties": {
-                  "total": {
+                  "value": {
                     "type": "number"
                   }
                 }
diff --git a/packages/cli/src/__tests__/fixtures/validate-adversarial/f5-contract-valid.json b/packages/cli/src/__tests__/fixtures/validate-adversarial/f5-contract-valid.json
index 58e22b35b..560624568 100644
--- a/packages/cli/src/__tests__/fixtures/validate-adversarial/f5-contract-valid.json
+++ b/packages/cli/src/__tests__/fixtures/validate-adversarial/f5-contract-valid.json
@@ -9,9 +9,9 @@
             "properties": {
               "data": {
                 "type": "object",
-                "required": ["total"],
+                "required": ["value"],
                 "properties": {
-                  "total": {
+                  "value": {
                     "type": "number"
                   }
                 }
diff --git a/packages/cli/src/__tests__/fixtures/validate-adversarial/f7-validate-step.json b/packages/cli/src/__tests__/fixtures/validate-adversarial/f7-validate-step.json
index a2ba23bd2..fd60fb7e1 100644
--- a/packages/cli/src/__tests__/fixtures/validate-adversarial/f7-validate-step.json
+++ b/packages/cli/src/__tests__/fixtures/validate-adversarial/f7-validate-step.json
@@ -9,9 +9,9 @@
             "properties": {
               "data": {
                 "type": "object",
-                "required": ["total"],
+                "required": ["value"],
                 "properties": {
-                  "total": {
+                  "value": {
                     "type": "number"
                   }
                 }
@@ -55,7 +55,7 @@
                 "entity": "order",
                 "action": "complete",
                 "data": {
-                  "total": 9.99
+                  "value": 9.99
                 },
                 "user": {},
                 "source": {
@@ -71,7 +71,7 @@
                       "entity": "order",
                       "action": "complete",
                       "data": {
-                        "total": 9.99
+                        "value": 9.99
                       },
                       "user": {},
                       "source": {
diff --git a/packages/cli/src/__tests__/integration/examples/aws-packed-flow.test.ts b/packages/cli/src/__tests__/integration/examples/aws-packed-flow.test.ts
index 911f0a4ec..3df0c7836 100644
--- a/packages/cli/src/__tests__/integration/examples/aws-packed-flow.test.ts
+++ b/packages/cli/src/__tests__/integration/examples/aws-packed-flow.test.ts
@@ -97,7 +97,7 @@ function startFakeAws(requests: FakeRequest[]): Promise {
 const RUNNER = `
 const { default: factory } = await import(process.argv[2]);
 const { collector } = await factory({});
-await collector.push({ name: 'order complete', data: { id: 'o-1', total: 42 } });
+await collector.push({ name: 'order complete', data: { id: 'o-1', value: 42 } });
 await collector.command('shutdown');
 process.stdout.write(JSON.stringify(collector.status.destinations));
 `;
@@ -254,7 +254,7 @@ describe('packed AWS destinations in a bundled JSON flow', () => {
     const record: unknown = JSON.parse(data);
     expect(record).toMatchObject({
       name: 'order complete',
-      data: { id: 'o-1', total: 42 },
+      data: { id: 'o-1', value: 42 },
     });
 
     // SNS: the same event as the message, to the configured topic.
diff --git a/packages/cli/src/__tests__/integration/examples/flow-complete-simulate.test.ts b/packages/cli/src/__tests__/integration/examples/flow-complete-simulate.test.ts
index b769ae06d..5e18d15b5 100644
--- a/packages/cli/src/__tests__/integration/examples/flow-complete-simulate.test.ts
+++ b/packages/cli/src/__tests__/integration/examples/flow-complete-simulate.test.ts
@@ -9,8 +9,6 @@
  * - Every simulation runs destinations on their packages' mock envs
  *   (Pub/Sub, BigQuery, Data Manager). The customers store is an fs store
  *   over the committed demo data (CUSTOMERS_DIR).
- * - Examples a simulation cannot reproduce yet are listed in WAITING with the
- *   reason; each shows up as a todo, never as a silent skip.
  */
 
 import fs from 'fs-extra';
@@ -107,13 +105,6 @@ const INGEST: Record> = {
   'server.destinations.datamanager.conversionWithGclid': CHROME,
 };
 
-const OWNER = 'docs/plans/2026-09-24-step-examples-real.md';
-
-/** Examples no simulation can reproduce yet, with the reason and owner. */
-const WAITING: Record = {
-  'server.transformers.file.walkerJs': `simulate transformer does not capture respond; the HTTP test below serves /walker.js (${OWNER})`,
-};
-
 /**
  * The collector's starting consent per source example: a consent-gated
  * source (the session source) waits for it. A step example has no consent
@@ -400,15 +391,8 @@ describe('flow-complete.json', () => {
   };
 
   const cases = collectCases(validateFlowConfig(fs.readJSONSync(configPath)));
-  const runnable = cases.filter((c) => !(c.id in WAITING));
-  const waiting = cases.filter((c) => c.id in WAITING);
-
-  it('lists only existing examples as waiting', () => {
-    const ids = new Set(cases.map((c) => c.id));
-    expect(Object.keys(WAITING).filter((id) => !ids.has(id))).toEqual([]);
-  });
 
-  it.each(runnable.map((c) => [c.id, c] as const))(
+  it.each(cases.map((c) => [c.id, c] as const))(
     'step example %s simulates to its out',
     async (id, c) => {
       const bundlePath = await bundleFor(c);
@@ -513,14 +497,6 @@ describe('flow-complete.json', () => {
     }
   }, 60000);
 
-  it.each(waiting.map((c) => [c.id, WAITING[c.id]] as const))(
-    'step example %s waits: %s',
-    (id) => {
-      const c = cases.find((item) => item.id === id);
-      expect(c?.example.out).toBeUndefined();
-    },
-  );
-
   describe('server flow over HTTP', () => {
     const headers = {
       'content-type': 'application/json',
@@ -547,7 +523,7 @@ describe('flow-complete.json', () => {
       name: 'order complete',
       entity: 'order',
       action: 'complete',
-      data: { id: orderId, total: 129.9, currency: 'EUR' },
+      data: { id: orderId, value: 129.9, currency: 'EUR' },
       nested: [
         {
           entity: 'product',
@@ -797,7 +773,7 @@ describe('flow-complete.json', () => {
             name: 'order complete',
             entity: 'order',
             action: 'complete',
-            data: { id: 'ORD-9', total: 129.9, currency: 'EUR' },
+            data: { id: 'ORD-9', value: 129.9, currency: 'EUR' },
             nested: [
               {
                 entity: 'product',
diff --git a/packages/cli/src/__tests__/unit/validate/flow.test.ts b/packages/cli/src/__tests__/unit/validate/flow.test.ts
index de1a6c4e4..83320efd0 100644
--- a/packages/cli/src/__tests__/unit/validate/flow.test.ts
+++ b/packages/cli/src/__tests__/unit/validate/flow.test.ts
@@ -861,8 +861,8 @@ describe('validateFlow', () => {
               properties: {
                 data: {
                   type: 'object',
-                  required: ['total'],
-                  properties: { total: { type: 'number' } },
+                  required: ['value'],
+                  properties: { value: { type: 'number' } },
                 },
               },
             },
@@ -885,7 +885,7 @@ describe('validateFlow', () => {
       name: 'order complete',
       entity: 'order',
       action: 'complete',
-      data: { total: 9.99 },
+      data: { value: 9.99 },
       user: { hash: 'h1' },
       source: { type: 'express' },
     };
@@ -893,7 +893,7 @@ describe('validateFlow', () => {
       name: 'order complete',
       entity: 'order',
       action: 'complete',
-      data: { total: 9.99 },
+      data: { value: 9.99 },
       user: {},
       source: { type: 'express' },
     };
diff --git a/packages/cli/src/commands/push/__tests__/simulate-isolation.test.ts b/packages/cli/src/commands/push/__tests__/simulate-isolation.test.ts
index 1d10ff2c1..71b5d47c4 100644
--- a/packages/cli/src/commands/push/__tests__/simulate-isolation.test.ts
+++ b/packages/cli/src/commands/push/__tests__/simulate-isolation.test.ts
@@ -268,7 +268,7 @@ describe('transformer simulation isolation — before chain', () => {
 
     const inputEvent = {
       name: 'order complete',
-      data: { total: 100 },
+      data: { value: 100 },
     } as WalkerOS.DeepPartialEvent;
 
     // Run before chain: validate, then its own next (enrich)
@@ -286,7 +286,7 @@ describe('transformer simulation isolation — before chain', () => {
 
     // Both validate and enrich ran
     expect(processedEvent.data).toEqual({
-      total: 100,
+      value: 100,
       valid: true,
       enriched: true,
     });
@@ -304,7 +304,7 @@ describe('transformer simulation isolation — before chain', () => {
     expect(mainPush).toHaveBeenCalledTimes(1);
     const receivedEvent = mainPush.mock.calls[0][0];
     expect(receivedEvent.data).toEqual({
-      total: 100,
+      value: 100,
       valid: true,
       enriched: true,
     });
diff --git a/packages/cli/src/examples/flow-complete.manifest.ts b/packages/cli/src/examples/flow-complete.manifest.ts
index e05f42d47..cb8e781cc 100644
--- a/packages/cli/src/examples/flow-complete.manifest.ts
+++ b/packages/cli/src/examples/flow-complete.manifest.ts
@@ -509,7 +509,7 @@ const teaching = {
     note: 'examples are the contract of a step: this in gives this out, checked by simulation.',
     docs: [{ page: 'getting-started/flow/step-examples' }],
     cli: [
-      `walkeros push ${FILE} -f web -e '{"name":"order complete","data":{"id":"ORD-100","total":149.8,"currency":"EUR"}}' --simulate destination.ga4`,
+      `walkeros push ${FILE} -f web -e '{"name":"order complete","data":{"id":"ORD-100","value":149.8,"currency":"EUR"}}' --simulate destination.ga4`,
     ],
   },
   'example-title': {
@@ -637,7 +637,7 @@ const teaching = {
     chapter: 'mapping',
     level: 'intermediate',
     useCases: [],
-    note: 'fn computes a value from the event, here the total rounded to cents. fn and key are alternatives: next to a key, fn never runs.',
+    note: 'fn computes a value from the event, here the order value rounded to cents. fn and key are alternatives: next to a key, fn never runs.',
     docs: [],
   },
   'value-loop': {
@@ -693,7 +693,7 @@ const teaching = {
     chapter: 'mapping',
     level: 'intermediate',
     useCases: [],
-    note: 'A rule condition skips the rule, here for orders without a total.',
+    note: 'A rule condition skips the rule, here for orders without a value.',
     docs: [],
   },
   'rule-settings': {
@@ -1530,7 +1530,6 @@ const teaching = {
     useCases: ['script-serving'],
     note: 'walker.js is served first party from the server flow.',
     docs: [{ page: 'transformers/file' }],
-    example: { step: 'file', name: 'walkerJs' },
     cli: [
       `walkeros bundle ${FILE} -f web -o packages/cli/examples/shared/walker.js`,
     ],
@@ -2212,7 +2211,7 @@ export const flowCompleteCli: CliEntry[] = [
     chapter: 'chains-routing',
   },
   {
-    command: `walkeros push ${FILE} -f web -e '{"name":"order complete","data":{"id":"ORD-100","total":149.8,"currency":"EUR"}}' --simulate destination.ga4`,
+    command: `walkeros push ${FILE} -f web -e '{"name":"order complete","data":{"id":"ORD-100","value":149.8,"currency":"EUR"}}' --simulate destination.ga4`,
     chapter: 'mapping',
   },
   {
diff --git a/packages/collector/src/__tests__/boundary-error.test.ts b/packages/collector/src/__tests__/boundary-error.test.ts
index 6e9e9cba9..25d396933 100644
--- a/packages/collector/src/__tests__/boundary-error.test.ts
+++ b/packages/collector/src/__tests__/boundary-error.test.ts
@@ -123,7 +123,7 @@ describe('push boundary', () => {
     // without carrying the buyer's address into stderr and the jsonl sink.
     const event: WalkerOS.DeepPartialEvent = {
       name: 'order complete',
-      data: { total: 42 },
+      data: { value: 42 },
       user: { id: 'buyer@example.com' },
     };
 
diff --git a/packages/collector/src/__tests__/mapping-failures.test.ts b/packages/collector/src/__tests__/mapping-failures.test.ts
index fe514265c..770369abf 100644
--- a/packages/collector/src/__tests__/mapping-failures.test.ts
+++ b/packages/collector/src/__tests__/mapping-failures.test.ts
@@ -90,7 +90,7 @@ describe('mapping failures (ruling 2)', () => {
   it('(b, c, d) a throwing fn and an operator-less value drop the value only: delivered, reported, no payload', async () => {
     const { lines, logger } = errors();
     const received: unknown[] = [];
-    const order: Record = { total: 'data.total' };
+    const order: Record = { total: 'data.value' };
     const destination: Destination.Instance = {
       type: 'probe',
       config: {},
@@ -125,7 +125,7 @@ describe('mapping failures (ruling 2)', () => {
     });
 
     const r = await elb('order complete', {
-      total: 42,
+      value: 42,
       currency: 'EUR',
       email: 'person@example.com',
     });
diff --git a/packages/collector/src/destination-outcome-state.ts b/packages/collector/src/destination-outcome-state.ts
index 7d7b02151..b9783a52f 100644
--- a/packages/collector/src/destination-outcome-state.ts
+++ b/packages/collector/src/destination-outcome-state.ts
@@ -34,6 +34,16 @@ export const HELD_WAIT: QueuedReason = {
   message: 'a state delivery to the destination is running or failed',
 };
 
+/**
+ * Waiting behind the destination's init, which another push started: the
+ * event goes out right after it, in push order.
+ */
+export const INIT_WAIT: QueuedReason = {
+  code: 'held',
+  message:
+    "the destination's init is running; the event goes out right after it",
+};
+
 const READY: DestinationState = { state: 'ready' };
 
 /**
diff --git a/packages/core/src/__tests__/eventGenerator.test.ts b/packages/core/src/__tests__/eventGenerator.test.ts
index 9b686f52c..09564b407 100644
--- a/packages/core/src/__tests__/eventGenerator.test.ts
+++ b/packages/core/src/__tests__/eventGenerator.test.ts
@@ -1,7 +1,7 @@
 import { createEvent, getEvent } from '..';
 
 describe('createEvent', () => {
-  const timestamp = new Date().setHours(0, 13, 37, 0);
+  const timestamp = 1700000000000;
 
   const defaultEvent = {
     name: 'entity action',
@@ -88,3 +88,39 @@ describe('createEvent', () => {
     );
   });
 });
+
+describe('getEvent', () => {
+  it('returns the same event on every call and every day', () => {
+    const a = getEvent('order complete');
+    jest.useFakeTimers().setSystemTime(new Date('2031-01-01T12:00:00Z'));
+    const b = getEvent('order complete');
+    jest.useRealTimers();
+    expect(b).toEqual(a);
+    expect(a.timestamp).toBe(1700000000000);
+  });
+
+  it('derives the id from the name', () => {
+    expect(getEvent('page view').id).toBe('f0cc5f6fec3d8eeb');
+    expect(getEvent('page view').id).not.toBe(getEvent('product view').id);
+  });
+
+  it('keeps explicit props', () => {
+    expect(getEvent('page view', { timestamp: 5 }).timestamp).toBe(5);
+    expect(getEvent('page view', { id: 'custom' }).id).toBe('custom');
+  });
+
+  it('carries the order value as data.value', () => {
+    expect(getEvent('order complete').data).toStrictEqual({
+      id: '0rd3r1d',
+      currency: 'EUR',
+      shipping: 5.22,
+      taxes: 73.76,
+      value: 555,
+    });
+  });
+
+  it('accepts only standard event names', () => {
+    // @ts-expect-error unknown standard event names do not compile
+    expect(getEvent('not a standard event').name).toBe('not a standard event');
+  });
+});
diff --git a/packages/core/src/__tests__/include.test.ts b/packages/core/src/__tests__/include.test.ts
index e055f38f7..c1bc4e684 100644
--- a/packages/core/src/__tests__/include.test.ts
+++ b/packages/core/src/__tests__/include.test.ts
@@ -12,7 +12,7 @@ describe('flattenIncludeSections', () => {
       data_currency: 'EUR',
       data_shipping: 5.22,
       data_taxes: 73.76,
-      data_total: 555,
+      data_value: 555,
     });
   });
 
@@ -60,7 +60,7 @@ describe('flattenIncludeSections', () => {
   test('"all" expands to every section', () => {
     const result = flattenIncludeSections(event, ['all']);
     // Should contain keys from all sections
-    expect(result.data_total).toBe(555);
+    expect(result.data_value).toBe(555);
     expect(result.globals_pagegroup).toBe('shop');
     expect(result.context_shopping).toBe('complete');
     expect(result.user_id).toBe('us3r');
@@ -70,7 +70,7 @@ describe('flattenIncludeSections', () => {
 
   test('multiple sections combined', () => {
     const result = flattenIncludeSections(event, ['data', 'globals']);
-    expect(result.data_total).toBe(555);
+    expect(result.data_value).toBe(555);
     expect(result.globals_pagegroup).toBe('shop');
     // No context keys
     expect(result.context_shopping).toBeUndefined();
@@ -162,7 +162,7 @@ describe('processEventMapping with include', () => {
 
     expect(isObject(result.data)).toBe(true);
     expect(result.data).toMatchObject({
-      data_total: 555,
+      data_value: 555,
       data_currency: 'EUR',
     });
   });
@@ -184,7 +184,7 @@ describe('processEventMapping with include', () => {
 
     // Rule-level include wins - only globals, no data
     expect(result.data).toMatchObject({ globals_pagegroup: 'shop' });
-    expect(isObject(result.data) && 'data_total' in result.data).toBe(false);
+    expect(isObject(result.data) && 'data_value' in result.data).toBe(false);
   });
 
   test('config.data wins over include on key conflict', async () => {
@@ -193,7 +193,7 @@ describe('processEventMapping with include', () => {
       include: ['data'],
       data: {
         map: {
-          data_total: { value: 'overridden' },
+          data_value: { value: 'overridden' },
         },
       },
     };
@@ -201,7 +201,7 @@ describe('processEventMapping with include', () => {
     const result = await processEventMapping(event, config, mockCollector);
 
     expect(result.data).toMatchObject({
-      data_total: 'overridden',
+      data_value: 'overridden',
       // Other include keys still present
       data_currency: 'EUR',
     });
diff --git a/packages/core/src/__tests__/mapping-error-visibility.test.ts b/packages/core/src/__tests__/mapping-error-visibility.test.ts
index c573b42bb..94d26cde5 100644
--- a/packages/core/src/__tests__/mapping-error-visibility.test.ts
+++ b/packages/core/src/__tests__/mapping-error-visibility.test.ts
@@ -186,12 +186,12 @@ describe('a value object without an operator (E14)', () => {
     const collector = createTestCollector();
     const event: WalkerOS.DeepPartialEvent = {
       name: 'order complete',
-      data: { total: 42, currency: 'EUR' },
+      data: { value: 42, currency: 'EUR' },
       user: { email: 'person@example.com' },
     };
 
     // A plain object of paths, the shape a hand-written config arrives in.
-    const order: Record = { total: 'data.total' };
+    const order: Record = { total: 'data.value' };
     const mapping: Mapping.Value = {
       map: { order, currency: 'data.currency' },
     };
diff --git a/packages/core/src/__tests__/mapping.test.ts b/packages/core/src/__tests__/mapping.test.ts
index 74049aa7a..5e7a76127 100644
--- a/packages/core/src/__tests__/mapping.test.ts
+++ b/packages/core/src/__tests__/mapping.test.ts
@@ -1040,16 +1040,16 @@ describe('processEventMapping', () => {
   test('config-level policy applied before event-level policy', async () => {
     const event = {
       name: 'order complete',
-      data: { total: 100 },
+      data: { value: 100 },
     };
 
     const config = {
       policy: {
         'data.configLevel': { value: 'config' },
-        'data.total': {
+        'data.value': {
           fn: (e: unknown) =>
-            isObject(e) && isObject(e.data) && typeof e.data.total === 'number'
-              ? e.data.total * 2
+            isObject(e) && isObject(e.data) && typeof e.data.value === 'number'
+              ? e.data.value * 2
               : undefined,
         },
       },
@@ -1058,12 +1058,12 @@ describe('processEventMapping', () => {
           complete: {
             policy: {
               'data.eventLevel': { value: 'event' },
-              'data.total': {
+              'data.value': {
                 fn: (e: unknown) =>
                   isObject(e) &&
                   isObject(e.data) &&
-                  typeof e.data.total === 'number'
-                    ? e.data.total + 50
+                  typeof e.data.value === 'number'
+                    ? e.data.value + 50
                     : undefined,
               },
             },
@@ -1076,7 +1076,7 @@ describe('processEventMapping', () => {
 
     // Config policy runs first: 100 * 2 = 200
     // Event policy runs second: 200 + 50 = 250
-    expect(result.event.data.total).toBe(250);
+    expect(result.event.data.value).toBe(250);
     expect(result.event.data.configLevel).toBe('config');
     expect(result.event.data.eventLevel).toBe('event');
   });
@@ -1333,7 +1333,7 @@ describe('processEventMapping', () => {
   test('remove strips paths from produced data after evaluation', async () => {
     const event = {
       name: 'order complete',
-      data: { total: 42, currency: 'EUR' },
+      data: { value: 42, currency: 'EUR' },
     };
     const config = {
       mapping: {
@@ -1341,7 +1341,7 @@ describe('processEventMapping', () => {
           complete: {
             data: {
               map: {
-                value: 'data.total',
+                value: 'data.value',
                 currency: 'data.currency',
               },
             },
diff --git a/packages/core/src/__tests__/mergeMapping.test.ts b/packages/core/src/__tests__/mergeMapping.test.ts
index 0956bef2f..f0c2b35a0 100644
--- a/packages/core/src/__tests__/mergeMapping.test.ts
+++ b/packages/core/src/__tests__/mergeMapping.test.ts
@@ -8,7 +8,7 @@ describe('mergeMappingRule', () => {
       map: {
         id: 'params.ep.transaction_id',
         currency: 'params.ep.currency',
-        total: 'params.epn.value',
+        value: 'params.epn.value',
       },
     },
     consent: { marketing: true },
@@ -26,7 +26,7 @@ describe('mergeMappingRule', () => {
       map: {
         id: 'params.ep.transaction_id',
         currency: 'params.ep.currency',
-        total: 'params.epn.value',
+        value: 'params.epn.value',
         coupon: 'params.ep.coupon',
       },
     });
@@ -40,7 +40,7 @@ describe('mergeMappingRule', () => {
       map: {
         id: 'params.ep.order_id',
         currency: 'params.ep.currency',
-        total: 'params.epn.value',
+        value: 'params.epn.value',
       },
     });
   });
@@ -58,7 +58,7 @@ describe('mergeMappingRule', () => {
       extend: { data: { map: { currency: null } } },
     });
     expect(result.data).toEqual({
-      map: { id: 'params.ep.transaction_id', total: 'params.epn.value' },
+      map: { id: 'params.ep.transaction_id', value: 'params.epn.value' },
     });
   });
   it('carries remove onto the merged rule', () => {
@@ -75,7 +75,7 @@ describe('mergeMappingRule', () => {
       map: {
         id: 'params.ep.transaction_id',
         currency: 'params.ep.currency',
-        total: 'params.epn.value',
+        value: 'params.epn.value',
         coupon: 'params.ep.coupon',
       },
     });
@@ -87,7 +87,7 @@ describe('mergeMappingRule', () => {
       map: {
         id: 'params.ep.transaction_id',
         currency: 'params.ep.currency',
-        total: 'params.epn.value',
+        value: 'params.epn.value',
       },
     });
   });
diff --git a/packages/core/src/__tests__/schemas/validate-flow-config.test.ts b/packages/core/src/__tests__/schemas/validate-flow-config.test.ts
index 9343c6b88..4072e22a3 100644
--- a/packages/core/src/__tests__/schemas/validate-flow-config.test.ts
+++ b/packages/core/src/__tests__/schemas/validate-flow-config.test.ts
@@ -258,11 +258,11 @@ describe('validateFlowConfig', () => {
                   properties: {
                     data: {
                       type: 'object',
-                      required: ['total'],
+                      required: ['value'],
                       properties: {
-                        total: {
+                        value: {
                           type: 'number',
-                          description: 'Order total in EUR',
+                          description: 'Order value in EUR',
                         },
                       },
                     },
@@ -280,9 +280,9 @@ describe('validateFlowConfig', () => {
     const result = validateFlowConfig(json);
     const order = result.context?.contract?.find((c) => c.entity === 'order');
     expect(order?.actions).toEqual(['complete']);
-    expect(order?.properties?.complete?.total).toEqual({
+    expect(order?.properties?.complete?.value).toEqual({
       type: 'number',
-      description: 'Order total in EUR',
+      description: 'Order value in EUR',
       required: true,
     });
   });
@@ -1513,7 +1513,7 @@ describe('validateFlowConfig', () => {
       ],
       [
         'bare event paths in destination data',
-        destination({ data: { map: { v: 'data.total' } } }),
+        destination({ data: { map: { v: 'data.value' } } }),
       ],
       [
         'a reference string',
diff --git a/packages/core/src/__tests__/step-examples.test.ts b/packages/core/src/__tests__/step-examples.test.ts
index bcbf7ed1e..59ef2236a 100644
--- a/packages/core/src/__tests__/step-examples.test.ts
+++ b/packages/core/src/__tests__/step-examples.test.ts
@@ -33,7 +33,7 @@ describe('Step Examples', () => {
               purchase: {
                 in: {
                   name: 'order complete',
-                  data: { id: 'ORD-123', total: 149.97 },
+                  data: { id: 'ORD-123', value: 149.97 },
                 },
                 out: [
                   [
diff --git a/packages/core/src/eventGenerator.ts b/packages/core/src/eventGenerator.ts
index 409d61fea..73403dca6 100644
--- a/packages/core/src/eventGenerator.ts
+++ b/packages/core/src/eventGenerator.ts
@@ -1,7 +1,30 @@
 import type { WalkerOS } from './types';
 import { assign } from './assign';
+import { fnv1a64 } from './deriveSpanId';
 import { getSpanId } from './getSpanId';
 
+/**
+ * Fixed timestamp of every generated event (2023-11-14T22:13:20Z, in
+ * milliseconds), so generated events are the same on every run and day.
+ */
+const TIMESTAMP = 1700000000000;
+
+/**
+ * Names `getEvent` knows: the standard events plus the generic default
+ * `entity action`. Any other name does not compile.
+ */
+export type StandardEventName =
+  | 'entity action'
+  | 'cart view'
+  | 'checkout view'
+  | 'order complete'
+  | 'page view'
+  | 'product add'
+  | 'product view'
+  | 'product visible'
+  | 'promotion visible'
+  | 'session start';
+
 /**
  * Creates a complete event with default values.
  * Used for testing and debugging.
@@ -16,7 +39,7 @@ import { getSpanId } from './getSpanId';
 export function createEvent(
   props: WalkerOS.DeepPartialEvent = {},
 ): WalkerOS.Event {
-  const timestamp = props.timestamp || new Date().setHours(0, 13, 37, 0);
+  const timestamp = props.timestamp || TIMESTAMP;
   const id = props.id || getSpanId();
 
   const defaultEvent: WalkerOS.Event = {
@@ -83,18 +106,23 @@ export function createEvent(
 }
 
 /**
- * Creates a complete event with default values based on the event name.
+ * Creates a complete standard event with default values based on its name.
  * Used for testing and debugging.
  *
- * @param name - The name of the event to create.
+ * The output is stable: the timestamp is fixed and the id is derived from the
+ * name (16 lowercase hex characters), so the same name gives the same event on
+ * every call, run and day. Events with other names use `createEvent`.
+ *
+ * @param name - The standard event to create.
  * @param props - Properties to override the default values.
  * @returns A complete event.
  */
 export function getEvent(
-  name: string = 'entity action',
+  name: StandardEventName = 'entity action',
   props: WalkerOS.DeepPartialEvent = {},
 ): WalkerOS.Event {
-  const timestamp = props.timestamp || new Date().setHours(0, 13, 37, 0);
+  const timestamp = props.timestamp || TIMESTAMP;
+  const id = props.id || fnv1a64(name);
 
   const quantity = 2;
   const product1 = {
@@ -115,7 +143,8 @@ export function getEvent(
     },
   };
 
-  const defaultEvents: Record = {
+  const defaultEvents: Record = {
+    'entity action': {},
     'cart view': {
       data: {
         currency: 'EUR',
@@ -163,7 +192,7 @@ export function getEvent(
         currency: 'EUR',
         shipping: 5.22,
         taxes: 73.76,
-        total: 555,
+        value: 555,
       },
       context: { shopping: ['complete', 0] },
       globals: { pagegroup: 'shop' },
@@ -273,5 +302,5 @@ export function getEvent(
     },
   };
 
-  return createEvent({ ...defaultEvents[name], ...props, name: name });
+  return createEvent({ ...defaultEvents[name], ...props, name, id });
 }
diff --git a/packages/core/src/mappingValue.ts b/packages/core/src/mappingValue.ts
index e1f71bd53..35d873026 100644
--- a/packages/core/src/mappingValue.ts
+++ b/packages/core/src/mappingValue.ts
@@ -130,8 +130,8 @@ async function processMappingValue(
 
       if (!Object.keys(mapping).length) return;
 
-      // An object without any operator (`{ total: 'data.total' }` where
-      // `{ map: { total: 'data.total' } }` was meant) would resolve to the
+      // An object without any operator (`{ total: 'data.value' }` where
+      // `{ map: { total: 'data.value' } }` was meant) would resolve to the
       // whole input. Drop it and say how to fix it.
       if (!VALUE_KEYS.some((name) => name in mapping)) {
         reportValue(
diff --git a/packages/mcps/mcp/src/__tests__/tools/examples.test.ts b/packages/mcps/mcp/src/__tests__/tools/examples.test.ts
index 3bd8452c9..fd93bfea6 100644
--- a/packages/mcps/mcp/src/__tests__/tools/examples.test.ts
+++ b/packages/mcps/mcp/src/__tests__/tools/examples.test.ts
@@ -98,10 +98,10 @@ const sampleConfig = {
           package: '@walkeros/web-destination-gtag',
           examples: {
             purchase: {
-              in: { name: 'order complete', data: { total: 42 } },
+              in: { name: 'order complete', data: { value: 42 } },
               mapping: {
                 name: 'purchase',
-                data: { map: { value: 'data.total' } },
+                data: { map: { value: 'data.value' } },
               },
               out: [
                 { type: 'call', path: 'gtag', args: ['event', 'purchase'] },
@@ -203,7 +203,7 @@ describe('flow_examples tool', () => {
     const purchase = exampleNamed(result, 'purchase');
     expect(purchase.mapping).toEqual({
       name: 'purchase',
-      data: { map: { value: 'data.total' } },
+      data: { map: { value: 'data.value' } },
     });
   });
 
diff --git a/packages/server/destinations/amplitude/src/examples/step.ts b/packages/server/destinations/amplitude/src/examples/step.ts
index 1452e679d..6610d3d6f 100644
--- a/packages/server/destinations/amplitude/src/examples/step.ts
+++ b/packages/server/destinations/amplitude/src/examples/step.ts
@@ -1,5 +1,5 @@
 import type { Flow, WalkerOS } from '@walkeros/core';
-import { getEvent } from '@walkeros/core';
+import { createEvent, getEvent } from '@walkeros/core';
 import type { Settings } from '../types';
 
 /**
@@ -33,7 +33,7 @@ export const defaultEventForwarding: AmplitudeStepExample = {
  */
 export const wildcardIgnored: AmplitudeStepExample = {
   public: false,
-  in: getEvent('debug noise', { timestamp: 1700000101 }),
+  in: createEvent({ name: 'debug noise', timestamp: 1700000101 }),
   mapping: { ignore: true },
   out: [],
 };
@@ -112,7 +112,8 @@ export const userLoginIdentify: AmplitudeStepExample = {
   title: 'User login identify',
   description:
     'A user login maps to amplitude.identify with set, setOnce, and add operations while the default track call is skipped.',
-  in: getEvent('user login', {
+  in: createEvent({
+    name: 'user login',
     timestamp: 1700000105,
     data: {
       user_id: 'new-user-123',
@@ -183,7 +184,8 @@ export const subscriptionRenewRevenue: AmplitudeStepExample = {
   title: 'Subscription revenue',
   description:
     'A subscription renewal fires a single amplitude.revenue call with productId, price, and a currency fallback.',
-  in: getEvent('subscription renew', {
+  in: createEvent({
+    name: 'subscription renew',
     timestamp: 1700000107,
     data: {
       plan_id: 'plan-pro',
@@ -285,7 +287,7 @@ export const orderCompleteMultiProduct: AmplitudeStepExample = {
         data_currency: 'EUR',
         data_shipping: 5.22,
         data_taxes: 73.76,
-        data_total: 555,
+        data_value: 555,
         globals_pagegroup: 'shop',
       },
     ],
@@ -300,7 +302,8 @@ export const groupAssignmentWithProperties: AmplitudeStepExample = {
   title: 'Group assignment',
   description:
     'A company update assigns the user to a group and sets group properties via setGroup plus groupIdentify.',
-  in: getEvent('company update', {
+  in: createEvent({
+    name: 'company update',
     timestamp: 1700000109,
     data: {
       company: 'Acme',
diff --git a/packages/server/destinations/api/src/examples/step.ts b/packages/server/destinations/api/src/examples/step.ts
index 3131e43fa..11427d081 100644
--- a/packages/server/destinations/api/src/examples/step.ts
+++ b/packages/server/destinations/api/src/examples/step.ts
@@ -1,5 +1,5 @@
 import type { Flow } from '@walkeros/core';
-import { getEvent } from '@walkeros/core';
+import { createEvent, getEvent } from '@walkeros/core';
 
 /**
  * API server destination step examples.
@@ -42,7 +42,8 @@ export const customHeaders: Flow.StepExample = {
   title: 'Custom headers',
   description:
     'A form submission is sent to the API with custom request headers such as an API key for authentication.',
-  in: getEvent('form submit', {
+  in: createEvent({
+    name: 'form submit',
     timestamp: 1700000801,
     data: { type: 'contact', email: 'user@example.com' },
     source: { type: 'express', platform: 'server' },
@@ -73,7 +74,7 @@ export const customTransform: Flow.StepExample = {
     'An order event is reshaped via a data mapping into a custom JSON payload with renamed fields for the API.',
   in: getEvent('order complete', {
     timestamp: 1700000802,
-    data: { id: 'ORD-500', total: 199.99, currency: 'USD' },
+    data: { id: 'ORD-500', value: 199.99, currency: 'USD' },
     user: { id: 'buyer-42' },
     source: { type: 'express', platform: 'server' },
   }),
@@ -81,7 +82,7 @@ export const customTransform: Flow.StepExample = {
     data: {
       map: {
         order_id: 'data.id',
-        amount: 'data.total',
+        amount: 'data.value',
         currency: 'data.currency',
         customer_id: 'user.id',
         event_type: 'name',
diff --git a/packages/server/destinations/aws/src/firehose/examples/step.ts b/packages/server/destinations/aws/src/firehose/examples/step.ts
index f54d7061d..ba6ea64fc 100644
--- a/packages/server/destinations/aws/src/firehose/examples/step.ts
+++ b/packages/server/destinations/aws/src/firehose/examples/step.ts
@@ -1,5 +1,5 @@
 import type { Flow } from '@walkeros/core';
-import { getEvent } from '@walkeros/core';
+import { createEvent, getEvent } from '@walkeros/core';
 import { toRecordData } from '../lib/records';
 
 /**
@@ -53,11 +53,12 @@ const pageEvent = getEvent('page view', {
 const orderEvt = getEvent('order complete', {
   id: 'ev-1700001001',
   timestamp: 1700001001,
-  data: { id: 'ORD-400', total: 99.99, currency: 'EUR' },
+  data: { id: 'ORD-400', value: 99.99, currency: 'EUR' },
   source: { type: 'express', platform: 'server' },
 });
 
-const signupEvt = getEvent('user signup', {
+const signupEvt = createEvent({
+  name: 'user signup',
   id: 'ev-1700001002',
   timestamp: 1700001002,
   data: { plan: 'pro', source: 'landing-page' },
@@ -115,7 +116,7 @@ export const mappedRecord: Flow.StepExample = {
         id: 'id',
         name: 'name',
         order_id: 'data.id',
-        total: 'data.total',
+        total: 'data.value',
       },
     },
   },
diff --git a/packages/server/destinations/aws/src/sns/examples/step.ts b/packages/server/destinations/aws/src/sns/examples/step.ts
index 9e0040880..8906f6ab1 100644
--- a/packages/server/destinations/aws/src/sns/examples/step.ts
+++ b/packages/server/destinations/aws/src/sns/examples/step.ts
@@ -57,7 +57,7 @@ const pageEvent = getEvent('page view', {
 const orderEvt = getEvent('order complete', {
   id: 'ev-1700001001',
   timestamp: 1700001001,
-  data: { id: 'ORD-400', total: 99.99, currency: 'EUR' },
+  data: { id: 'ORD-400', value: 99.99, currency: 'EUR' },
   user: { id: 'usr-789' },
   source: { type: 'express', platform: 'server' },
 });
@@ -140,7 +140,7 @@ export const mappedMessage: Flow.StepExample = {
   in: orderEvt,
   mapping: {
     data: {
-      map: { id: 'id', name: 'name', order_id: 'data.id', total: 'data.total' },
+      map: { id: 'id', name: 'name', order_id: 'data.id', total: 'data.value' },
     },
   },
   out: [
diff --git a/packages/server/destinations/bing/src/__tests__/index.test.ts b/packages/server/destinations/bing/src/__tests__/index.test.ts
index bff676439..61679d3e6 100644
--- a/packages/server/destinations/bing/src/__tests__/index.test.ts
+++ b/packages/server/destinations/bing/src/__tests__/index.test.ts
@@ -2,10 +2,11 @@ import type { WalkerOS, Collector } from '@walkeros/core';
 import type { Config, Destination, Settings } from '../types';
 import {
   clone,
+  createEvent,
   createIngest,
-  getEvent,
   createMockContext,
   createMockLogger,
+  getEvent,
 } from '@walkeros/core';
 import { startFlow } from '@walkeros/collector';
 import { examples } from '../dev';
@@ -203,7 +204,7 @@ describe('Server Destination Bing', () => {
     const dest = jest.requireActual('../').default;
     const event = getEvent('order complete', {
       timestamp: 1700000900000,
-      data: { id: 'ORD-1', total: 100, currency: 'EUR' },
+      data: { id: 'ORD-1', value: 100, currency: 'EUR' },
       source: {
         type: 'browser',
         platform: 'web',
@@ -223,7 +224,7 @@ describe('Server Destination Bing', () => {
                 map: {
                   customData: {
                     map: {
-                      value: 'data.total',
+                      value: 'data.value',
                       currency: 'data.currency',
                       transactionId: 'data.id',
                     },
@@ -290,7 +291,8 @@ describe('Server Destination Bing', () => {
   });
 
   test('user data is hashed (em)', async () => {
-    const event = getEvent('form submit', {
+    const event = createEvent({
+      name: 'form submit',
       user: { email: 'test@example.com' },
     });
     const config: Config = {
@@ -318,7 +320,8 @@ describe('Server Destination Bing', () => {
   });
 
   test('user_data mapping resolves from settings', async () => {
-    const event = getEvent('form submit', {
+    const event = createEvent({
+      name: 'form submit',
       user: { email: 'u@example.com', id: 'user-123' },
     });
     const config: Config = {
diff --git a/packages/server/destinations/bing/src/examples/step.ts b/packages/server/destinations/bing/src/examples/step.ts
index dc0634f62..c44d2ba0b 100644
--- a/packages/server/destinations/bing/src/examples/step.ts
+++ b/packages/server/destinations/bing/src/examples/step.ts
@@ -1,5 +1,5 @@
 import type { Flow } from '@walkeros/core';
-import { getEvent, isObject } from '@walkeros/core';
+import { createEvent, getEvent, isObject } from '@walkeros/core';
 
 /**
  * Bing UET CAPI step examples.
@@ -40,7 +40,7 @@ export const purchase: Flow.StepExample = {
   in: getEvent('order complete', {
     id: 'b1c2d3e4f5a60001',
     timestamp: 1700000900000,
-    data: { id: 'ORD-300', total: 249.99, currency: 'EUR' },
+    data: { id: 'ORD-300', value: 249.99, currency: 'EUR' },
     nested: [
       {
         entity: 'product',
@@ -60,7 +60,7 @@ export const purchase: Flow.StepExample = {
       map: {
         customData: {
           map: {
-            value: 'data.total',
+            value: 'data.value',
             currency: { key: 'data.currency', value: 'EUR' },
             transactionId: 'data.id',
             pageType: { value: 'purchase' },
@@ -170,7 +170,8 @@ export const lead: Flow.StepExample = {
   title: 'Lead',
   description:
     'A newsletter form submission is sent to Bing UET as a lead event with the SHA-256 hashed email.',
-  in: getEvent('form submit', {
+  in: createEvent({
+    name: 'form submit',
     id: 'b1c2d3e4f5a60003',
     timestamp: 1700000902000,
     data: { type: 'newsletter' },
diff --git a/packages/server/destinations/clickhouse/src/examples/step.ts b/packages/server/destinations/clickhouse/src/examples/step.ts
index 6ed523930..26248cc9a 100644
--- a/packages/server/destinations/clickhouse/src/examples/step.ts
+++ b/packages/server/destinations/clickhouse/src/examples/step.ts
@@ -167,7 +167,7 @@ const purchaseEvent = getEvent('order complete', {
   data: {
     id: 'ORD-500',
     currency: 'EUR',
-    total: 199.99,
+    value: 199.99,
     items: [
       { sku: 'SKU-1', quantity: 2 },
       { sku: 'SKU-2', quantity: 1 },
diff --git a/packages/server/destinations/criteo/src/__tests__/hash.test.ts b/packages/server/destinations/criteo/src/__tests__/hash.test.ts
index df7fceafe..64f0c9b58 100644
--- a/packages/server/destinations/criteo/src/__tests__/hash.test.ts
+++ b/packages/server/destinations/criteo/src/__tests__/hash.test.ts
@@ -1,7 +1,7 @@
 import { createHash } from 'node:crypto';
 import type { WalkerOS } from '@walkeros/core';
 import type { Config, CriteoEmailHashes, Settings } from '../types';
-import { clone, createMockContext, getEvent } from '@walkeros/core';
+import { clone, createEvent, createMockContext } from '@walkeros/core';
 import { examples } from '../dev';
 
 const { env } = examples;
@@ -25,7 +25,7 @@ describe('Criteo email hashing', () => {
 
   async function pushEmail(email: string): Promise {
     const destination = jest.requireActual('../').default;
-    const event = getEvent('form submit', { user: { email } });
+    const event = createEvent({ name: 'form submit', user: { email } });
     const config: Config = {
       settings: {
         partnerId,
diff --git a/packages/server/destinations/criteo/src/__tests__/index.test.ts b/packages/server/destinations/criteo/src/__tests__/index.test.ts
index 6b4fd9a9d..9b904a3ec 100644
--- a/packages/server/destinations/criteo/src/__tests__/index.test.ts
+++ b/packages/server/destinations/criteo/src/__tests__/index.test.ts
@@ -2,10 +2,11 @@ import type { WalkerOS, Collector } from '@walkeros/core';
 import type { Config, Destination, Settings } from '../types';
 import {
   clone,
+  createEvent,
   createIngest,
-  getEvent,
   createMockContext,
   createMockLogger,
+  getEvent,
 } from '@walkeros/core';
 import { startFlow } from '@walkeros/collector';
 import { examples } from '../dev';
@@ -241,7 +242,8 @@ describe('Server Destination Criteo', () => {
   });
 
   test('raw email is hashed into md5, sha256, sha256_md5', async () => {
-    const event = getEvent('form submit', {
+    const event = createEvent({
+      name: 'form submit',
       user: { email: 'user@example.com' },
     });
     const config: Config = {
@@ -351,7 +353,7 @@ describe('Server Destination Criteo', () => {
     const dest = jest.requireActual('../').default;
     const event = getEvent('order complete', {
       timestamp: 1700000900000,
-      data: { id: 'ORD-1', total: 100, currency: 'EUR' },
+      data: { id: 'ORD-1', value: 100, currency: 'EUR' },
       nested: [
         {
           entity: 'product',
diff --git a/packages/server/destinations/criteo/src/examples/step.ts b/packages/server/destinations/criteo/src/examples/step.ts
index 38475f99c..76d2f57a5 100644
--- a/packages/server/destinations/criteo/src/examples/step.ts
+++ b/packages/server/destinations/criteo/src/examples/step.ts
@@ -29,7 +29,7 @@ export const purchase: Flow.StepExample = {
     'A completed order is posted to the Criteo Events API as a trackTransaction event with items.',
   in: getEvent('order complete', {
     timestamp: 1700000900000,
-    data: { id: 'ORD-300', total: 249.99, currency: 'EUR' },
+    data: { id: 'ORD-300', value: 249.99, currency: 'EUR' },
     nested: [
       {
         entity: 'product',
diff --git a/packages/server/destinations/customerio/src/examples/step.ts b/packages/server/destinations/customerio/src/examples/step.ts
index f3d58c4c4..4fdc35595 100644
--- a/packages/server/destinations/customerio/src/examples/step.ts
+++ b/packages/server/destinations/customerio/src/examples/step.ts
@@ -1,5 +1,5 @@
 import type { Flow } from '@walkeros/core';
-import { getEvent } from '@walkeros/core';
+import { createEvent, getEvent } from '@walkeros/core';
 import type { Settings } from '../types';
 
 /**
@@ -71,14 +71,14 @@ export const mappedData: CustomerIoStepExample = {
   in: getEvent('order complete', {
     timestamp: 1700000102,
     user: { id: 'us3r', session: 's3ss10n' },
-    data: { id: '0rd3r1d', total: 555, currency: 'EUR' },
+    data: { id: '0rd3r1d', value: 555, currency: 'EUR' },
   }),
   mapping: {
     name: 'purchase',
     data: {
       map: {
         order_id: 'data.id',
-        value: 'data.total',
+        value: 'data.value',
         currency: 'data.currency',
       },
     },
@@ -163,7 +163,8 @@ export const userLoginIdentify: CustomerIoStepExample = {
   title: 'User login identify',
   description:
     'A user login triggers only a Customer.io identify call with profile attributes, skipping the track.',
-  in: getEvent('user login', {
+  in: createEvent({
+    name: 'user login',
     timestamp: 1700000105,
     user: { id: 'us3r', session: 's3ss10n' },
     data: {
@@ -237,7 +238,8 @@ export const destroyPerson: CustomerIoStepExample = {
   title: 'Destroy person',
   description:
     'A user delete event permanently removes the person from Customer.io via trackClient.destroy.',
-  in: getEvent('user delete', {
+  in: createEvent({
+    name: 'user delete',
     timestamp: 1700000107,
     user: { id: 'us3r', session: 's3ss10n' },
   }),
@@ -257,7 +259,8 @@ export const suppressPerson: CustomerIoStepExample = {
   title: 'Suppress person',
   description:
     'A user suppress event stops messaging for the person without deleting their profile data.',
-  in: getEvent('user suppress', {
+  in: createEvent({
+    name: 'user suppress',
     timestamp: 1700000108,
     user: { id: 'us3r', session: 's3ss10n' },
   }),
@@ -277,7 +280,8 @@ export const unsuppressPerson: CustomerIoStepExample = {
   title: 'Unsuppress person',
   description:
     'A user unsuppress event resumes messaging for a previously suppressed Customer.io profile.',
-  in: getEvent('user unsuppress', {
+  in: createEvent({
+    name: 'user unsuppress',
     timestamp: 1700000109,
     user: { id: 'us3r', session: 's3ss10n' },
   }),
@@ -296,7 +300,8 @@ export const unsuppressPerson: CustomerIoStepExample = {
  */
 export const wildcardIgnored: CustomerIoStepExample = {
   public: false,
-  in: getEvent('debug noise', {
+  in: createEvent({
+    name: 'debug noise',
     timestamp: 1700000110,
     user: { id: 'us3r', session: 's3ss10n' },
   }),
diff --git a/packages/server/destinations/datamanager/src/__tests__/index.test.ts b/packages/server/destinations/datamanager/src/__tests__/index.test.ts
index 2579692d3..b9a655d0f 100644
--- a/packages/server/destinations/datamanager/src/__tests__/index.test.ts
+++ b/packages/server/destinations/datamanager/src/__tests__/index.test.ts
@@ -276,7 +276,7 @@ describe('Server Destination Data Manager', () => {
     test('sends event to Data Manager API', async () => {
       const event = getEvent('order complete');
       (event.data as Record).id = 'ORDER-123';
-      (event.data as Record).total = 99.99;
+      (event.data as Record).value = 99.99;
       (event.data as Record).currency = 'USD';
 
       const config: Config = {
diff --git a/packages/server/destinations/datamanager/src/examples/mapping.ts b/packages/server/destinations/datamanager/src/examples/mapping.ts
index 1efd15734..ee5af12db 100644
--- a/packages/server/destinations/datamanager/src/examples/mapping.ts
+++ b/packages/server/destinations/datamanager/src/examples/mapping.ts
@@ -10,7 +10,7 @@ export const Purchase: DestinationDataManager.Rule = {
     map: {
       // Required fields
       transactionId: 'data.id',
-      conversionValue: 'data.total',
+      conversionValue: 'data.value',
       currency: { key: 'data.currency', value: 'USD' },
       eventName: { value: 'purchase' },
 
diff --git a/packages/server/destinations/datamanager/src/examples/step.ts b/packages/server/destinations/datamanager/src/examples/step.ts
index a06bad770..62c221710 100644
--- a/packages/server/destinations/datamanager/src/examples/step.ts
+++ b/packages/server/destinations/datamanager/src/examples/step.ts
@@ -1,5 +1,5 @@
 import type { Flow } from '@walkeros/core';
-import { getEvent } from '@walkeros/core';
+import { createEvent, getEvent } from '@walkeros/core';
 
 /**
  * Google Data Manager step examples.
@@ -51,7 +51,7 @@ export const purchase: Flow.StepExample = {
   in: getEvent('order complete', {
     id: 'ev-1700000900000',
     timestamp: 1700000900000,
-    data: { id: 'ORD-600', total: 149.99, currency: 'EUR' },
+    data: { id: 'ORD-600', value: 149.99, currency: 'EUR' },
     user: { id: 'user-abc', email: 'buyer@example.com' },
     source: { type: 'express', platform: 'server' },
   }),
@@ -60,7 +60,7 @@ export const purchase: Flow.StepExample = {
     data: {
       map: {
         transactionId: 'data.id',
-        conversionValue: 'data.total',
+        conversionValue: 'data.value',
         currency: { key: 'data.currency', value: 'USD' },
         eventName: { value: 'purchase' },
         userId: 'user.id',
@@ -106,7 +106,8 @@ export const lead: Flow.StepExample = {
   title: 'Lead',
   description:
     'A demo request form submission is sent to Data Manager as a generate_lead conversion with a hashed email.',
-  in: getEvent('form submit', {
+  in: createEvent({
+    name: 'form submit',
     id: 'ev-1700000901000',
     timestamp: 1700000901000,
     data: { type: 'demo-request' },
diff --git a/packages/server/destinations/file/src/examples/step.ts b/packages/server/destinations/file/src/examples/step.ts
index 1113d22a1..d564735a5 100644
--- a/packages/server/destinations/file/src/examples/step.ts
+++ b/packages/server/destinations/file/src/examples/step.ts
@@ -1,5 +1,5 @@
 import type { Flow } from '@walkeros/core';
-import { getEvent } from '@walkeros/core';
+import { createEvent, getEvent } from '@walkeros/core';
 import type { Format } from '../types';
 
 /**
@@ -96,7 +96,8 @@ export const jsonlTenantShardKey: FileStepExample = {
   title: 'Tenant sharding',
   description:
     'The filename is resolved from an event field so events are partitioned into per-tenant JSONL files.',
-  in: getEvent('custom event', {
+  in: createEvent({
+    name: 'custom event',
     timestamp: 1700000000000,
     data: { tenant: 'acme' },
   }),
diff --git a/packages/server/destinations/gcp/src/bigquery/examples/step.ts b/packages/server/destinations/gcp/src/bigquery/examples/step.ts
index 325c3882d..5c297d13a 100644
--- a/packages/server/destinations/gcp/src/bigquery/examples/step.ts
+++ b/packages/server/destinations/gcp/src/bigquery/examples/step.ts
@@ -39,7 +39,7 @@ const purchaseEvent = getEvent('order complete', {
   timestamp: 1700001101,
   data: {
     id: 'ORD-500',
-    total: 199.99,
+    value: 199.99,
     items: [{ sku: 'SKU-1', qty: 2 }],
   },
   source: { type: 'express', platform: 'server' },
diff --git a/packages/server/destinations/gcp/src/hints.ts b/packages/server/destinations/gcp/src/hints.ts
index 64be8c931..227c0da1a 100644
--- a/packages/server/destinations/gcp/src/hints.ts
+++ b/packages/server/destinations/gcp/src/hints.ts
@@ -22,7 +22,7 @@ export const hints: Hint.Hints = {
     code: [
       {
         lang: 'sql',
-        code: "SELECT JSON_EXTRACT_SCALAR(data, '$.total') AS total FROM `project.walkeros.events` WHERE entity = 'order'",
+        code: "SELECT JSON_EXTRACT_SCALAR(data, '$.value') AS value FROM `project.walkeros.events` WHERE entity = 'order'",
       },
     ],
   },
diff --git a/packages/server/destinations/gcp/src/pubsub/examples/step.ts b/packages/server/destinations/gcp/src/pubsub/examples/step.ts
index 6ebf707b5..411d17af6 100644
--- a/packages/server/destinations/gcp/src/pubsub/examples/step.ts
+++ b/packages/server/destinations/gcp/src/pubsub/examples/step.ts
@@ -19,7 +19,7 @@ declare const __VERSION__: string;
 // preserves these (stamp-if-absent), keeping the example deterministic.
 const orderEvent = getEvent('order complete', {
   timestamp: 1700001100,
-  data: { id: 'ORD-500', total: 199.99, currency: 'EUR' },
+  data: { id: 'ORD-500', value: 199.99, currency: 'EUR' },
   user: { id: 'usr-789' },
   source: {
     type: 'express',
@@ -162,7 +162,7 @@ export const mappedData: Flow.StepExample = {
     data: {
       map: {
         order_id: 'data.id',
-        revenue: 'data.total',
+        revenue: 'data.value',
         currency: 'data.currency',
       },
     },
diff --git a/packages/server/destinations/hubspot/src/examples/step.ts b/packages/server/destinations/hubspot/src/examples/step.ts
index 477b85577..30f3fa18a 100644
--- a/packages/server/destinations/hubspot/src/examples/step.ts
+++ b/packages/server/destinations/hubspot/src/examples/step.ts
@@ -1,5 +1,5 @@
 import type { Flow } from '@walkeros/core';
-import { getEvent } from '@walkeros/core';
+import { createEvent, getEvent } from '@walkeros/core';
 import type { Settings } from '../types';
 
 /**
@@ -61,7 +61,7 @@ export const mappedEventName: HubSpotStepExample = {
   in: getEvent('order complete', {
     timestamp: 1700000101,
     user: { email: 'user@example.com' },
-    data: { total: 99.5, currency: 'EUR', id: 'ord-123' },
+    data: { value: 99.5, currency: 'EUR', id: 'ord-123' },
   }),
   mapping: {
     name: 'order complete',
@@ -69,7 +69,7 @@ export const mappedEventName: HubSpotStepExample = {
       eventName: 'purchase_completed',
       properties: {
         map: {
-          revenue: 'data.total',
+          revenue: 'data.value',
           currency: 'data.currency',
           order_id: 'data.id',
         },
@@ -179,7 +179,8 @@ export const userLoginIdentify: HubSpotStepExample = {
   title: 'User login identify',
   description:
     'A user login only upserts the HubSpot contact with profile and lifecycle properties, skipping the event send.',
-  in: getEvent('user login', {
+  in: createEvent({
+    name: 'user login',
     timestamp: 1700000104,
     user: { email: 'user@example.com' },
     data: {
@@ -270,7 +271,8 @@ export const noIdentity: HubSpotStepExample = {
  */
 export const wildcardIgnored: HubSpotStepExample = {
   public: false,
-  in: getEvent('debug noise', {
+  in: createEvent({
+    name: 'debug noise',
     timestamp: 1700000107,
     user: { email: 'user@example.com' },
   }),
diff --git a/packages/server/destinations/kafka/src/examples/step.ts b/packages/server/destinations/kafka/src/examples/step.ts
index 5917c3998..97de76084 100644
--- a/packages/server/destinations/kafka/src/examples/step.ts
+++ b/packages/server/destinations/kafka/src/examples/step.ts
@@ -1,5 +1,5 @@
 import type { Flow } from '@walkeros/core';
-import { getEvent } from '@walkeros/core';
+import { createEvent, getEvent } from '@walkeros/core';
 import type { Settings } from '../types';
 
 /**
@@ -84,14 +84,14 @@ export const mappedData: KafkaStepExample = {
     'A data mapping transforms the event payload before producing it as the Kafka message value.',
   in: getEvent('order complete', {
     timestamp: 1700000102,
-    data: { id: 'ORD-400', total: 99.99, currency: 'EUR' },
+    data: { id: 'ORD-400', value: 99.99, currency: 'EUR' },
   }),
   mapping: {
     name: 'purchase',
     data: {
       map: {
         order_id: 'data.id',
-        revenue: 'data.total',
+        revenue: 'data.value',
         currency: 'data.currency',
       },
     },
@@ -124,7 +124,8 @@ export const keyFromUser: KafkaStepExample = {
   title: 'Key from user id',
   description:
     'A settings.kafka.key path resolves the message key from the event, here using user.id for per-user partitioning.',
-  in: getEvent('user signup', {
+  in: createEvent({
+    name: 'user signup',
     timestamp: 1700000103,
     user: { id: 'usr-789' },
     data: { plan: 'pro' },
@@ -166,7 +167,7 @@ export const topicOverride: KafkaStepExample = {
     'A mapping rule overrides the destination topic so specific events are routed to a dedicated stream.',
   in: getEvent('order complete', {
     timestamp: 1700000104,
-    data: { id: 'ORD-500', total: 42 },
+    data: { id: 'ORD-500', value: 42 },
   }),
   mapping: {
     settings: {
@@ -198,9 +199,7 @@ export const topicOverride: KafkaStepExample = {
  */
 export const ignoredEvent: KafkaStepExample = {
   public: false,
-  in: getEvent('debug noise', {
-    timestamp: 1700000105,
-  }),
+  in: createEvent({ name: 'debug noise', timestamp: 1700000105 }),
   mapping: { ignore: true },
   out: [],
 };
diff --git a/packages/server/destinations/klaviyo/src/examples/step.ts b/packages/server/destinations/klaviyo/src/examples/step.ts
index 3e50557b0..db2920794 100644
--- a/packages/server/destinations/klaviyo/src/examples/step.ts
+++ b/packages/server/destinations/klaviyo/src/examples/step.ts
@@ -1,5 +1,5 @@
 import type { Flow } from '@walkeros/core';
-import { getEvent } from '@walkeros/core';
+import { createEvent, getEvent } from '@walkeros/core';
 import type { Settings } from '../types';
 
 /**
@@ -147,7 +147,7 @@ export const revenueEvent: KlaviyoStepExample = {
     user: { id: 'us3r', email: 'user@example.com' },
     data: {
       id: 'ORD-123',
-      total: 99.99,
+      value: 99.99,
       itemNames: ['Widget A', 'Widget B'],
     },
   }),
@@ -163,7 +163,7 @@ export const revenueEvent: KlaviyoStepExample = {
       },
     },
     settings: {
-      value: 'data.total',
+      value: 'data.value',
     },
   },
   out: [
@@ -211,7 +211,8 @@ export const userLoginIdentify: KlaviyoStepExample = {
   title: 'User login identify',
   description:
     'A user login upserts the Klaviyo profile with name, organization, and custom properties without firing an event.',
-  in: getEvent('user login', {
+  in: createEvent({
+    name: 'user login',
     id: 'a1b2c3d4e5f60103',
     timestamp: 1700000103,
     user: { id: 'us3r', email: 'user@acme.com' },
@@ -333,7 +334,8 @@ export const emailOnly: KlaviyoStepExample = {
   title: 'Email only',
   description:
     'A newsletter signup uses only the email address as the Klaviyo profile identifier, with no external id.',
-  in: getEvent('newsletter signup', {
+  in: createEvent({
+    name: 'newsletter signup',
     id: 'a1b2c3d4e5f60105',
     timestamp: 1700000105,
     user: { email: 'subscriber@example.com' },
@@ -378,7 +380,8 @@ export const emailOnly: KlaviyoStepExample = {
  */
 export const wildcardIgnored: KlaviyoStepExample = {
   public: false,
-  in: getEvent('debug noise', {
+  in: createEvent({
+    name: 'debug noise',
     id: 'a1b2c3d4e5f60106',
     timestamp: 1700000106,
     user: { id: 'us3r', email: 'user@example.com' },
@@ -401,7 +404,7 @@ export const dedupedOrder: KlaviyoStepExample = {
     id: 'a1b2c3d4e5f60107',
     timestamp: 1700000107,
     user: { id: 'us3r', email: 'user@example.com' },
-    data: { id: 'ORD-123', total: 49.5 },
+    data: { id: 'ORD-123', value: 49.5 },
   }),
   mapping: {
     name: 'Placed Order',
diff --git a/packages/server/destinations/linkedin/src/__tests__/index.test.ts b/packages/server/destinations/linkedin/src/__tests__/index.test.ts
index 2f5162472..79b86d9ed 100644
--- a/packages/server/destinations/linkedin/src/__tests__/index.test.ts
+++ b/packages/server/destinations/linkedin/src/__tests__/index.test.ts
@@ -2,9 +2,10 @@ import type { WalkerOS, Collector } from '@walkeros/core';
 import type { Config, Destination, Rules, Settings } from '../types';
 import {
   clone,
-  getEvent,
+  createEvent,
   createMockContext,
   createMockLogger,
+  getEvent,
 } from '@walkeros/core';
 import { startFlow } from '@walkeros/collector';
 import { examples } from '../dev';
@@ -99,7 +100,8 @@ describe('Server Destination LinkedIn', () => {
     customSendServer.mockResolvedValue({ ok: true, data: {} });
 
     const customEnv = { sendServer: customSendServer };
-    const event = getEvent('form submit', {
+    const event = createEvent({
+      name: 'form submit',
       user: { email: 'test@test.com' },
     });
     const config: Config = {
@@ -126,7 +128,8 @@ describe('Server Destination LinkedIn', () => {
       error: '401 Unauthorized',
     });
 
-    const event = getEvent('form submit', {
+    const event = createEvent({
+      name: 'form submit',
       user: { email: 'test@test.com' },
     });
     const config: Config = {
@@ -146,7 +149,8 @@ describe('Server Destination LinkedIn', () => {
   });
 
   test('headers - all 5 required headers are sent', async () => {
-    const event = getEvent('form submit', {
+    const event = createEvent({
+      name: 'form submit',
       user: { email: 'test@test.com' },
     });
     const config: Config = {
@@ -178,7 +182,8 @@ describe('Server Destination LinkedIn', () => {
   });
 
   test('headers - custom apiVersion in Linkedin-Version header', async () => {
-    const event = getEvent('form submit', {
+    const event = createEvent({
+      name: 'form submit',
       user: { email: 'test@test.com' },
     });
     const config: Config = {
@@ -203,7 +208,8 @@ describe('Server Destination LinkedIn', () => {
   });
 
   test('email hashing - email is SHA-256 hashed, lowercase trimmed', async () => {
-    const event = getEvent('form submit', {
+    const event = createEvent({
+      name: 'form submit',
       user: { email: '  User@Example.COM  ' },
     });
     const config: Config = {
@@ -230,7 +236,8 @@ describe('Server Destination LinkedIn', () => {
   });
 
   test('email hashing - doNotHash skips hashing', async () => {
-    const event = getEvent('form submit', {
+    const event = createEvent({
+      name: 'form submit',
       user: { email: 'already-hashed-value' },
     });
     const config: Config = {
@@ -257,7 +264,8 @@ describe('Server Destination LinkedIn', () => {
   });
 
   test('no userIds skips - event without email or li_fat_id does not call sendServer', async () => {
-    const event = getEvent('form submit', {
+    const event = createEvent({
+      name: 'form submit',
       user: { id: 'user-123' },
     });
     const config: Config = {
@@ -279,7 +287,7 @@ describe('Server Destination LinkedIn', () => {
   test('conversion value - amount is string in payload', async () => {
     const event = getEvent('order complete', {
       timestamp: 1700000000000,
-      data: { total: 99.99, currency: 'EUR' },
+      data: { value: 99.99, currency: 'EUR' },
       user: { email: 'test@test.com' },
     });
     const config: Config = {
@@ -290,7 +298,7 @@ describe('Server Destination LinkedIn', () => {
             settings: {
               conversion: {
                 map: {
-                  value: 'data.total',
+                  value: 'data.value',
                   currency: { key: 'data.currency', value: 'EUR' },
                 },
               },
@@ -352,7 +360,8 @@ describe('Server Destination LinkedIn', () => {
   });
 
   test('conversionHappenedAt uses event.timestamp in milliseconds', async () => {
-    const event = getEvent('form submit', {
+    const event = createEvent({
+      name: 'form submit',
       timestamp: 1700000900000,
       user: { email: 'test@test.com' },
     });
@@ -374,7 +383,8 @@ describe('Server Destination LinkedIn', () => {
   });
 
   test('eventId from event.id for deduplication', async () => {
-    const event = getEvent('form submit', {
+    const event = createEvent({
+      name: 'form submit',
       timestamp: 1700000900000,
       user: { email: 'test@test.com' },
     });
@@ -396,7 +406,8 @@ describe('Server Destination LinkedIn', () => {
   });
 
   test('endpoint uses default URL', async () => {
-    const event = getEvent('form submit', {
+    const event = createEvent({
+      name: 'form submit',
       user: { email: 'test@test.com' },
     });
     const config: Config = {
@@ -417,7 +428,8 @@ describe('Server Destination LinkedIn', () => {
   });
 
   test('endpoint uses custom URL', async () => {
-    const event = getEvent('form submit', {
+    const event = createEvent({
+      name: 'form submit',
       user: { email: 'test@test.com' },
     });
     const config: Config = {
diff --git a/packages/server/destinations/linkedin/src/examples/step.ts b/packages/server/destinations/linkedin/src/examples/step.ts
index f3f45de04..27cda5c5a 100644
--- a/packages/server/destinations/linkedin/src/examples/step.ts
+++ b/packages/server/destinations/linkedin/src/examples/step.ts
@@ -1,5 +1,5 @@
 import type { Flow } from '@walkeros/core';
-import { getEvent } from '@walkeros/core';
+import { createEvent, getEvent } from '@walkeros/core';
 
 /**
  * LinkedIn Conversions API step examples.
@@ -40,7 +40,7 @@ export const purchase: Flow.StepExample = {
   in: getEvent('order complete', {
     id: 'ev-1700000900000',
     timestamp: 1700000900000,
-    data: { total: 249.99, currency: 'EUR' },
+    data: { value: 249.99, currency: 'EUR' },
     user: { email: 'jane@example.com' },
     source: { type: 'express', platform: 'server' },
   }),
@@ -48,7 +48,7 @@ export const purchase: Flow.StepExample = {
     settings: {
       conversion: {
         map: {
-          value: 'data.total',
+          value: 'data.value',
           currency: { key: 'data.currency', value: 'EUR' },
         },
       },
@@ -90,7 +90,8 @@ export const lead: Flow.StepExample = {
   title: 'Lead',
   description:
     'A form submission is posted to LinkedIn as a conversion with the SHA-256 hashed email as the user identifier.',
-  in: getEvent('form submit', {
+  in: createEvent({
+    name: 'form submit',
     id: 'ev-1700000901000',
     timestamp: 1700000901000,
     user: { email: 'user@example.com' },
@@ -132,7 +133,7 @@ export const purchaseWithLiFatId: Flow.StepExample = {
   in: getEvent('order complete', {
     id: 'ev-1700000902000',
     timestamp: 1700000902000,
-    data: { total: 89.99, currency: 'USD' },
+    data: { value: 89.99, currency: 'USD' },
     user: { email: 'buyer@co.com' },
     context: { li_fat_id: ['abc123-fat-id', 0] },
     source: { type: 'express', platform: 'server' },
@@ -141,7 +142,7 @@ export const purchaseWithLiFatId: Flow.StepExample = {
     settings: {
       conversion: {
         map: {
-          value: 'data.total',
+          value: 'data.value',
           currency: { key: 'data.currency', value: 'USD' },
         },
       },
diff --git a/packages/server/destinations/meta/src/examples/step.ts b/packages/server/destinations/meta/src/examples/step.ts
index 7f6d49e37..f0a7d3951 100644
--- a/packages/server/destinations/meta/src/examples/step.ts
+++ b/packages/server/destinations/meta/src/examples/step.ts
@@ -1,5 +1,5 @@
 import type { Flow, WalkerOS } from '@walkeros/core';
-import { getEvent } from '@walkeros/core';
+import { createEvent, getEvent } from '@walkeros/core';
 
 /**
  * Meta Conversions API step examples.
@@ -42,7 +42,7 @@ export const purchase: Flow.StepExample = {
   in: getEvent('order complete', {
     id: 'c1d2e3f4a5b60001',
     timestamp: 1700000900000,
-    data: { id: 'ORD-300', total: 249.99, currency: 'EUR' },
+    data: { id: 'ORD-300', value: 249.99, currency: 'EUR' },
     nested: [
       { entity: 'product', data: { id: 'SKU-A1', price: 129.99, quantity: 2 } },
     ],
@@ -59,7 +59,7 @@ export const purchase: Flow.StepExample = {
       map: {
         order_id: 'data.id',
         currency: { key: 'data.currency', value: 'EUR' },
-        value: 'data.total',
+        value: 'data.value',
         contents: {
           loop: [
             'nested',
@@ -116,7 +116,8 @@ export const lead: Flow.StepExample = {
   title: 'Form submit',
   description:
     'A form submission is forwarded to Meta CAPI as a custom event with the event source URL.',
-  in: getEvent('form submit', {
+  in: createEvent({
+    name: 'form submit',
     id: 'c1d2e3f4a5b60002',
     timestamp: 1700000901000,
     data: { type: 'newsletter' },
@@ -156,7 +157,7 @@ export const purchaseWithClickAttribution: Flow.StepExample = {
   in: getEvent('order complete', {
     id: 'c1d2e3f4a5b60003',
     timestamp: 1700000902000,
-    data: { id: 'ORD-700', total: 89.99, currency: 'USD' },
+    data: { id: 'ORD-700', value: 89.99, currency: 'USD' },
     user: { id: 'cust-42' },
     context: { fbclid: ['abc123xyz', 0] },
     source: {
@@ -170,7 +171,7 @@ export const purchaseWithClickAttribution: Flow.StepExample = {
     data: {
       map: {
         currency: { key: 'data.currency', value: 'EUR' },
-        value: 'data.total',
+        value: 'data.value',
         order_id: 'data.id',
         user_data: {
           map: {
diff --git a/packages/server/destinations/mixpanel/src/examples/step.ts b/packages/server/destinations/mixpanel/src/examples/step.ts
index 13deeb86e..61dfddbe8 100644
--- a/packages/server/destinations/mixpanel/src/examples/step.ts
+++ b/packages/server/destinations/mixpanel/src/examples/step.ts
@@ -1,5 +1,5 @@
 import type { Flow } from '@walkeros/core';
-import { getEvent } from '@walkeros/core';
+import { createEvent, getEvent } from '@walkeros/core';
 import type { Settings } from '../types';
 
 /**
@@ -93,7 +93,8 @@ export const perEventIdentify: MixpanelStepExample = {
   title: 'Per-event identify',
   description:
     'A mapping-level identify overrides the destination default to resolve the distinct_id from event data.',
-  in: getEvent('user login', {
+  in: createEvent({
+    name: 'user login',
     timestamp: 1700000102,
     data: {
       user_id: 'resolved-id',
@@ -170,7 +171,8 @@ export const userLoginPeopleSet: MixpanelStepExample = {
   title: 'User login people',
   description:
     'A user login fires Mixpanel people.set, set_once, and increment operations without sending a track event.',
-  in: getEvent('user login', {
+  in: createEvent({
+    name: 'user login',
     timestamp: 1700000104,
     data: {
       user_id: 'new-user-123',
@@ -230,7 +232,8 @@ export const allPeopleOperations: MixpanelStepExample = {
   title: 'Common people operations',
   description:
     'A profile update exercises the common Mixpanel people vocabulary: set, set_once, increment, append, union, remove, and unset. delete_user is not covered by this example.',
-  in: getEvent('profile update', {
+  in: createEvent({
+    name: 'profile update',
     timestamp: 1700000105,
     data: {
       name: 'Jane Doe',
@@ -304,7 +307,8 @@ export const companyGroupProfile: MixpanelStepExample = {
   title: 'Group profile',
   description:
     'A company update sets Mixpanel group profile properties via groups.set and groups.set_once.',
-  in: getEvent('company update', {
+  in: createEvent({
+    name: 'company update',
     timestamp: 1700000106,
     data: {
       company_id: 'acme-inc',
@@ -358,7 +362,7 @@ export const historicalImport: MixpanelStepExample = {
   in: getEvent('order complete', {
     timestamp: 1700000107,
     data: {
-      total: 99.99,
+      value: 99.99,
     },
   }),
   settings: {
@@ -379,7 +383,8 @@ export const aliasBeforeTrack: MixpanelStepExample = {
   title: 'Alias before track',
   description:
     'A user login merges a prior anonymous id into the new user id via mp.alias before sending the track event.',
-  in: getEvent('user login', {
+  in: createEvent({
+    name: 'user login',
     timestamp: 1700000108,
     data: {
       user_id: 'new-user-456',
@@ -407,7 +412,7 @@ export const aliasBeforeTrack: MixpanelStepExample = {
  */
 export const wildcardIgnored: MixpanelStepExample = {
   public: false,
-  in: getEvent('debug noise', { timestamp: 1700000109 }),
+  in: createEvent({ name: 'debug noise', timestamp: 1700000109 }),
   mapping: { ignore: true },
   out: [],
 };
diff --git a/packages/server/destinations/mparticle/src/__tests__/push.test.ts b/packages/server/destinations/mparticle/src/__tests__/push.test.ts
index 5f911acc2..700f45ce0 100644
--- a/packages/server/destinations/mparticle/src/__tests__/push.test.ts
+++ b/packages/server/destinations/mparticle/src/__tests__/push.test.ts
@@ -109,7 +109,7 @@ describe('push', () => {
   test('uses commerce_event when mapping.eventType is commerce_event', async () => {
     const event = getEvent('order complete', {
       timestamp: 1700000300000,
-      data: { id: 'ORD-1', total: 99.99, currency: 'USD' },
+      data: { id: 'ORD-1', value: 99.99, currency: 'USD' },
       user: { id: 'u1' },
     });
     const config: Config = {
@@ -130,7 +130,7 @@ describe('push', () => {
                     map: {
                       action: { value: 'purchase' },
                       transaction_id: 'data.id',
-                      total_amount: 'data.total',
+                      total_amount: 'data.value',
                     },
                   },
                 },
diff --git a/packages/server/destinations/mparticle/src/examples/step.ts b/packages/server/destinations/mparticle/src/examples/step.ts
index 7476c766d..6eee5880b 100644
--- a/packages/server/destinations/mparticle/src/examples/step.ts
+++ b/packages/server/destinations/mparticle/src/examples/step.ts
@@ -1,5 +1,5 @@
 import type { Flow } from '@walkeros/core';
-import { getEvent } from '@walkeros/core';
+import { createEvent, getEvent } from '@walkeros/core';
 
 /**
  * mParticle Events API step examples.
@@ -133,7 +133,7 @@ export const commercePurchase: Flow.StepExample = {
   in: getEvent('order complete', {
     id: 'ev-1700000300000',
     timestamp: 1700000300000,
-    data: { id: 'ORD-300', total: 249.99, currency: 'EUR' },
+    data: { id: 'ORD-300', value: 249.99, currency: 'EUR' },
     user: { id: 'user-123' },
     source: { type: 'express', platform: 'server' },
   }),
@@ -147,7 +147,7 @@ export const commercePurchase: Flow.StepExample = {
             map: {
               action: { value: 'purchase' },
               transaction_id: 'data.id',
-              total_amount: 'data.total',
+              total_amount: 'data.value',
             },
           },
         },
@@ -193,7 +193,8 @@ export const identityAndAttributes: Flow.StepExample = {
   title: 'User identities',
   description:
     'A form submission sends a custom_event whose batch carries user_identities resolved from destination settings.',
-  in: getEvent('form submit', {
+  in: createEvent({
+    name: 'form submit',
     id: 'ev-1700000400000',
     timestamp: 1700000400000,
     data: { type: 'newsletter' },
diff --git a/packages/server/destinations/pinterest/src/examples/step.ts b/packages/server/destinations/pinterest/src/examples/step.ts
index 1bb7d60bf..80b1f72eb 100644
--- a/packages/server/destinations/pinterest/src/examples/step.ts
+++ b/packages/server/destinations/pinterest/src/examples/step.ts
@@ -37,7 +37,7 @@ export const checkout: Flow.StepExample = {
   in: getEvent('order complete', {
     id: 'a1b2c3d4e5f60001',
     timestamp: 1700000900,
-    data: { id: 'ORD-300', total: 249.99, currency: 'EUR' },
+    data: { id: 'ORD-300', value: 249.99, currency: 'EUR' },
     nested: [
       {
         entity: 'product',
@@ -62,7 +62,7 @@ export const checkout: Flow.StepExample = {
       map: {
         custom_data: {
           map: {
-            value: 'data.total',
+            value: 'data.value',
             currency: { key: 'data.currency', value: 'EUR' },
             order_id: 'data.id',
             num_items: {
diff --git a/packages/server/destinations/piwikpro/src/__tests__/hits.test.ts b/packages/server/destinations/piwikpro/src/__tests__/hits.test.ts
index 8b2fc4260..dd2a12bf8 100644
--- a/packages/server/destinations/piwikpro/src/__tests__/hits.test.ts
+++ b/packages/server/destinations/piwikpro/src/__tests__/hits.test.ts
@@ -1,7 +1,7 @@
-import type { Collector, WalkerOS } from '@walkeros/core';
+import type { Collector, StandardEventName, WalkerOS } from '@walkeros/core';
 import type { Hit, Rule, Settings } from '../types';
 import type { BuildResult } from '../hits';
-import { getEvent } from '@walkeros/core';
+import { createEvent, getEvent } from '@walkeros/core';
 import { startFlow } from '@walkeros/collector';
 import { getHashServer } from '@walkeros/server-core';
 import { buildHits } from '../hits';
@@ -23,7 +23,7 @@ describe('buildHits', () => {
   });
 
   function event(
-    name: string,
+    name: StandardEventName,
     props: WalkerOS.DeepPartialEvent = {},
   ): WalkerOS.Event {
     return getEvent(name, {
@@ -33,6 +33,19 @@ describe('buildHits', () => {
     });
   }
 
+  /** An event outside the standard set, with the same timestamp and source. */
+  function otherEvent(
+    name: string,
+    props: WalkerOS.DeepPartialEvent = {},
+  ): WalkerOS.Event {
+    return createEvent({
+      timestamp: 1700000300000,
+      source: SOURCE,
+      ...props,
+      name,
+    });
+  }
+
   /** The hits of a successful build, as parameter maps. */
   function params(result: BuildResult): Array> {
     if ('skip' in result) throw new Error(`unexpected skip: ${result.skip}`);
@@ -91,13 +104,13 @@ describe('buildHits', () => {
     ],
     [
       'a method outside the table',
-      event('setUserId'),
+      otherEvent('setUserId'),
       { name: 'setUserId' },
       'unknown method setUserId',
     ],
     [
       'silent without a goal',
-      event('trackEvent'),
+      otherEvent('trackEvent'),
       { name: 'trackEvent', silent: true },
       'silent',
     ],
@@ -116,7 +129,11 @@ describe('buildHits', () => {
   it('skips an invalid method call with its reason', async () => {
     expect(
       await buildHits(
-        { event: event('trackEvent'), rule: { name: 'trackEvent' }, data: [] },
+        {
+          event: otherEvent('trackEvent'),
+          rule: { name: 'trackEvent' },
+          data: [],
+        },
         settings,
         collector,
       ),
@@ -132,7 +149,7 @@ describe('buildHits', () => {
     const [main, goal] = params(
       await buildHits(
         {
-          event: event('trackEvent', { data: { position: 3 } }),
+          event: otherEvent('trackEvent', { data: { position: 3 } }),
           rule: goalRule,
           data: ['promotion', 'visible'],
         },
@@ -153,7 +170,7 @@ describe('buildHits', () => {
       await buildHits(
         {
           event: event('order complete'),
-          rule: { settings: { goalId: 'g1', goalValue: 'data.total' } },
+          rule: { settings: { goalId: 'g1', goalValue: 'data.value' } },
         },
         settings,
         collector,
@@ -169,7 +186,7 @@ describe('buildHits', () => {
     const hits = params(
       await buildHits(
         {
-          event: event('trackEvent', { data: { position: 3 } }),
+          event: otherEvent('trackEvent', { data: { position: 3 } }),
           rule: { ...goalRule, silent: true },
         },
         settings,
@@ -183,7 +200,7 @@ describe('buildHits', () => {
   it('sets url to the trackLink address', async () => {
     const result = await buildHits(
       {
-        event: event('trackLink'),
+        event: otherEvent('trackLink'),
         rule: { name: 'trackLink' },
         data: ['https://example.com/a.pdf', 'download'],
       },
diff --git a/packages/server/destinations/piwikpro/src/__tests__/push.test.ts b/packages/server/destinations/piwikpro/src/__tests__/push.test.ts
index 583e078be..f82620b8d 100644
--- a/packages/server/destinations/piwikpro/src/__tests__/push.test.ts
+++ b/packages/server/destinations/piwikpro/src/__tests__/push.test.ts
@@ -1,6 +1,12 @@
-import type { Collector, Logger, SendResponse, WalkerOS } from '@walkeros/core';
+import type {
+  Collector,
+  Logger,
+  SendResponse,
+  StandardEventName,
+  WalkerOS,
+} from '@walkeros/core';
 import type { PartialConfig } from '../types';
-import { createIngest, getEvent, Level } from '@walkeros/core';
+import { createEvent, createIngest, getEvent, Level } from '@walkeros/core';
 import { startFlow } from '@walkeros/collector';
 import { destinationPiwikPro } from '..';
 
@@ -37,7 +43,10 @@ describe('push', () => {
     return collector;
   }
 
-  function event(name: string, props: WalkerOS.DeepPartialEvent = {}) {
+  function event(
+    name: StandardEventName,
+    props: WalkerOS.DeepPartialEvent = {},
+  ) {
     return getEvent(name, {
       timestamp: 1700000300000,
       source: SOURCE,
@@ -179,8 +188,20 @@ describe('push', () => {
   it('warns once per skip reason, then logs at debug', async () => {
     await setup();
     // An event name no other test pushes, so the warned key is this test's own
-    await collector.push(event('skip probe'));
-    await collector.push(event('skip probe'));
+    await collector.push(
+      createEvent({
+        name: 'skip probe',
+        timestamp: 1700000300000,
+        source: SOURCE,
+      }),
+    );
+    await collector.push(
+      createEvent({
+        name: 'skip probe',
+        timestamp: 1700000300000,
+        source: SOURCE,
+      }),
+    );
     const skips = logs.filter(([, message]) => message.includes('skipped'));
     expect(skips).toEqual([
       [Level.WARN, 'Piwik PRO skipped an event: unmapped'],
diff --git a/packages/server/destinations/piwikpro/src/examples/step.ts b/packages/server/destinations/piwikpro/src/examples/step.ts
index 74461347f..d38bf28ce 100644
--- a/packages/server/destinations/piwikpro/src/examples/step.ts
+++ b/packages/server/destinations/piwikpro/src/examples/step.ts
@@ -1,10 +1,11 @@
 import type {
   Flow,
   Mapping as WalkerOSMapping,
+  StandardEventName,
   WalkerOS,
 } from '@walkeros/core';
 import type { Hit, Rule, Settings } from '../types';
-import { getEvent } from '@walkeros/core';
+import { createEvent, getEvent } from '@walkeros/core';
 
 // Build-time version define, sent as ts_v on every hit.
 declare const __VERSION__: string;
@@ -63,12 +64,20 @@ const IDENTIFIED: Hit = [
 ];
 
 function event(
-  name: string,
+  name: StandardEventName,
   props: WalkerOS.DeepPartialEvent = {},
 ): WalkerOS.Event {
   return getEvent(name, { timestamp: TIMESTAMP, source: SOURCE, ...props });
 }
 
+/** An event outside the standard set, with the same timestamp and source. */
+function otherEvent(
+  name: string,
+  props: WalkerOS.DeepPartialEvent = {},
+): WalkerOS.Event {
+  return createEvent({ timestamp: TIMESTAMP, source: SOURCE, ...props, name });
+}
+
 /** The one sendServer call for these hits, each already in parameter order. */
 function send(...hits: Hit[]): Flow.StepOut {
   const requests = hits.map((hit) => `?${new URLSearchParams(hit).toString()}`);
@@ -167,9 +176,9 @@ export const eventWithGoalValue: StepExample = {
   in: event('order complete'),
   mapping: {
     name: 'trackEvent',
-    settings: { goalId: GOAL_ID, goalValue: 'data.total' },
+    settings: { goalId: GOAL_ID, goalValue: 'data.value' },
     data: {
-      set: [{ value: 'order' }, { value: 'complete' }, 'data.id', 'data.total'],
+      set: [{ value: 'order' }, { value: 'complete' }, 'data.id', 'data.value'],
     },
   },
   out: send(
@@ -214,7 +223,7 @@ export const siteSearch: StepExample = {
   title: 'Site search',
   description:
     'A search event maps to trackSiteSearch with keyword, category and result count.',
-  in: event('search submit', {
+  in: otherEvent('search submit', {
     data: { query: 'rucksack', category: 'bags', results: 12 },
   }),
   mapping: {
@@ -236,7 +245,7 @@ export const trackLink: StepExample = {
   title: 'Download link',
   description:
     'A tagged download maps to trackLink, which sends the file address as download and as the hit url.',
-  in: event('file download', {
+  in: otherEvent('file download', {
     data: { url: 'https://www.example.com/whitepaper.pdf' },
   }),
   mapping: {
@@ -257,7 +266,7 @@ export const ping: StepExample = {
   title: 'Ping',
   description:
     'A heartbeat event maps to ping, which extends the session without counting as a page view.',
-  in: event('page ping'),
+  in: otherEvent('page ping'),
   mapping: {
     name: 'ping',
   },
@@ -296,7 +305,7 @@ export const ecommerceRemoveFromCart: StepExample = {
   title: 'Remove from cart',
   description:
     'A product remove maps to ecommerceRemoveFromCart with the removed product.',
-  in: event('product remove', {
+  in: otherEvent('product remove', {
     data: {
       id: 'ers',
       name: 'Everyday Ruck Snack',
@@ -347,7 +356,7 @@ export const ecommerceOrder: StepExample = {
         {
           map: {
             orderId: 'data.id',
-            grandTotal: 'data.total',
+            grandTotal: 'data.value',
             tax: 'data.taxes',
             shipping: 'data.shipping',
           },
diff --git a/packages/server/destinations/piwikpro/src/schemas/mapping.ts b/packages/server/destinations/piwikpro/src/schemas/mapping.ts
index 68045bb44..a3b87bd19 100644
--- a/packages/server/destinations/piwikpro/src/schemas/mapping.ts
+++ b/packages/server/destinations/piwikpro/src/schemas/mapping.ts
@@ -9,7 +9,7 @@ export const MappingSchema = z.object({
     )
     .optional(),
   goalValue: schemas.ValueSchema.describe(
-    'Goal revenue, a mapping value resolved against the event (like "data.total"), sent as revenue on the goal hit',
+    'Goal revenue, a mapping value resolved against the event (like "data.value"), sent as revenue on the goal hit',
   ).optional(),
   customDimensions: CustomDimensionsSchema.describe(
     'Custom dimensions for this rule, keyed by bare dimension id. Values are mapping values resolved against the event (like { "1": "data.size" }). Wins per key over the destination customDimensions.',
diff --git a/packages/server/destinations/posthog/src/examples/step.ts b/packages/server/destinations/posthog/src/examples/step.ts
index a90b0f227..43876b1e0 100644
--- a/packages/server/destinations/posthog/src/examples/step.ts
+++ b/packages/server/destinations/posthog/src/examples/step.ts
@@ -1,5 +1,5 @@
 import type { Flow, WalkerOS } from '@walkeros/core';
-import { getEvent } from '@walkeros/core';
+import { createEvent, getEvent } from '@walkeros/core';
 import type { Settings } from '../types';
 
 /**
@@ -61,7 +61,7 @@ export const captureWithInclude: PostHogStepExample = {
           data_currency: 'EUR',
           data_shipping: 5.22,
           data_taxes: 73.76,
-          data_total: 555,
+          data_value: 555,
           globals_pagegroup: 'shop',
         },
       },
@@ -77,7 +77,8 @@ export const identifyWithSetAndSetOnce: PostHogStepExample = {
   title: 'Identify with $set',
   description:
     'A user login fires PostHog identify with $set and $set_once person properties and skips the capture.',
-  in: getEvent('user login', {
+  in: createEvent({
+    name: 'user login',
     timestamp: 1700000102,
     data: {
       user_id: 'new-user-123',
@@ -133,7 +134,8 @@ export const groupIdentifyWithProperties: PostHogStepExample = {
   title: 'Group identify',
   description:
     'A company update fires PostHog groupIdentify with group type, key, and associated group properties.',
-  in: getEvent('company update', {
+  in: createEvent({
+    name: 'company update',
     timestamp: 1700000103,
     data: {
       company_id: 'company_123',
diff --git a/packages/server/destinations/reddit/src/examples/step.ts b/packages/server/destinations/reddit/src/examples/step.ts
index 92aa2d023..d4e9d40de 100644
--- a/packages/server/destinations/reddit/src/examples/step.ts
+++ b/packages/server/destinations/reddit/src/examples/step.ts
@@ -1,5 +1,5 @@
 import type { Flow, WalkerOS } from '@walkeros/core';
-import { getEvent, isObject } from '@walkeros/core';
+import { createEvent, getEvent, isObject } from '@walkeros/core';
 
 /**
  * Reddit Conversions API step examples.
@@ -38,7 +38,7 @@ export const purchase: Flow.StepExample = {
   in: getEvent('order complete', {
     id: 'ev-1700000900',
     timestamp: 1700000900,
-    data: { id: 'ORD-300', total: 249.99, currency: 'EUR' },
+    data: { id: 'ORD-300', value: 249.99, currency: 'EUR' },
     nested: [
       {
         entity: 'product',
@@ -60,7 +60,7 @@ export const purchase: Flow.StepExample = {
       map: {
         event_metadata: {
           map: {
-            value_decimal: 'data.total',
+            value_decimal: 'data.value',
             currency: { key: 'data.currency', value: 'EUR' },
             item_count: {
               fn: (event: unknown) =>
@@ -232,7 +232,8 @@ export const lead: Flow.StepExample = {
   title: 'Lead',
   description:
     'A form submission is sent to Reddit as a Lead conversion with the SHA-256 hashed email and external id.',
-  in: getEvent('form submit', {
+  in: createEvent({
+    name: 'form submit',
     id: 'ev-1700000903',
     timestamp: 1700000903,
     data: { form: 'contact' },
@@ -285,7 +286,8 @@ export const signUp: Flow.StepExample = {
   title: 'Sign up',
   description:
     'A user signup is sent to Reddit as a SignUp conversion with hashed user identifiers.',
-  in: getEvent('user signup', {
+  in: createEvent({
+    name: 'user signup',
     id: 'ev-1700000904',
     timestamp: 1700000904,
     data: { method: 'email' },
@@ -338,7 +340,8 @@ export const search: Flow.StepExample = {
   title: 'Search',
   description:
     'A site search is sent to Reddit as a Search conversion with an item count in event_metadata.',
-  in: getEvent('site search', {
+  in: createEvent({
+    name: 'site search',
     id: 'ev-1700000905',
     timestamp: 1700000905,
     data: { query: 'walkerOS destinations' },
diff --git a/packages/server/destinations/redis/src/examples/step.ts b/packages/server/destinations/redis/src/examples/step.ts
index f831f1682..6de12f39e 100644
--- a/packages/server/destinations/redis/src/examples/step.ts
+++ b/packages/server/destinations/redis/src/examples/step.ts
@@ -1,5 +1,5 @@
 import type { Flow } from '@walkeros/core';
-import { getEvent } from '@walkeros/core';
+import { createEvent, getEvent } from '@walkeros/core';
 import type { Settings } from '../types';
 
 /**
@@ -31,7 +31,7 @@ export const orderComplete: RedisStepExample = {
     'An order complete event is appended to the Redis stream alongside other event types for downstream consumers.',
   in: getEvent('order complete', {
     timestamp: 1700000101,
-    data: { id: 'ORD-400', total: 99.99, currency: 'EUR' },
+    data: { id: 'ORD-400', value: 99.99, currency: 'EUR' },
   }),
   out: [['client.xadd', ['walkeros:events', '*', 'event', 'json:event']]],
 };
@@ -95,7 +95,7 @@ export const streamKeyOverride: RedisStepExample = {
     'A mapping rule routes the event to a dedicated Redis stream instead of the destination default.',
   in: getEvent('order complete', {
     timestamp: 1700000104,
-    data: { id: 'ORD-500', total: 42 },
+    data: { id: 'ORD-500', value: 42 },
   }),
   mapping: {
     settings: {
@@ -110,9 +110,7 @@ export const streamKeyOverride: RedisStepExample = {
  */
 export const ignoredEvent: RedisStepExample = {
   public: false,
-  in: getEvent('debug noise', {
-    timestamp: 1700000105,
-  }),
+  in: createEvent({ name: 'debug noise', timestamp: 1700000105 }),
   mapping: { ignore: true },
   out: [],
 };
diff --git a/packages/server/destinations/rudderstack/src/examples/step.ts b/packages/server/destinations/rudderstack/src/examples/step.ts
index 4533382d4..7517fb6ca 100644
--- a/packages/server/destinations/rudderstack/src/examples/step.ts
+++ b/packages/server/destinations/rudderstack/src/examples/step.ts
@@ -1,5 +1,5 @@
 import type { Flow } from '@walkeros/core';
-import { getEvent } from '@walkeros/core';
+import { createEvent, getEvent } from '@walkeros/core';
 import type { Settings } from '../types';
 
 /**
@@ -127,7 +127,8 @@ export const userLoginIdentify: RudderStackStepExample = {
   title: 'User login identify',
   description:
     'A user login fires only an analytics.identify call with the resolved userId and traits, skipping the track.',
-  in: getEvent('user login', {
+  in: createEvent({
+    name: 'user login',
     timestamp: 1700000103,
     user: { id: 'us3r', session: 's3ss10n' },
     data: {
@@ -174,7 +175,8 @@ export const companyGroup: RudderStackStepExample = {
   title: 'Group company',
   description:
     'A company update fires analytics.group with the groupId and group traits for B2B account tracking.',
-  in: getEvent('company update', {
+  in: createEvent({
+    name: 'company update',
     timestamp: 1700000104,
     user: { id: 'us3r', session: 's3ss10n' },
     data: {
@@ -268,7 +270,8 @@ export const screenView: RudderStackStepExample = {
   title: 'Screen view',
   description:
     'A screen view from a mobile app backend fires analytics.screen with a screen name and properties.',
-  in: getEvent('screen view', {
+  in: createEvent({
+    name: 'screen view',
     timestamp: 1700000106,
     user: { id: 'us3r', session: 's3ss10n' },
     data: {
@@ -314,7 +317,8 @@ export const aliasUser: RudderStackStepExample = {
   title: 'Alias user',
   description:
     'An identity merge fires analytics.alias to link a previous anonymous id to the newly registered user id.',
-  in: getEvent('identity merge', {
+  in: createEvent({
+    name: 'identity merge',
     timestamp: 1700000107,
     user: { id: 'registered-456', session: 's3ss10n' },
     data: {
@@ -376,7 +380,8 @@ export const anonymousOnly: RudderStackStepExample = {
  */
 export const wildcardIgnored: RudderStackStepExample = {
   public: false,
-  in: getEvent('debug noise', {
+  in: createEvent({
+    name: 'debug noise',
     timestamp: 1700000109,
     user: { id: 'us3r', session: 's3ss10n' },
   }),
diff --git a/packages/server/destinations/segment/src/examples/step.ts b/packages/server/destinations/segment/src/examples/step.ts
index 5ceffcf67..522ee668c 100644
--- a/packages/server/destinations/segment/src/examples/step.ts
+++ b/packages/server/destinations/segment/src/examples/step.ts
@@ -1,5 +1,5 @@
 import type { Flow } from '@walkeros/core';
-import { getEvent } from '@walkeros/core';
+import { createEvent, getEvent } from '@walkeros/core';
 import type { Settings } from '../types';
 
 /**
@@ -127,7 +127,8 @@ export const userLoginIdentify: SegmentStepExample = {
   title: 'User login identify',
   description:
     'A user login fires only Segment analytics.identify with the userId and traits, skipping the track.',
-  in: getEvent('user login', {
+  in: createEvent({
+    name: 'user login',
     timestamp: 1700000103,
     user: { id: 'us3r', session: 's3ss10n' },
     data: {
@@ -174,7 +175,8 @@ export const companyGroup: SegmentStepExample = {
   title: 'Group company',
   description:
     'A company update fires Segment analytics.group with groupId and group traits for account-level tracking.',
-  in: getEvent('company update', {
+  in: createEvent({
+    name: 'company update',
     timestamp: 1700000104,
     user: { id: 'us3r', session: 's3ss10n' },
     data: {
@@ -270,7 +272,8 @@ export const screenView: SegmentStepExample = {
   title: 'Screen view',
   description:
     'A screen view fires Segment analytics.screen with name, category, and properties for mobile app tracking.',
-  in: getEvent('screen view', {
+  in: createEvent({
+    name: 'screen view',
     timestamp: 1700000106,
     user: { id: 'us3r', session: 's3ss10n' },
     data: {
@@ -384,7 +387,8 @@ export const consentForwarding: SegmentStepExample = {
  */
 export const wildcardIgnored: SegmentStepExample = {
   public: false,
-  in: getEvent('debug noise', {
+  in: createEvent({
+    name: 'debug noise',
     timestamp: 1700000109,
     user: { id: 'us3r', session: 's3ss10n' },
   }),
diff --git a/packages/server/destinations/slack/src/__tests__/message.test.ts b/packages/server/destinations/slack/src/__tests__/message.test.ts
index c2684b3b7..e19535f75 100644
--- a/packages/server/destinations/slack/src/__tests__/message.test.ts
+++ b/packages/server/destinations/slack/src/__tests__/message.test.ts
@@ -1,12 +1,12 @@
-import { getEvent } from '@walkeros/core';
+import { createEvent, getEvent } from '@walkeros/core';
 import { buildMessage, interpolate } from '../message';
 
 describe('interpolate', () => {
   it('replaces ${data.field} from the event', () => {
     const event = getEvent('order complete', {
-      data: { id: 'ORD-1', total: 50 },
+      data: { id: 'ORD-1', value: 50 },
     });
-    expect(interpolate('Order ${data.id} for ${data.total}', event)).toBe(
+    expect(interpolate('Order ${data.id} for ${data.value}', event)).toBe(
       'Order ORD-1 for 50',
     );
   });
@@ -27,10 +27,10 @@ describe('buildMessage', () => {
 
   it('uses interpolated text from rule.settings when provided', () => {
     const event = getEvent('order complete', {
-      data: { id: 'ORD-1', total: 50 },
+      data: { id: 'ORD-1', value: 50 },
     });
     const msg = buildMessage(event, baseSettings, {
-      text: 'Order ${data.id}: ${data.total}',
+      text: 'Order ${data.id}: ${data.value}',
     });
     expect(msg.text).toBe('Order ORD-1: 50');
     expect(msg.blocks).toBeUndefined();
@@ -44,7 +44,8 @@ describe('buildMessage', () => {
   });
 
   it('auto-generates default blocks when no text or blocks supplied', () => {
-    const event = getEvent('lead submit', {
+    const event = createEvent({
+      name: 'lead submit',
       data: { name: 'Acme', email: 'sales@acme.test' },
       source: { type: 'crm', platform: 'server' },
     });
diff --git a/packages/server/destinations/slack/src/examples/step.ts b/packages/server/destinations/slack/src/examples/step.ts
index 96050e444..7b11b662e 100644
--- a/packages/server/destinations/slack/src/examples/step.ts
+++ b/packages/server/destinations/slack/src/examples/step.ts
@@ -1,5 +1,5 @@
 import type { Flow } from '@walkeros/core';
-import { getEvent } from '@walkeros/core';
+import { createEvent, getEvent } from '@walkeros/core';
 import type { Settings } from '../types';
 
 /**
@@ -39,7 +39,7 @@ export const purchaseAlert: SlackStepExample = {
     timestamp: 1700000100,
     data: {
       id: 'ORD-500',
-      total: 299.99,
+      value: 299.99,
       currency: 'EUR',
       product: 'Pro Plan',
     },
@@ -48,7 +48,7 @@ export const purchaseAlert: SlackStepExample = {
   mapping: {
     settings: {
       channel: '#sales',
-      text: ':moneybag: New order: ${data.id} - ${data.total} ${data.currency}',
+      text: ':moneybag: New order: ${data.id} - ${data.value} ${data.currency}',
     },
   },
   out: [
@@ -72,7 +72,8 @@ export const errorAlert: SlackStepExample = {
   title: 'Error alert',
   description:
     'An error event posts a critical alert to an engineering channel using a mapping-level channel override.',
-  in: getEvent('error occur', {
+  in: createEvent({
+    name: 'error occur',
     timestamp: 1700000200,
     data: {
       message: 'Payment gateway timeout',
@@ -107,7 +108,8 @@ export const welcomeDM: SlackStepExample = {
   title: 'Welcome DM',
   description:
     'A user signup opens a Slack DM channel and posts a welcome message with the selected plan.',
-  in: getEvent('user signup', {
+  in: createEvent({
+    name: 'user signup',
     timestamp: 1700000300,
     data: { plan: 'enterprise' },
     user: { id: 'U-NEW-USER' },
@@ -141,7 +143,8 @@ export const threadedCheckoutStep: SlackStepExample = {
   title: 'Threaded reply',
   description:
     'A checkout step posts as a threaded reply in Slack via thread_ts with broadcast to the channel.',
-  in: getEvent('checkout step', {
+  in: createEvent({
+    name: 'checkout step',
     timestamp: 1700000400,
     data: { step: 'payment', sessionTs: '1700000000.000050' },
   }),
@@ -176,7 +179,8 @@ export const ephemeralMessage: SlackStepExample = {
   title: 'Ephemeral message',
   description:
     'A quota warning posts an ephemeral Slack message visible only to a target admin user.',
-  in: getEvent('quota warning', {
+  in: createEvent({
+    name: 'quota warning',
     timestamp: 1700000500,
     data: { remaining: 5 },
   }),
@@ -211,7 +215,8 @@ export const defaultBlocks: SlackStepExample = {
   title: 'Default blocks',
   description:
     'With no custom text the destination auto-generates a Block Kit message from event data and source.',
-  in: getEvent('lead submit', {
+  in: createEvent({
+    name: 'lead submit',
     timestamp: 1700000600,
     data: { name: 'Acme', email: 'sales@acme.test' },
     source: { type: 'crm', platform: 'server' },
@@ -261,7 +266,8 @@ export const deployNotification: SlackStepExample = {
   title: 'Webhook deploy',
   description:
     'Without a token the destination posts to an incoming Slack webhook URL with the rendered message body.',
-  in: getEvent('deploy complete', {
+  in: createEvent({
+    name: 'deploy complete',
     timestamp: 1700000700,
     data: { version: '1.4.2', environment: 'prod' },
   }),
diff --git a/packages/server/destinations/snapchat/src/__tests__/index.test.ts b/packages/server/destinations/snapchat/src/__tests__/index.test.ts
index c19985d3f..833e5ca7d 100644
--- a/packages/server/destinations/snapchat/src/__tests__/index.test.ts
+++ b/packages/server/destinations/snapchat/src/__tests__/index.test.ts
@@ -2,10 +2,11 @@ import type { WalkerOS, Collector } from '@walkeros/core';
 import type { Config, Destination, Settings } from '../types';
 import {
   clone,
+  createEvent,
   createIngest,
-  getEvent,
   createMockContext,
   createMockLogger,
+  getEvent,
 } from '@walkeros/core';
 import { startFlow } from '@walkeros/collector';
 import { examples } from '../dev';
@@ -167,7 +168,7 @@ describe('Server Destination Snapchat', () => {
     const dest = jest.requireActual('../').default;
     const event = getEvent('order complete', {
       timestamp: 1700000900000,
-      data: { id: 'ORD-1', total: 100, currency: 'EUR' },
+      data: { id: 'ORD-1', value: 100, currency: 'EUR' },
       source: {
         type: 'browser',
         platform: 'web',
@@ -187,7 +188,7 @@ describe('Server Destination Snapchat', () => {
                 map: {
                   custom_data: {
                     map: {
-                      value: 'data.total',
+                      value: 'data.value',
                       currency: 'data.currency',
                       transaction_id: 'data.id',
                     },
@@ -257,7 +258,8 @@ describe('Server Destination Snapchat', () => {
   });
 
   test('user data is hashed', async () => {
-    const event = getEvent('form submit', {
+    const event = createEvent({
+      name: 'form submit',
       user: { email: 'test@example.com' },
     });
     const config: Config = {
@@ -285,7 +287,8 @@ describe('Server Destination Snapchat', () => {
   });
 
   test('user_data mapping resolves from settings', async () => {
-    const event = getEvent('form submit', {
+    const event = createEvent({
+      name: 'form submit',
       user: { email: 'u@example.com', id: 'user-123' },
     });
     const config: Config = {
diff --git a/packages/server/destinations/snapchat/src/examples/step.ts b/packages/server/destinations/snapchat/src/examples/step.ts
index 1c95a76d3..acabc4ecc 100644
--- a/packages/server/destinations/snapchat/src/examples/step.ts
+++ b/packages/server/destinations/snapchat/src/examples/step.ts
@@ -1,5 +1,5 @@
 import type { Flow } from '@walkeros/core';
-import { getEvent, isObject } from '@walkeros/core';
+import { createEvent, getEvent, isObject } from '@walkeros/core';
 
 /**
  * Snapchat Conversions API step examples.
@@ -33,7 +33,7 @@ export const purchase: Flow.StepExample = {
   in: getEvent('order complete', {
     id: 'e1f2a3b4c5d60001',
     timestamp: 1700000900000,
-    data: { id: 'ORD-300', total: 249.99, currency: 'EUR' },
+    data: { id: 'ORD-300', value: 249.99, currency: 'EUR' },
     nested: [
       {
         entity: 'product',
@@ -53,7 +53,7 @@ export const purchase: Flow.StepExample = {
       map: {
         custom_data: {
           map: {
-            value: 'data.total',
+            value: 'data.value',
             currency: { key: 'data.currency', value: 'EUR' },
             transaction_id: 'data.id',
             contents: {
@@ -230,7 +230,8 @@ export const lead: Flow.StepExample = {
   title: 'Sign up',
   description:
     'A newsletter form submission is sent to Snapchat as a SIGN_UP conversion with a hashed email.',
-  in: getEvent('form submit', {
+  in: createEvent({
+    name: 'form submit',
     id: 'e1f2a3b4c5d60004',
     timestamp: 1700000903000,
     data: { type: 'newsletter' },
diff --git a/packages/server/destinations/sqlite/src/examples/step.ts b/packages/server/destinations/sqlite/src/examples/step.ts
index 8454486e5..9595b1982 100644
--- a/packages/server/destinations/sqlite/src/examples/step.ts
+++ b/packages/server/destinations/sqlite/src/examples/step.ts
@@ -1,5 +1,5 @@
 import type { Flow } from '@walkeros/core';
-import { getEvent } from '@walkeros/core';
+import { createEvent, getEvent } from '@walkeros/core';
 import type { Settings } from '../types';
 
 /**
@@ -60,7 +60,8 @@ export const customTable: SqliteStepExample = {
   title: 'Custom table',
   description:
     'A destination-level table setting inserts events into a custom SQLite table with the same column layout.',
-  in: getEvent('form submit', {
+  in: createEvent({
+    name: 'form submit',
     timestamp: 1700000101,
     id: 'evt-2',
     user: { session: 'sess-99', id: '' },
@@ -112,7 +113,7 @@ export const orderComplete: SqliteStepExample = {
     timestamp: 1700000102,
     id: 'evt-3',
     user: { session: '', id: '' },
-    data: { id: 'ORD-1', total: 99 },
+    data: { id: 'ORD-1', value: 99 },
     source: { type: 'collector', schema: '4' },
     globals: {},
     consent: {},
@@ -131,7 +132,7 @@ export const orderComplete: SqliteStepExample = {
         '',
         '',
         '',
-        JSON.stringify({ id: 'ORD-1', total: 99 }),
+        JSON.stringify({ id: 'ORD-1', value: 99 }),
         JSON.stringify({}),
         JSON.stringify({}),
       ],
@@ -150,7 +151,7 @@ export const tableOverride: SqliteStepExample = {
     timestamp: 1700000103,
     id: 'evt-4',
     user: { session: '', id: '' },
-    data: { id: 'ORD-2', total: 42 },
+    data: { id: 'ORD-2', value: 42 },
     source: { type: 'collector', schema: '4' },
     globals: {},
     consent: {},
@@ -174,7 +175,7 @@ export const tableOverride: SqliteStepExample = {
         '',
         '',
         '',
-        JSON.stringify({ id: 'ORD-2', total: 42 }),
+        JSON.stringify({ id: 'ORD-2', value: 42 }),
         JSON.stringify({}),
         JSON.stringify({}),
       ],
@@ -187,7 +188,8 @@ export const tableOverride: SqliteStepExample = {
  */
 export const ignoredEvent: SqliteStepExample = {
   public: false,
-  in: getEvent('debug noise', {
+  in: createEvent({
+    name: 'debug noise',
     timestamp: 1700000104,
     id: 'evt-5',
     source: { type: 'collector', schema: '4' },
diff --git a/packages/server/destinations/tiktok/src/__tests__/index.test.ts b/packages/server/destinations/tiktok/src/__tests__/index.test.ts
index 6e9490ddf..eda1d5aed 100644
--- a/packages/server/destinations/tiktok/src/__tests__/index.test.ts
+++ b/packages/server/destinations/tiktok/src/__tests__/index.test.ts
@@ -2,10 +2,11 @@ import type { WalkerOS, Collector } from '@walkeros/core';
 import type { Config, Destination, Rules, Settings } from '../types';
 import {
   clone,
+  createEvent,
   createIngest,
-  getEvent,
   createMockContext,
   createMockLogger,
+  getEvent,
 } from '@walkeros/core';
 import { startFlow } from '@walkeros/collector';
 import { examples } from '../dev';
@@ -226,7 +227,8 @@ describe('Server Destination TikTok', () => {
   });
 
   test('user data is hashed', async () => {
-    const event = getEvent('form submit', {
+    const event = createEvent({
+      name: 'form submit',
       user: { email: 'test@example.com' },
     });
     const config: Config = {
diff --git a/packages/server/destinations/tiktok/src/examples/step.ts b/packages/server/destinations/tiktok/src/examples/step.ts
index 44054677c..0ada1762d 100644
--- a/packages/server/destinations/tiktok/src/examples/step.ts
+++ b/packages/server/destinations/tiktok/src/examples/step.ts
@@ -1,5 +1,5 @@
 import type { Flow } from '@walkeros/core';
-import { getEvent, isObject } from '@walkeros/core';
+import { createEvent, getEvent, isObject } from '@walkeros/core';
 
 /**
  * TikTok Events API step examples.
@@ -34,7 +34,7 @@ export const purchase: Flow.StepExample = {
   in: getEvent('order complete', {
     id: 'd1e2f3a4b5c60001',
     timestamp: 1700000900000,
-    data: { id: 'ORD-300', total: 249.99, currency: 'EUR' },
+    data: { id: 'ORD-300', value: 249.99, currency: 'EUR' },
     nested: [
       {
         entity: 'product',
@@ -52,7 +52,7 @@ export const purchase: Flow.StepExample = {
     name: 'CompletePayment',
     data: {
       map: {
-        value: 'data.total',
+        value: 'data.value',
         currency: { key: 'data.currency', value: 'EUR' },
         order_id: 'data.id',
         content_type: { value: 'product' },
@@ -259,7 +259,8 @@ export const lead: Flow.StepExample = {
   title: 'Submit form',
   description:
     'A newsletter form submission is sent to TikTok as a SubmitForm event with the hashed email in user context.',
-  in: getEvent('form submit', {
+  in: createEvent({
+    name: 'form submit',
     id: 'd1e2f3a4b5c60004',
     timestamp: 1700000903000,
     data: { type: 'newsletter' },
diff --git a/packages/server/destinations/twitter/src/__tests__/index.test.ts b/packages/server/destinations/twitter/src/__tests__/index.test.ts
index bd3befc2c..ea4a662d0 100644
--- a/packages/server/destinations/twitter/src/__tests__/index.test.ts
+++ b/packages/server/destinations/twitter/src/__tests__/index.test.ts
@@ -2,10 +2,11 @@ import type { WalkerOS, Collector } from '@walkeros/core';
 import type { Config, Destination, Rules, Settings } from '../types';
 import {
   clone,
+  createEvent,
   createIngest,
-  getEvent,
   createMockContext,
   createMockLogger,
+  getEvent,
 } from '@walkeros/core';
 import { startFlow } from '@walkeros/collector';
 import { examples } from '../dev';
@@ -171,7 +172,8 @@ describe('Server Destination Twitter', () => {
     customSendServer.mockResolvedValue({ ok: true, data: {} });
 
     const customEnv = { sendServer: customSendServer };
-    const event = getEvent('form submit', {
+    const event = createEvent({
+      name: 'form submit',
       user: { email: 'test@test.com' },
     });
     const config: Config = { settings: baseSettings };
@@ -196,7 +198,8 @@ describe('Server Destination Twitter', () => {
       error: '401 Unauthorized',
     });
 
-    const event = getEvent('form submit', {
+    const event = createEvent({
+      name: 'form submit',
       user: { email: 'test@test.com' },
     });
     const config: Config = { settings: baseSettings };
@@ -214,7 +217,8 @@ describe('Server Destination Twitter', () => {
   });
 
   test('OAuth Authorization header is present and well-formed', async () => {
-    const event = getEvent('form submit', {
+    const event = createEvent({
+      name: 'form submit',
       user: { email: 'test@test.com' },
     });
     const config: Config = { settings: baseSettings };
@@ -236,7 +240,8 @@ describe('Server Destination Twitter', () => {
   });
 
   test('endpoint uses default URL with apiVersion and pixelId', async () => {
-    const event = getEvent('form submit', {
+    const event = createEvent({
+      name: 'form submit',
       user: { email: 'test@test.com' },
     });
     const config: Config = { settings: baseSettings };
@@ -253,7 +258,8 @@ describe('Server Destination Twitter', () => {
   });
 
   test('endpoint uses custom URL override', async () => {
-    const event = getEvent('form submit', {
+    const event = createEvent({
+      name: 'form submit',
       user: { email: 'test@test.com' },
     });
     const config: Config = {
@@ -272,7 +278,8 @@ describe('Server Destination Twitter', () => {
   });
 
   test('email hashing - email is SHA-256 hashed, trimmed, lowercased', async () => {
-    const event = getEvent('form submit', {
+    const event = createEvent({
+      name: 'form submit',
       user: { email: '  User@Example.COM  ' },
     });
     const config: Config = { settings: baseSettings };
@@ -292,7 +299,8 @@ describe('Server Destination Twitter', () => {
   });
 
   test('email hashing - doNotHash skips hashing', async () => {
-    const event = getEvent('form submit', {
+    const event = createEvent({
+      name: 'form submit',
       user: { email: 'already-hashed-email' },
     });
     const config: Config = {
@@ -311,7 +319,8 @@ describe('Server Destination Twitter', () => {
   });
 
   test('phone hashing - phone is SHA-256 hashed', async () => {
-    const event = getEvent('form submit', {
+    const event = createEvent({
+      name: 'form submit',
       user: { phone: '+1234567890' },
     });
     const config: Config = { settings: baseSettings };
@@ -332,7 +341,8 @@ describe('Server Destination Twitter', () => {
   });
 
   test('twclid passthrough - not hashed', async () => {
-    const event = getEvent('form submit', {
+    const event = createEvent({
+      name: 'form submit',
       user: { email: 'test@test.com' },
       context: { twclid: ['raw-twclid-value', 0] },
     });
@@ -357,7 +367,8 @@ describe('Server Destination Twitter', () => {
   });
 
   test('no primary identifier - event is skipped', async () => {
-    const event = getEvent('form submit', {
+    const event = createEvent({
+      name: 'form submit',
       user: { id: 'user-123' },
     });
     const config: Config = { settings: baseSettings };
@@ -371,7 +382,8 @@ describe('Server Destination Twitter', () => {
   });
 
   test('conversion_time is ISO 8601 format', async () => {
-    const event = getEvent('form submit', {
+    const event = createEvent({
+      name: 'form submit',
       timestamp: 1700000900000,
       user: { email: 'test@test.com' },
     });
@@ -389,7 +401,8 @@ describe('Server Destination Twitter', () => {
   });
 
   test('conversion_id equals event.id', async () => {
-    const event = getEvent('form submit', {
+    const event = createEvent({
+      name: 'form submit',
       user: { email: 'test@test.com' },
     });
     const config: Config = { settings: baseSettings };
@@ -406,7 +419,7 @@ describe('Server Destination Twitter', () => {
   test('value is cast to string in payload', async () => {
     const event = getEvent('order complete', {
       timestamp: 1700000000000,
-      data: { total: 99.99, currency: 'EUR' },
+      data: { value: 99.99, currency: 'EUR' },
       user: { email: 'test@test.com' },
     });
     const config: Config = {
@@ -415,7 +428,7 @@ describe('Server Destination Twitter', () => {
         order: {
           complete: {
             settings: {
-              value: 'data.total',
+              value: 'data.value',
               currency: { key: 'data.currency', value: 'EUR' },
             },
           },
@@ -535,7 +548,7 @@ describe('Server Destination Twitter', () => {
 
     test('fills client IP and user agent from ingest by default', async () => {
       await destination.push(
-        getEvent('form submit', { user: { email: 'test@test.com' } }),
+        createEvent({ name: 'form submit', user: { email: 'test@test.com' } }),
         createMockContext({
           config: { settings: { ...baseSettings } },
           env: testEnv,
@@ -548,7 +561,8 @@ describe('Server Destination Twitter', () => {
 
     test('falls back to event.user.ip and userAgent', async () => {
       await destination.push(
-        getEvent('form submit', {
+        createEvent({
+          name: 'form submit',
           user: {
             email: 'test@test.com',
             ip: '198.51.100.2',
@@ -566,7 +580,7 @@ describe('Server Destination Twitter', () => {
 
     test('sends neither when disabled with false', async () => {
       await destination.push(
-        getEvent('form submit', { user: { email: 'test@test.com' } }),
+        createEvent({ name: 'form submit', user: { email: 'test@test.com' } }),
         createMockContext({
           config: {
             settings: { ...baseSettings, ip: false, userAgent: false },
@@ -581,7 +595,7 @@ describe('Server Destination Twitter', () => {
 
     test('lets an explicitly mapped value win', async () => {
       await destination.push(
-        getEvent('form submit', { user: { email: 'test@test.com' } }),
+        createEvent({ name: 'form submit', user: { email: 'test@test.com' } }),
         createMockContext({
           config: { settings: { ...baseSettings } },
           data: { user_data: { ip_address: '10.0.0.1' } },
diff --git a/packages/server/destinations/twitter/src/examples/step.ts b/packages/server/destinations/twitter/src/examples/step.ts
index 2f1120bac..01bcaa6a6 100644
--- a/packages/server/destinations/twitter/src/examples/step.ts
+++ b/packages/server/destinations/twitter/src/examples/step.ts
@@ -1,5 +1,5 @@
 import type { Flow } from '@walkeros/core';
-import { getEvent } from '@walkeros/core';
+import { createEvent, getEvent } from '@walkeros/core';
 
 /**
  * X (Twitter) Conversions API step examples.
@@ -35,13 +35,13 @@ export const purchase: Flow.StepExample = {
   in: getEvent('order complete', {
     id: 'ev-1700000900000',
     timestamp: 1700000900000,
-    data: { total: 249.99, currency: 'EUR' },
+    data: { value: 249.99, currency: 'EUR' },
     user: { email: 'jane@example.com' },
     source: { type: 'express', platform: 'server' },
   }),
   mapping: {
     settings: {
-      value: 'data.total',
+      value: 'data.value',
       currency: { key: 'data.currency', value: 'EUR' },
     },
   },
@@ -74,7 +74,8 @@ export const lead: Flow.StepExample = {
   title: 'Lead',
   description:
     'A form submission is sent to X as a conversion with the SHA-256 hashed email as the identifier.',
-  in: getEvent('form submit', {
+  in: createEvent({
+    name: 'form submit',
     id: 'ev-1700000901000',
     timestamp: 1700000901000,
     user: { email: 'user@example.com' },
@@ -112,14 +113,14 @@ export const purchaseWithTwclid: Flow.StepExample = {
   in: getEvent('order complete', {
     id: 'ev-1700000902000',
     timestamp: 1700000902000,
-    data: { total: 89.99, currency: 'USD' },
+    data: { value: 89.99, currency: 'USD' },
     user: { email: 'buyer@co.com' },
     context: { twclid: ['23opevjt88psuo13lu8d020qkn', 0] },
     source: { type: 'express', platform: 'server' },
   }),
   mapping: {
     settings: {
-      value: 'data.total',
+      value: 'data.value',
       currency: { key: 'data.currency', value: 'USD' },
     },
     data: {
diff --git a/packages/server/sources/aws/src/sqs/examples/step.ts b/packages/server/sources/aws/src/sqs/examples/step.ts
index feb5e7593..92e4a4f75 100644
--- a/packages/server/sources/aws/src/sqs/examples/step.ts
+++ b/packages/server/sources/aws/src/sqs/examples/step.ts
@@ -17,7 +17,7 @@ const eventPayload = {
 
 const orderPayload = {
   event: 'order complete',
-  data: { id: 'ORD-500', total: 199.99, currency: 'EUR' },
+  data: { id: 'ORD-500', value: 199.99, currency: 'EUR' },
 };
 
 export const pageView: Flow.StepExample = {
diff --git a/packages/server/sources/fetch/src/examples/inputs.ts b/packages/server/sources/fetch/src/examples/inputs.ts
index 5ba09961c..b34174585 100644
--- a/packages/server/sources/fetch/src/examples/inputs.ts
+++ b/packages/server/sources/fetch/src/examples/inputs.ts
@@ -59,7 +59,7 @@ export const completeEvent: WalkerOS.DeepPartialEvent = {
   name: 'order complete',
   data: {
     id: 'ORDER-123',
-    total: 999.99,
+    value: 999.99,
     currency: 'USD',
   },
   context: {
diff --git a/packages/server/sources/gcp/src/cloudfunction/examples/step.ts b/packages/server/sources/gcp/src/cloudfunction/examples/step.ts
index 79411e294..697090ce8 100644
--- a/packages/server/sources/gcp/src/cloudfunction/examples/step.ts
+++ b/packages/server/sources/gcp/src/cloudfunction/examples/step.ts
@@ -33,7 +33,7 @@ export const orderEvent: Flow.StepExample = {
     method: 'POST',
     body: {
       name: 'order complete',
-      data: { id: 'ORD-700', total: 99.99, currency: 'EUR' },
+      data: { id: 'ORD-700', value: 99.99, currency: 'EUR' },
     },
     headers: { 'content-type': 'application/json' },
   },
@@ -42,7 +42,7 @@ export const orderEvent: Flow.StepExample = {
       'elb',
       {
         name: 'order complete',
-        data: { id: 'ORD-700', total: 99.99, currency: 'EUR' },
+        data: { id: 'ORD-700', value: 99.99, currency: 'EUR' },
       },
     ],
   ],
diff --git a/packages/server/sources/gcp/src/pubsub/pull/examples/step.ts b/packages/server/sources/gcp/src/pubsub/pull/examples/step.ts
index 734eb3f29..b157b856d 100644
--- a/packages/server/sources/gcp/src/pubsub/pull/examples/step.ts
+++ b/packages/server/sources/gcp/src/pubsub/pull/examples/step.ts
@@ -19,7 +19,7 @@ const eventPayload = {
 
 const orderPayload = {
   name: 'order complete',
-  data: { id: 'ORD-500', total: 199.99, currency: 'EUR' },
+  data: { id: 'ORD-500', value: 199.99, currency: 'EUR' },
 };
 
 export const pageView: Flow.StepExample = {
diff --git a/packages/server/sources/gcp/src/pubsub/push/examples/step.ts b/packages/server/sources/gcp/src/pubsub/push/examples/step.ts
index 3ed51e52e..e8938ff3d 100644
--- a/packages/server/sources/gcp/src/pubsub/push/examples/step.ts
+++ b/packages/server/sources/gcp/src/pubsub/push/examples/step.ts
@@ -16,7 +16,7 @@ const eventPayload = {
 
 const orderPayload = {
   name: 'order complete',
-  data: { id: 'ORD-500', total: 199.99, currency: 'EUR' },
+  data: { id: 'ORD-500', value: 199.99, currency: 'EUR' },
 };
 
 function encodeData(value: unknown): string {
diff --git a/packages/transformers/ga4/src/__tests__/map.test.ts b/packages/transformers/ga4/src/__tests__/map.test.ts
index 676307b66..0af799fce 100644
--- a/packages/transformers/ga4/src/__tests__/map.test.ts
+++ b/packages/transformers/ga4/src/__tests__/map.test.ts
@@ -358,7 +358,7 @@ describe('mapHitToEvents — purchase (canary)', () => {
       data: {
         id: 'T-9001',
         currency: 'EUR',
-        total: 129.9,
+        value: 129.9,
         tax: 20.78,
         shipping: 4.99,
         coupon: 'SUMMER10',
@@ -548,7 +548,7 @@ describe('mapHitToEvents — ecommerce family', () => {
       name: 'order refund',
       entity: 'order',
       action: 'refund',
-      data: { id: 'T-9001', currency: 'EUR', total: 129.9 },
+      data: { id: 'T-9001', currency: 'EUR', value: 129.9 },
     });
   });
 
diff --git a/packages/transformers/ga4/src/__tests__/transformer.test.ts b/packages/transformers/ga4/src/__tests__/transformer.test.ts
index 511dc8c9c..0da73b68d 100644
--- a/packages/transformers/ga4/src/__tests__/transformer.test.ts
+++ b/packages/transformers/ga4/src/__tests__/transformer.test.ts
@@ -319,7 +319,7 @@ describe('transformerGa4', () => {
           name: 'order complete',
           data: expect.objectContaining({
             id: 'T-99',
-            total: 42,
+            value: 42,
             currency: 'EUR',
           }),
         }),
@@ -349,7 +349,7 @@ describe('transformerGa4', () => {
           data: expect.objectContaining({
             id: 'T-1',
             currency: 'EUR',
-            total: 10,
+            value: 10,
             affiliation: 'partner-1',
           }),
         }),
@@ -367,7 +367,7 @@ describe('transformerGa4', () => {
       const result = await instance.push(emptyEvent, ctx);
       expect(result).toEqual({
         event: expect.objectContaining({
-          data: expect.objectContaining({ id: 'T-1', total: 10 }),
+          data: expect.objectContaining({ id: 'T-1', value: 10 }),
         }),
       });
       const event = (result as { event: WalkerOS.DeepPartialEvent }).event;
diff --git a/packages/transformers/ga4/src/defaults.ts b/packages/transformers/ga4/src/defaults.ts
index 7d2be4a57..7dc5d9059 100644
--- a/packages/transformers/ga4/src/defaults.ts
+++ b/packages/transformers/ga4/src/defaults.ts
@@ -6,7 +6,7 @@ import type { GA4Mapping } from './types';
  * The `'*'` key is the fallback for unknown event names.
  *
  * Field naming for `data.map`:
- * - Monetary: `currency` from `ep.currency`, `value`/`total` from `epn.value`,
+ * - Monetary: `currency` from `ep.currency`, `value` from `epn.value`,
  *   `tax` from `epn.tax`, `shipping` from `epn.shipping`, `coupon` from
  *   `ep.coupon`.
  * - Promotions read from `firstItem` (synthetic `items[0]` reference exposed by
@@ -32,7 +32,7 @@ export const defaultMapping: GA4Mapping = {
       map: {
         id: 'params.ep.transaction_id',
         currency: 'params.ep.currency',
-        total: 'params.epn.value',
+        value: 'params.epn.value',
         tax: 'params.epn.tax',
         shipping: 'params.epn.shipping',
         coupon: 'params.ep.coupon',
@@ -111,7 +111,7 @@ export const defaultMapping: GA4Mapping = {
       map: {
         id: 'params.ep.transaction_id',
         currency: 'params.ep.currency',
-        total: 'params.epn.value',
+        value: 'params.epn.value',
       },
     },
   },
diff --git a/packages/transformers/ga4/src/examples/step.ts b/packages/transformers/ga4/src/examples/step.ts
index ab0aec133..d7a81e6b6 100644
--- a/packages/transformers/ga4/src/examples/step.ts
+++ b/packages/transformers/ga4/src/examples/step.ts
@@ -94,7 +94,7 @@ export const pageView: Flow.StepExample = {
 export const purchase: Flow.StepExample = {
   title: 'Purchase (canary)',
   description:
-    'A GA4 purchase hit decoded to a walkerOS order complete event with id, currency, total, tax, shipping, and coupon.',
+    'A GA4 purchase hit decoded to a walkerOS order complete event with id, currency, value, tax, shipping, and coupon.',
   in: {
     url:
       'https://www.google-analytics.com/g/collect' +
@@ -118,7 +118,7 @@ export const purchase: Flow.StepExample = {
         data: {
           id: 'T-9001',
           currency: 'EUR',
-          total: 149.97,
+          value: 149.97,
           tax: 23.97,
           shipping: 4.95,
           coupon: 'WELCOME10',
diff --git a/packages/transformers/validate/README.md b/packages/transformers/validate/README.md
index 3d53a2319..f4520f39c 100644
--- a/packages/transformers/validate/README.md
+++ b/packages/transformers/validate/README.md
@@ -48,7 +48,7 @@ The same step in a bundled flow, referencing a named contract from the top-level
       "events": {
         "order": {
           "complete": {
-            "properties": { "data": { "required": ["total", "currency"] } }
+            "properties": { "data": { "required": ["value", "currency"] } }
           }
         }
       }
diff --git a/packages/transformers/validate/src/__tests__/validate.test.ts b/packages/transformers/validate/src/__tests__/validate.test.ts
index abcd7c97a..3890ac674 100644
--- a/packages/transformers/validate/src/__tests__/validate.test.ts
+++ b/packages/transformers/validate/src/__tests__/validate.test.ts
@@ -164,7 +164,7 @@ describe('validateEventAgainstContract', () => {
       name: 'order complete',
       entity: 'order',
       action: 'complete',
-      data: { id: 'O-1', total: 42.5 },
+      data: { id: 'O-1', value: 42.5 },
       context: { stage: ['checkout', 0] },
       timestamp: Date.now(),
     };
diff --git a/packages/web/destinations/amplitude/src/examples/step.ts b/packages/web/destinations/amplitude/src/examples/step.ts
index 3b84576fb..f1cd56f0e 100644
--- a/packages/web/destinations/amplitude/src/examples/step.ts
+++ b/packages/web/destinations/amplitude/src/examples/step.ts
@@ -1,5 +1,5 @@
 import type { Flow, WalkerOS } from '@walkeros/core';
-import { getEvent } from '@walkeros/core';
+import { createEvent, getEvent } from '@walkeros/core';
 import type { Settings } from '../types';
 
 /**
@@ -35,7 +35,7 @@ export const defaultEventForwarding: AmplitudeStepExample = {
  */
 export const wildcardIgnored: AmplitudeStepExample = {
   public: false,
-  in: getEvent('debug noise', { timestamp: 1700000101 }),
+  in: createEvent({ name: 'debug noise', timestamp: 1700000101 }),
   mapping: { ignore: true },
   out: [],
 };
@@ -136,7 +136,8 @@ export const userLoginIdentify: AmplitudeStepExample = {
   title: 'User login identify',
   description:
     'A user login sets the Amplitude userId and runs identify with set, setOnce, and add operations.',
-  in: getEvent('user login', {
+  in: createEvent({
+    name: 'user login',
     timestamp: 1700000105000,
     data: {
       user_id: 'new-user-123',
@@ -202,7 +203,7 @@ export const userLogoutReset: AmplitudeStepExample = {
   title: 'User logout reset',
   description:
     'A user logout calls amplitude.reset to clear the userId and regenerate the deviceId.',
-  in: getEvent('user logout', { timestamp: 1700000106 }),
+  in: createEvent({ name: 'user logout', timestamp: 1700000106 }),
   mapping: {
     silent: true,
     settings: {
@@ -223,7 +224,8 @@ export const subscriptionRenewRevenue: AmplitudeStepExample = {
   title: 'Subscription revenue',
   description:
     'A subscription renewal fires a single amplitude.revenue call with productId, price, and currency fallback.',
-  in: getEvent('subscription renew', {
+  in: createEvent({
+    name: 'subscription renew',
     timestamp: 1700000107,
     data: {
       plan_id: 'plan-pro',
@@ -326,7 +328,7 @@ export const orderCompleteMultiProduct: AmplitudeStepExample = {
         data_currency: 'EUR',
         data_shipping: 5.22,
         data_taxes: 73.76,
-        data_total: 555,
+        data_value: 555,
         globals_pagegroup: 'shop',
       },
     ],
@@ -341,7 +343,8 @@ export const groupAssignmentWithProperties: AmplitudeStepExample = {
   title: 'Group assignment',
   description:
     'A company update assigns the user to a group and sets group properties via setGroup and groupIdentify.',
-  in: getEvent('company update', {
+  in: createEvent({
+    name: 'company update',
     timestamp: 1700000109,
     data: {
       company: 'Acme',
diff --git a/packages/web/destinations/api/src/examples/step.ts b/packages/web/destinations/api/src/examples/step.ts
index 5c2a176df..f9919791f 100644
--- a/packages/web/destinations/api/src/examples/step.ts
+++ b/packages/web/destinations/api/src/examples/step.ts
@@ -92,7 +92,7 @@ export const customTransform: Flow.StepExample = {
     data: {
       map: {
         order_id: 'data.id',
-        amount: 'data.total',
+        amount: 'data.value',
         tax: 'data.taxes',
         shipping_cost: 'data.shipping',
         currency: 'data.currency',
diff --git a/packages/web/destinations/api/src/index.test.ts b/packages/web/destinations/api/src/index.test.ts
index b1774c7da..2540d03d4 100644
--- a/packages/web/destinations/api/src/index.test.ts
+++ b/packages/web/destinations/api/src/index.test.ts
@@ -2,11 +2,11 @@ import type { WalkerOS } from '@walkeros/core';
 import type { DestinationAPI } from '.';
 import { startFlow } from '@walkeros/collector';
 import {
-  createEvent,
-  getEvent,
   clone,
+  createEvent,
   createMockContext,
   createMockLogger,
+  getEvent,
   isObject,
 } from '@walkeros/core';
 import { examples } from './dev';
@@ -277,7 +277,10 @@ describe('Destination API', () => {
 
   describe('pushBatch', () => {
     test('sends array of events', () => {
-      const events = [getEvent('product view'), getEvent('product click')];
+      const events = [
+        getEvent('product view'),
+        createEvent({ name: 'product click' }),
+      ];
       const batch = {
         key: 'product view',
         events,
@@ -303,7 +306,7 @@ describe('Destination API', () => {
 
     test('sends entry data when present, raw event otherwise', () => {
       const e0 = getEvent('product view');
-      const e1 = getEvent('product click');
+      const e1 = createEvent({ name: 'product click' });
       const mapped = { custom: 'data1' };
       const batch = {
         key: 'product view',
@@ -329,7 +332,7 @@ describe('Destination API', () => {
 
     test('delivers every entry when some have mapped data and some do not', () => {
       const e0 = getEvent('product view');
-      const e1 = getEvent('product click');
+      const e1 = createEvent({ name: 'product click' });
       const mapped = { mapped: 1 };
       const batch = {
         key: 'product view',
@@ -356,7 +359,10 @@ describe('Destination API', () => {
     });
 
     test('applies transform to each batch item', () => {
-      const events = [getEvent('product view'), getEvent('product click')];
+      const events = [
+        getEvent('product view'),
+        createEvent({ name: 'product click' }),
+      ];
       const batch = {
         key: 'product view',
         events,
@@ -390,7 +396,10 @@ describe('Destination API', () => {
     });
 
     test('sends a JSON array of the transformed bodies, not of strings', () => {
-      const events = [getEvent('product view'), getEvent('product click')];
+      const events = [
+        getEvent('product view'),
+        createEvent({ name: 'product click' }),
+      ];
       const batch = {
         key: 'product view',
         events,
diff --git a/packages/web/destinations/clarity/src/examples/step.ts b/packages/web/destinations/clarity/src/examples/step.ts
index fecfe68fb..625f9c7d5 100644
--- a/packages/web/destinations/clarity/src/examples/step.ts
+++ b/packages/web/destinations/clarity/src/examples/step.ts
@@ -1,5 +1,5 @@
 import type { Flow, WalkerOS } from '@walkeros/core';
-import { getEvent } from '@walkeros/core';
+import { createEvent, getEvent } from '@walkeros/core';
 import type { Settings } from '../types';
 
 /**
@@ -35,7 +35,7 @@ export const defaultEventForwarding: ClarityStepExample = {
  */
 export const wildcardIgnored: ClarityStepExample = {
   public: false,
-  in: getEvent('debug noise', { timestamp: 1700000101 }),
+  in: createEvent({ name: 'debug noise', timestamp: 1700000101 }),
   mapping: { ignore: true },
   out: [],
 };
@@ -50,7 +50,8 @@ export const userLoginIdentify: ClarityStepExample = {
   title: 'User login identify',
   description:
     'A user login fires Clarity.identify with custom id, session id, page id, and friendly name then tracks the event.',
-  in: getEvent('user login', {
+  in: createEvent({
+    name: 'user login',
     timestamp: 1700000102,
     data: { id: 'u-123', name: 'Jane Doe' },
     globals: { pagegroup: 'shop', pagetype: 'home' },
@@ -199,7 +200,7 @@ export const orderCompleteInclude: ClarityStepExample = {
     ['clarity.setTag', 'data_currency', 'EUR'],
     ['clarity.setTag', 'data_shipping', '5.22'],
     ['clarity.setTag', 'data_taxes', '73.76'],
-    ['clarity.setTag', 'data_total', '555'],
+    ['clarity.setTag', 'data_value', '555'],
     ['clarity.event', 'order complete'],
   ],
 };
diff --git a/packages/web/destinations/d8a/README.md b/packages/web/destinations/d8a/README.md
index de4704275..abf17a81b 100644
--- a/packages/web/destinations/d8a/README.md
+++ b/packages/web/destinations/d8a/README.md
@@ -65,7 +65,7 @@ mapping: {
       data: {
         map: {
           transaction_id: 'data.id',
-          value: 'data.total',
+          value: 'data.value',
           currency: { key: 'data.currency', value: 'EUR' },
         },
       },
diff --git a/packages/web/destinations/d8a/src/examples/step.ts b/packages/web/destinations/d8a/src/examples/step.ts
index 501edfa2c..e4af4beee 100644
--- a/packages/web/destinations/d8a/src/examples/step.ts
+++ b/packages/web/destinations/d8a/src/examples/step.ts
@@ -94,7 +94,7 @@ export const purchase: Flow.StepExample = {
     data: {
       map: {
         transaction_id: 'data.id',
-        value: 'data.total',
+        value: 'data.value',
         tax: 'data.taxes',
         shipping: 'data.shipping',
         currency: { key: 'data.currency', value: 'EUR' },
@@ -136,7 +136,7 @@ export const purchase: Flow.StepExample = {
         data_currency: 'EUR',
         data_shipping: 5.22,
         data_taxes: 73.76,
-        data_total: 555,
+        data_value: 555,
         context_shopping: 'complete',
         send_to: PROPERTY_ID,
       },
diff --git a/packages/web/destinations/fullstory/src/examples/step.ts b/packages/web/destinations/fullstory/src/examples/step.ts
index c31601083..b61847383 100644
--- a/packages/web/destinations/fullstory/src/examples/step.ts
+++ b/packages/web/destinations/fullstory/src/examples/step.ts
@@ -1,5 +1,5 @@
 import type { Flow, WalkerOS } from '@walkeros/core';
-import { getEvent } from '@walkeros/core';
+import { createEvent, getEvent } from '@walkeros/core';
 import type { Settings } from '../types';
 
 /**
@@ -29,7 +29,7 @@ export const defaultEventForwarding: FullStoryStepExample = {
  */
 export const wildcardIgnored: FullStoryStepExample = {
   public: false,
-  in: getEvent('debug noise', { timestamp: 1700000101 }),
+  in: createEvent({ name: 'debug noise', timestamp: 1700000101 }),
   mapping: { ignore: true },
   out: [],
 };
@@ -57,7 +57,8 @@ export const userLoginIdentify: FullStoryStepExample = {
   title: 'User login identify',
   description:
     'A user login fires FullStory setIdentity with uid and profile properties before tracking the event.',
-  in: getEvent('user login', {
+  in: createEvent({
+    name: 'user login',
     timestamp: 1700000103,
     data: { id: 'u-123', name: 'Jane Doe', email: 'jane@example.com' },
   }),
@@ -124,7 +125,7 @@ export const setUserProperties: FullStoryStepExample = {
     settings: {
       set: {
         map: {
-          revenue: 'data.total',
+          revenue: 'data.value',
           currency: 'data.currency',
         },
       },
@@ -203,7 +204,8 @@ export const combinedFeatures: FullStoryStepExample = {
  */
 export const silentWithIdentify: FullStoryStepExample = {
   public: false,
-  in: getEvent('user login', {
+  in: createEvent({
+    name: 'user login',
     timestamp: 1700000108,
     data: { id: 'u-123', name: 'Jane Doe' },
   }),
diff --git a/packages/web/destinations/gtag/src/__tests__/ads.test.ts b/packages/web/destinations/gtag/src/__tests__/ads.test.ts
index f9bd72c44..58ada5cde 100644
--- a/packages/web/destinations/gtag/src/__tests__/ads.test.ts
+++ b/packages/web/destinations/gtag/src/__tests__/ads.test.ts
@@ -104,7 +104,7 @@ describe('Google Ads Implementation', () => {
 
   describe('pushAdsEvent', () => {
     const mockEvent = getEvent('order complete', {
-      data: { id: 'order-123', total: 99.99 },
+      data: { id: 'order-123', value: 99.99 },
     });
 
     const settings: AdsSettings = {
diff --git a/packages/web/destinations/gtag/src/__tests__/index.test.ts b/packages/web/destinations/gtag/src/__tests__/index.test.ts
index f0cce954b..35faeca18 100644
--- a/packages/web/destinations/gtag/src/__tests__/index.test.ts
+++ b/packages/web/destinations/gtag/src/__tests__/index.test.ts
@@ -4,6 +4,7 @@ import type { Settings, Rule, Env } from '../types';
 import { type Collector } from '@walkeros/core';
 import {
   clone,
+  createEvent,
   createMockContext,
   createMockLogger,
   getEvent,
@@ -611,7 +612,10 @@ describe('Unified Gtag Destination', () => {
         });
 
         // Call a regular event
-        const event = getEvent('button click', { data: { id: 'test-btn' } });
+        const event = createEvent({
+          name: 'button click',
+          data: { id: 'test-btn' },
+        });
         destination.push(
           event,
           createMockContext({
diff --git a/packages/web/destinations/gtag/src/__tests__/userData.test.ts b/packages/web/destinations/gtag/src/__tests__/userData.test.ts
index 557e5035c..22004b825 100644
--- a/packages/web/destinations/gtag/src/__tests__/userData.test.ts
+++ b/packages/web/destinations/gtag/src/__tests__/userData.test.ts
@@ -19,7 +19,7 @@ describe('resolveUserData', () => {
       const event = getEvent('order complete', {
         data: {
           id: '0rd3r1d',
-          total: 555,
+          value: 555,
           customerEmail: 'buyer@shop.com',
           customerPhone: '+1234567890',
         },
@@ -67,7 +67,7 @@ describe('resolveUserData', () => {
       const event = getEvent('order complete', {
         data: {
           id: '0rd3r1d',
-          total: 555,
+          value: 555,
           city: 'Berlin',
           country: 'DE',
         },
@@ -89,7 +89,7 @@ describe('resolveUserData', () => {
 
     it('should skip fields that resolve to nothing', async () => {
       const event = getEvent('order complete', {
-        data: { id: '0rd3r1d', total: 555, customerEmail: 'buyer@shop.com' },
+        data: { id: '0rd3r1d', value: 555, customerEmail: 'buyer@shop.com' },
       });
 
       const result = await resolveUserData(event, settings, mockCollector);
@@ -99,7 +99,7 @@ describe('resolveUserData', () => {
 
     it('should return undefined when no mapped fields resolve', async () => {
       const event = getEvent('order complete', {
-        data: { id: '0rd3r1d', total: 555 },
+        data: { id: '0rd3r1d', value: 555 },
       });
 
       const result = await resolveUserData(event, settings, mockCollector);
diff --git a/packages/web/destinations/gtag/src/examples/step.ts b/packages/web/destinations/gtag/src/examples/step.ts
index fc45a47da..feb0a0d56 100644
--- a/packages/web/destinations/gtag/src/examples/step.ts
+++ b/packages/web/destinations/gtag/src/examples/step.ts
@@ -82,7 +82,7 @@ export const purchase: Flow.StepExample = {
     data: {
       map: {
         transaction_id: 'data.id',
-        value: 'data.total',
+        value: 'data.value',
         tax: 'data.taxes',
         shipping: 'data.shipping',
         currency: { key: 'data.currency', value: 'EUR' },
@@ -123,7 +123,7 @@ export const purchase: Flow.StepExample = {
         data_currency: 'EUR',
         data_shipping: 5.22,
         data_taxes: 73.76,
-        data_total: 555,
+        data_value: 555,
         context_shopping: 'complete',
         send_to: 'G-XXXXXXXXXX',
       },
@@ -205,7 +205,7 @@ export const googleAdsConversion: Flow.StepExample = {
     },
     data: {
       map: {
-        value: 'data.total',
+        value: 'data.value',
       },
     },
   },
@@ -240,7 +240,7 @@ export const adsEnhancedConversions: Flow.StepExample = {
       currency: 'EUR',
       shipping: 5.22,
       taxes: 73.76,
-      total: 555,
+      value: 555,
       customerEmail: 'buyer@shop.com',
       customerPhone: '+1234567890',
     },
@@ -252,7 +252,7 @@ export const adsEnhancedConversions: Flow.StepExample = {
     },
     data: {
       map: {
-        value: 'data.total',
+        value: 'data.value',
       },
     },
   },
@@ -448,7 +448,7 @@ export const multipleToolsSimultaneous: Flow.StepExample = {
     },
     data: {
       map: {
-        value: 'data.total',
+        value: 'data.value',
         currency: { key: 'data.currency', value: 'EUR' },
       },
     },
@@ -466,7 +466,7 @@ export const multipleToolsSimultaneous: Flow.StepExample = {
         data_currency: 'EUR',
         data_shipping: 5.22,
         data_taxes: 73.76,
-        data_total: 555,
+        data_value: 555,
         send_to: 'G-XXXXXXXXXX',
       },
     ],
@@ -482,7 +482,7 @@ export const multipleToolsSimultaneous: Flow.StepExample = {
         data_currency: 'EUR',
         data_shipping: 5.22,
         data_taxes: 73.76,
-        data_total: 555,
+        data_value: 555,
       },
     ],
     [
@@ -495,7 +495,7 @@ export const multipleToolsSimultaneous: Flow.StepExample = {
         data_currency: 'EUR',
         data_shipping: 5.22,
         data_taxes: 73.76,
-        data_total: 555,
+        data_value: 555,
       },
     ],
   ],
diff --git a/packages/web/destinations/heap/src/examples/step.ts b/packages/web/destinations/heap/src/examples/step.ts
index e0adc13c5..103b6f4e4 100644
--- a/packages/web/destinations/heap/src/examples/step.ts
+++ b/packages/web/destinations/heap/src/examples/step.ts
@@ -1,5 +1,5 @@
 import type { Flow, WalkerOS } from '@walkeros/core';
-import { getEvent } from '@walkeros/core';
+import { createEvent, getEvent } from '@walkeros/core';
 import type { Settings } from '../types';
 
 /**
@@ -39,7 +39,7 @@ export const destinationLevelInclude: HeapStepExample = {
     data: {
       map: {
         order_id: 'data.id',
-        total: 'data.total',
+        total: 'data.value',
         currency: { key: 'data.currency', value: 'EUR' },
       },
     },
@@ -79,7 +79,8 @@ export const userLoginIdentify: HeapStepExample = {
   title: 'User login identify',
   description:
     'A user login identifies the Heap user by email and adds user properties while skipping the track.',
-  in: getEvent('user login', {
+  in: createEvent({
+    name: 'user login',
     timestamp: 1700000103,
     data: {
       email: 'user@example.com',
@@ -112,7 +113,7 @@ export const userLogoutReset: HeapStepExample = {
   title: 'User logout reset',
   description:
     'A user logout calls heap.resetIdentity to clear the identified user from the Heap client.',
-  in: getEvent('user logout', { timestamp: 1700000104 }),
+  in: createEvent({ name: 'user logout', timestamp: 1700000104 }),
   mapping: {
     silent: true,
     settings: {
@@ -134,7 +135,7 @@ export const eventWithUserProperties: HeapStepExample = {
     timestamp: 1700000105,
     data: {
       id: '0rd3r1d',
-      total: 555,
+      value: 555,
       currency: 'EUR',
     },
   }),
@@ -142,7 +143,7 @@ export const eventWithUserProperties: HeapStepExample = {
     settings: {
       userProperties: {
         map: {
-          last_order_value: 'data.total',
+          last_order_value: 'data.value',
           last_order_currency: 'data.currency',
         },
       },
diff --git a/packages/web/destinations/hotjar/src/examples/step.ts b/packages/web/destinations/hotjar/src/examples/step.ts
index a49e89f92..28edf4625 100644
--- a/packages/web/destinations/hotjar/src/examples/step.ts
+++ b/packages/web/destinations/hotjar/src/examples/step.ts
@@ -1,5 +1,5 @@
 import type { Flow } from '@walkeros/core';
-import { getEvent } from '@walkeros/core';
+import { createEvent, getEvent } from '@walkeros/core';
 import type { Settings } from '../types';
 
 /**
@@ -27,7 +27,7 @@ export const defaultEventForwarding: HotjarStepExample = {
  */
 export const wildcardIgnored: HotjarStepExample = {
   public: false,
-  in: getEvent('debug noise', { timestamp: 1700000101 }),
+  in: createEvent({ name: 'debug noise', timestamp: 1700000101 }),
   mapping: { ignore: true },
   out: [],
 };
@@ -55,7 +55,8 @@ export const userLoginIdentify: HotjarStepExample = {
   title: 'User login identify',
   description:
     'A user login fires Hotjar.identify with userId and attributes before firing the event.',
-  in: getEvent('user login', {
+  in: createEvent({
+    name: 'user login',
     timestamp: 1700000103,
     data: { id: 'u-123', email: 'jane@example.com', plan: 'premium' },
   }),
diff --git a/packages/web/destinations/linkedin/src/examples/step.ts b/packages/web/destinations/linkedin/src/examples/step.ts
index 2cc505402..3a036c79a 100644
--- a/packages/web/destinations/linkedin/src/examples/step.ts
+++ b/packages/web/destinations/linkedin/src/examples/step.ts
@@ -1,5 +1,5 @@
 import type { Flow } from '@walkeros/core';
-import { getEvent } from '@walkeros/core';
+import { createEvent, getEvent } from '@walkeros/core';
 import type { Settings } from '../types';
 
 /**
@@ -37,7 +37,7 @@ export const simpleConversionId: LinkedInStepExample = {
   title: 'Simple conversion',
   description:
     'A form submit fires a LinkedIn lintrk track with only a conversion_id from Campaign Manager.',
-  in: getEvent('form submit', { timestamp: 1700000101 }),
+  in: createEvent({ name: 'form submit', timestamp: 1700000101 }),
   mapping: {
     settings: {
       conversion: {
@@ -78,7 +78,7 @@ export const orderCompleteFullConversion: LinkedInStepExample = {
       conversion: {
         map: {
           id: { value: 67890 },
-          value: 'data.total',
+          value: 'data.value',
           currency: { key: 'data.currency', value: 'EUR' },
           eventId: 'id',
         },
@@ -151,7 +151,8 @@ export const demoRequestLead: LinkedInStepExample = {
   title: 'Lead conversion',
   description:
     'A demo request fires a LinkedIn lintrk track for a lead conversion with id and event_id only.',
-  in: getEvent('demo request', {
+  in: createEvent({
+    name: 'demo request',
     timestamp: 1700000104,
     id: '1700000104abcdef',
   }),
@@ -187,7 +188,7 @@ export const demoRequestLead: LinkedInStepExample = {
  */
 export const conversionSilenced: LinkedInStepExample = {
   public: false,
-  in: getEvent('form submit', { timestamp: 1700000105 }),
+  in: createEvent({ name: 'form submit', timestamp: 1700000105 }),
   mapping: {
     silent: true,
     settings: {
@@ -213,7 +214,7 @@ export const conversionSilenced: LinkedInStepExample = {
  */
 export const missingConversionIdIgnored: LinkedInStepExample = {
   public: false,
-  in: getEvent('form submit', { timestamp: 1700000106 }),
+  in: createEvent({ name: 'form submit', timestamp: 1700000106 }),
   mapping: {
     settings: {
       conversion: {
diff --git a/packages/web/destinations/matomo/src/examples/step.ts b/packages/web/destinations/matomo/src/examples/step.ts
index ab73dab24..f51786f5c 100644
--- a/packages/web/destinations/matomo/src/examples/step.ts
+++ b/packages/web/destinations/matomo/src/examples/step.ts
@@ -1,6 +1,6 @@
 import type { Flow } from '@walkeros/core';
 import type { Settings } from '../types';
-import { getEvent } from '@walkeros/core';
+import { createEvent, getEvent } from '@walkeros/core';
 
 /**
  * Matomo step example. The test runner registers the destination with the
@@ -135,7 +135,7 @@ export const ecommerceOrder: StepExample = {
         {
           map: {
             orderId: 'data.id',
-            grandTotal: 'data.total',
+            grandTotal: 'data.value',
             tax: 'data.taxes',
             shipping: 'data.shipping',
           },
@@ -288,7 +288,8 @@ export const siteSearch: StepExample = {
   title: 'Site search',
   description:
     'A search submit fires Matomo trackSiteSearch with the keyword, category, and number of results.',
-  in: getEvent('search submit', {
+  in: createEvent({
+    name: 'search submit',
     timestamp: 1700000405,
     data: { query: 'shoes', category: 'products', resultsCount: 42 },
   }),
diff --git a/packages/web/destinations/matomo/src/index.test.ts b/packages/web/destinations/matomo/src/index.test.ts
index bb0fef63b..67f290b8a 100644
--- a/packages/web/destinations/matomo/src/index.test.ts
+++ b/packages/web/destinations/matomo/src/index.test.ts
@@ -1,7 +1,7 @@
 import type { Logger, WalkerOS } from '@walkeros/core';
 import type { DestinationMatomo } from '.';
 import { startFlow } from '@walkeros/collector';
-import { createMockLogger, getEvent, Level } from '@walkeros/core';
+import { createEvent, createMockLogger, getEvent, Level } from '@walkeros/core';
 import { destinationMatomo } from '.';
 import { MappingSchema, SettingsSchema } from './schemas';
 
@@ -177,7 +177,7 @@ describe('Destination Matomo', () => {
             complete: {
               settings: {
                 goalId: '1',
-                goalValue: 'data.total',
+                goalValue: 'data.value',
                 customDimensions: { '2': 'data.id' },
               },
             },
@@ -246,7 +246,7 @@ describe('Destination Matomo', () => {
         mapping: { user: { login: { name: 'setUserId', data: 'user.id' } } },
       });
 
-      await elb(getEvent('user login'));
+      await elb(createEvent({ name: 'user login' }));
 
       expect(paq).toEqual([['setUserId', 'us3r']]);
     });
@@ -374,7 +374,8 @@ describe('Destination Matomo', () => {
       });
 
       await elb(
-        getEvent('search submit', {
+        createEvent({
+          name: 'search submit',
           data: { query: 'shoes', category: 'products' },
         }),
       );
@@ -395,7 +396,7 @@ describe('Destination Matomo', () => {
               data: { set: [{ value: 'order' }, { value: 'complete' }] },
               settings: {
                 goalId: '1',
-                goalValue: 'data.total',
+                goalValue: 'data.value',
                 customDimensions: { '2': 'data.id' },
               },
             },
@@ -505,7 +506,7 @@ describe('Destination Matomo', () => {
       ).toBe(true);
       expect(
         MappingSchema.safeParse({
-          goalValue: { key: 'data.total' },
+          goalValue: { key: 'data.value' },
           customDimensions: { '2': { map: { size: 'data.size' } } },
         }).success,
       ).toBe(true);
diff --git a/packages/web/destinations/meta/src/examples/step.ts b/packages/web/destinations/meta/src/examples/step.ts
index 794e98ba7..43aa6ecc1 100644
--- a/packages/web/destinations/meta/src/examples/step.ts
+++ b/packages/web/destinations/meta/src/examples/step.ts
@@ -1,5 +1,5 @@
 import type { Flow, WalkerOS } from '@walkeros/core';
-import { getEvent, isObject } from '@walkeros/core';
+import { createEvent, getEvent, isObject } from '@walkeros/core';
 
 /**
  * Destination bootstrap.
@@ -32,7 +32,7 @@ export const purchase: Flow.StepExample = {
     name: 'Purchase',
     data: {
       map: {
-        value: 'data.total',
+        value: 'data.value',
         currency: { value: 'EUR' },
         contents: {
           loop: [
@@ -233,7 +233,8 @@ export const customEventWithTrackCustom: Flow.StepExample = {
   title: 'Custom event',
   description:
     'A video complete event is sent as a Meta Pixel trackCustom call with a custom event name and parameters.',
-  in: getEvent('video complete', {
+  in: createEvent({
+    name: 'video complete',
     timestamp: 1700000005,
     id: '1700000005-gr0up-1',
     data: { video_id: 'v1d30', duration: 120 },
diff --git a/packages/web/destinations/mixpanel/src/examples/step.ts b/packages/web/destinations/mixpanel/src/examples/step.ts
index d081cc969..5093a2f19 100644
--- a/packages/web/destinations/mixpanel/src/examples/step.ts
+++ b/packages/web/destinations/mixpanel/src/examples/step.ts
@@ -1,5 +1,5 @@
 import type { Flow, WalkerOS } from '@walkeros/core';
-import { getEvent } from '@walkeros/core';
+import { createEvent, getEvent } from '@walkeros/core';
 import type { Settings } from '../types';
 
 /**
@@ -33,7 +33,7 @@ export const defaultEventForwarding: MixpanelStepExample = {
  */
 export const wildcardIgnored: MixpanelStepExample = {
   public: false,
-  in: getEvent('debug noise', { timestamp: 1700000101 }),
+  in: createEvent({ name: 'debug noise', timestamp: 1700000101 }),
   mapping: { ignore: true },
   out: [],
 };
@@ -121,7 +121,8 @@ export const userLoginIdentifyAndPeople: MixpanelStepExample = {
   title: 'User login identify',
   description:
     'A user login identifies the user and fires Mixpanel people set, set_once, and increment operations.',
-  in: getEvent('user login', {
+  in: createEvent({
+    name: 'user login',
     timestamp: 1700000105000,
     data: {
       user_id: 'new-user-123',
@@ -195,7 +196,8 @@ export const profileUpdateAllPeopleOperations: MixpanelStepExample = {
   title: 'All people operations',
   description:
     'A profile update exercises the full Mixpanel people vocabulary including set, increment, append, union, and remove.',
-  in: getEvent('profile update', {
+  in: createEvent({
+    name: 'profile update',
     timestamp: 1700000106,
     data: {
       name: 'Jane Doe',
@@ -265,7 +267,7 @@ export const accountDeleteUser: MixpanelStepExample = {
   title: 'Delete user',
   description:
     'An account delete fires Mixpanel people.delete_user to remove the user profile from the project.',
-  in: getEvent('account delete', { timestamp: 1700000107 }),
+  in: createEvent({ name: 'account delete', timestamp: 1700000107 }),
   mapping: {
     silent: true,
     settings: {
@@ -287,7 +289,7 @@ export const userLogoutReset: MixpanelStepExample = {
   title: 'User logout reset',
   description:
     'A user logout calls mixpanel.reset to clear persistence and generate a new anonymous distinct id.',
-  in: getEvent('user logout', { timestamp: 1700000108 }),
+  in: createEvent({ name: 'user logout', timestamp: 1700000108 }),
   mapping: {
     silent: true,
     settings: {
@@ -305,7 +307,8 @@ export const userGroupAssociation: MixpanelStepExample = {
   title: 'Group association',
   description:
     'A user login associates the user to a company group via mixpanel.set_group and fires the default track.',
-  in: getEvent('user login', {
+  in: createEvent({
+    name: 'user login',
     timestamp: 1700000109,
     data: {
       user_id: 'user-456',
@@ -358,7 +361,8 @@ export const companyUpdateGroupProfile: MixpanelStepExample = {
   title: 'Group profile',
   description:
     'A company update sets Mixpanel group profile properties via get_group.set and get_group.set_once.',
-  in: getEvent('company update', {
+  in: createEvent({
+    name: 'company update',
     timestamp: 1700000110,
     data: {
       company_id: 'acme-inc',
diff --git a/packages/web/destinations/optimizely/src/examples/step.ts b/packages/web/destinations/optimizely/src/examples/step.ts
index d9c677700..f702924ab 100644
--- a/packages/web/destinations/optimizely/src/examples/step.ts
+++ b/packages/web/destinations/optimizely/src/examples/step.ts
@@ -1,5 +1,5 @@
 import type { Flow, WalkerOS } from '@walkeros/core';
-import { getEvent } from '@walkeros/core';
+import { createEvent, getEvent } from '@walkeros/core';
 import type { Settings } from '../types';
 
 /**
@@ -51,7 +51,7 @@ export const orderCompleteRevenue: OptimizelyStepExample = {
     timestamp: 1700000102,
     data: {
       revenue_cents: 55500,
-      total: 555,
+      value: 555,
       currency: 'EUR',
       item_count: 3,
     },
@@ -60,7 +60,7 @@ export const orderCompleteRevenue: OptimizelyStepExample = {
     name: 'purchase',
     settings: {
       revenue: 'data.revenue_cents',
-      value: 'data.total',
+      value: 'data.value',
       eventTags: {
         map: {
           currency: 'data.currency',
@@ -86,7 +86,8 @@ export const signupWithAttributes: OptimizelyStepExample = {
   title: 'Signup with attributes',
   description:
     'A user signup sets per-event Optimizely attributes via setAttribute before firing the trackEvent.',
-  in: getEvent('user signup', {
+  in: createEvent({
+    name: 'user signup',
     timestamp: 1700000103,
     data: {
       method: 'google',
@@ -117,7 +118,7 @@ export const signupWithAttributes: OptimizelyStepExample = {
  */
 export const wildcardIgnored: OptimizelyStepExample = {
   public: false,
-  in: getEvent('debug noise', { timestamp: 1700000104 }),
+  in: createEvent({ name: 'debug noise', timestamp: 1700000104 }),
   mapping: { ignore: true },
   out: [],
 };
@@ -130,7 +131,8 @@ export const attributesOnlySilent: OptimizelyStepExample = {
   title: 'Attributes only',
   description:
     'A profile update sets Optimizely user attributes without firing a trackEvent for user enrichment.',
-  in: getEvent('profile update', {
+  in: createEvent({
+    name: 'profile update',
     timestamp: 1700000105,
     data: {
       plan: 'premium',
diff --git a/packages/web/destinations/piano/src/examples/step.ts b/packages/web/destinations/piano/src/examples/step.ts
index cab56cbeb..3203b0db0 100644
--- a/packages/web/destinations/piano/src/examples/step.ts
+++ b/packages/web/destinations/piano/src/examples/step.ts
@@ -57,7 +57,7 @@ export const purchase: Flow.StepExample = {
     data: {
       map: {
         transaction_id: 'data.id',
-        revenue: 'data.total',
+        revenue: 'data.value',
         currency: 'data.currency',
       },
     },
diff --git a/packages/web/destinations/pinterest/src/examples/step.ts b/packages/web/destinations/pinterest/src/examples/step.ts
index 541a43283..bfb47cf67 100644
--- a/packages/web/destinations/pinterest/src/examples/step.ts
+++ b/packages/web/destinations/pinterest/src/examples/step.ts
@@ -1,5 +1,5 @@
 import type { Flow, WalkerOS } from '@walkeros/core';
-import { getEvent } from '@walkeros/core';
+import { createEvent, getEvent } from '@walkeros/core';
 
 /**
  * Output convention:
@@ -28,7 +28,7 @@ export const defaultForward: Flow.StepExample = {
  */
 export const wildcardIgnored: Flow.StepExample = {
   public: false,
-  in: getEvent('debug noise', { timestamp: 1700000101 }),
+  in: createEvent({ name: 'debug noise', timestamp: 1700000101 }),
   mapping: { ignore: true },
   out: [],
 };
@@ -55,7 +55,8 @@ export const siteSearch: Flow.StepExample = {
   title: 'Search',
   description:
     'A site search fires Pinterest search with the search_query field resolved from event data.',
-  in: getEvent('site search', {
+  in: createEvent({
+    name: 'site search',
     timestamp: 1700000103,
     id: 'ev-1700000103',
     data: { query: 'leather jacket' },
@@ -190,7 +191,7 @@ export const orderCompleteCheckout: Flow.StepExample = {
     name: 'checkout',
     data: {
       map: {
-        value: 'data.total',
+        value: 'data.value',
         order_id: 'data.id',
         currency: { key: 'data.currency', value: 'EUR' },
         line_items: {
@@ -248,7 +249,8 @@ export const userLoginLead: Flow.StepExample = {
   title: 'Lead with matching',
   description:
     'A user login fires pintrk set for enhanced matching and then a lead conversion track call.',
-  in: getEvent('user login', {
+  in: createEvent({
+    name: 'user login',
     timestamp: 1700000107,
     id: 'ev-1700000107',
     data: {
@@ -294,7 +296,8 @@ export const identifyOnlySilent: Flow.StepExample = {
   title: 'Identify only',
   description:
     'A user update fires pintrk set for enhanced matching without a track call via mapping silent.',
-  in: getEvent('user update', {
+  in: createEvent({
+    name: 'user update',
     timestamp: 1700000108,
     id: 'ev-1700000108',
     data: {
diff --git a/packages/web/destinations/piwikpro/src/examples/step.ts b/packages/web/destinations/piwikpro/src/examples/step.ts
index 59096b530..77f32ad28 100644
--- a/packages/web/destinations/piwikpro/src/examples/step.ts
+++ b/packages/web/destinations/piwikpro/src/examples/step.ts
@@ -1,5 +1,5 @@
 import type { Flow } from '@walkeros/core';
-import { getEvent } from '@walkeros/core';
+import { createEvent, getEvent } from '@walkeros/core';
 
 /**
  * Destination bootstrap.
@@ -76,7 +76,7 @@ export const ecommerceOrder: Flow.StepExample = {
         {
           map: {
             orderId: 'data.id',
-            grandTotal: 'data.total',
+            grandTotal: 'data.value',
             tax: 'data.taxes',
             shipping: 'data.shipping',
           },
@@ -322,7 +322,7 @@ export const eventWithGoalValue: Flow.StepExample = {
     },
     settings: {
       goalId: 'goal_1',
-      goalValue: 'data.total',
+      goalValue: 'data.value',
     },
   },
   out: [
@@ -335,7 +335,8 @@ export const ecommerceRemoveFromCart: Flow.StepExample = {
   title: 'Remove from cart',
   description:
     'A product remove fires Piwik PRO ecommerceRemoveFromCart with the removed item.',
-  in: getEvent('product remove', {
+  in: createEvent({
+    name: 'product remove',
     timestamp: 1700000308,
     data: {
       id: 'ers',
@@ -385,7 +386,8 @@ export const siteSearch: Flow.StepExample = {
   title: 'Site search',
   description:
     'A search submit fires Piwik PRO trackSiteSearch with the keyword, category, and result count.',
-  in: getEvent('search submit', {
+  in: createEvent({
+    name: 'search submit',
     timestamp: 1700000309,
     data: { query: 'rucksack', category: 'bags', results: 12 },
   }),
@@ -402,7 +404,8 @@ export const linkDownload: Flow.StepExample = {
   title: 'Download link',
   description:
     'A file download fires Piwik PRO trackLink with the file URL and the download link type.',
-  in: getEvent('file download', {
+  in: createEvent({
+    name: 'file download',
     timestamp: 1700000310,
     data: { url: 'https://www.example.com/whitepaper.pdf' },
   }),
@@ -419,7 +422,11 @@ export const ping: Flow.StepExample = {
   title: 'Ping',
   description:
     'A pulse-triggered event fires Piwik PRO ping to extend the time spent on the page.',
-  in: getEvent('page ping', { timestamp: 1700000311, trigger: 'pulse' }),
+  in: createEvent({
+    name: 'page ping',
+    timestamp: 1700000311,
+    trigger: 'pulse',
+  }),
   mapping: {
     name: 'ping',
   },
diff --git a/packages/web/destinations/piwikpro/src/hints.ts b/packages/web/destinations/piwikpro/src/hints.ts
index ce63a4a3b..3943cd942 100644
--- a/packages/web/destinations/piwikpro/src/hints.ts
+++ b/packages/web/destinations/piwikpro/src/hints.ts
@@ -24,7 +24,7 @@ export const hints: Hint.Hints = {
     code: [
       {
         lang: 'json',
-        code: '{ "order": { "complete": { "name": "trackEvent", "data": { "set": ["entity", "action", "data.id"] }, "settings": { "goalId": "goal-uuid", "goalValue": "data.total" } } } }',
+        code: '{ "order": { "complete": { "name": "trackEvent", "data": { "set": ["entity", "action", "data.id"] }, "settings": { "goalId": "goal-uuid", "goalValue": "data.value" } } } }',
       },
     ],
   },
diff --git a/packages/web/destinations/piwikpro/src/index.test.ts b/packages/web/destinations/piwikpro/src/index.test.ts
index 8b0dd429a..d2a990f29 100644
--- a/packages/web/destinations/piwikpro/src/index.test.ts
+++ b/packages/web/destinations/piwikpro/src/index.test.ts
@@ -1,7 +1,13 @@
 import type { Logger, WalkerOS } from '@walkeros/core';
 import type { DestinationPiwikPro } from '.';
 import { startFlow } from '@walkeros/collector';
-import { createMockLogger, getEvent, isObject, Level } from '@walkeros/core';
+import {
+  createEvent,
+  createMockLogger,
+  getEvent,
+  isObject,
+  Level,
+} from '@walkeros/core';
 import { destinationPiwikPro } from '.';
 import { hints } from './hints';
 import { MappingSchema, SettingsSchema, settings } from './schemas';
@@ -175,7 +181,7 @@ describe('Destination PiwikPro', () => {
             complete: {
               settings: {
                 goalId: 'g1',
-                goalValue: 'data.total',
+                goalValue: 'data.value',
                 customDimensions: { '1': 'data.id' },
               },
             },
@@ -212,7 +218,7 @@ describe('Destination PiwikPro', () => {
         mapping: { user: { login: { name: 'setUserId', data: 'user.id' } } },
       });
 
-      await elb(getEvent('user login'));
+      await elb(createEvent({ name: 'user login' }));
 
       expect(paq).toEqual([['setUserId', 'us3r']]);
     });
@@ -224,7 +230,7 @@ describe('Destination PiwikPro', () => {
             complete: {
               name: 'trackEvent',
               data: { set: [{ value: 'order' }, { value: 'complete' }] },
-              settings: { goalId: 'g1', goalValue: 'data.total' },
+              settings: { goalId: 'g1', goalValue: 'data.value' },
             },
           },
         },
@@ -444,7 +450,7 @@ describe('Destination PiwikPro', () => {
         },
       });
 
-      await elb(getEvent('user login'));
+      await elb(createEvent({ name: 'user login' }));
 
       expect(paq).toEqual([['setUserId', 'us3r']]);
     });
@@ -591,7 +597,7 @@ describe('Destination PiwikPro', () => {
         mapping: { user: { login: { name: 'setUserId', data: 'user.id' } } },
       });
 
-      await elb(getEvent('user login'));
+      await elb(createEvent({ name: 'user login' }));
       await elb(pageView);
 
       expect(paq).toEqual([
@@ -633,7 +639,7 @@ describe('Destination PiwikPro', () => {
       expect(MappingSchema.safeParse({ goalValue: 10 }).success).toBe(true);
       expect(
         MappingSchema.safeParse({
-          goalValue: { map: { total: 'data.total' } },
+          goalValue: { map: { total: 'data.value' } },
           customDimensions: { '1': { map: { size: 'data.size' } } },
         }).success,
       ).toBe(true);
@@ -655,7 +661,7 @@ describe('Destination PiwikPro', () => {
     test('mapping accepts a rule without goalId (W8)', () => {
       expect(MappingSchema.safeParse({}).success).toBe(true);
       expect(
-        MappingSchema.safeParse({ goalId: 7, goalValue: 'data.total' }).success,
+        MappingSchema.safeParse({ goalId: 7, goalValue: 'data.value' }).success,
       ).toBe(true);
     });
   });
diff --git a/packages/web/destinations/piwikpro/src/schemas/mapping.ts b/packages/web/destinations/piwikpro/src/schemas/mapping.ts
index 2b0d06e22..ada566431 100644
--- a/packages/web/destinations/piwikpro/src/schemas/mapping.ts
+++ b/packages/web/destinations/piwikpro/src/schemas/mapping.ts
@@ -9,7 +9,7 @@ export const MappingSchema = z.object({
     )
     .optional(),
   goalValue: schemas.ValueSchema.describe(
-    'Goal revenue, resolved from the event (like "data.total"), passed as the trackGoal conversion value',
+    'Goal revenue, resolved from the event (like "data.value"), passed as the trackGoal conversion value',
   ).optional(),
   customDimensions: CustomDimensionsSchema.describe(
     'Custom dimensions for this rule, keyed by bare dimension id (like { "1": "data.size" }). Wins per key over the destination customDimensions.',
diff --git a/packages/web/destinations/piwikpro/src/types/index.ts b/packages/web/destinations/piwikpro/src/types/index.ts
index 72120bdd7..02317bef3 100644
--- a/packages/web/destinations/piwikpro/src/types/index.ts
+++ b/packages/web/destinations/piwikpro/src/types/index.ts
@@ -32,7 +32,7 @@ export type InitSettings = Partial;
 export interface Mapping {
   /** Piwik PRO goal id (UUID or legacy integer); adds a trackGoal hit. */
   goalId?: string | number;
-  /** Goal revenue, resolved from the event (like "data.total"). */
+  /** Goal revenue, resolved from the event (like "data.value"). */
   goalValue?: WalkerOSMapping.Value;
   /** Custom dimensions for this rule; win per key over the destination ones. */
   customDimensions?: CustomDimensions;
diff --git a/packages/web/destinations/plausible/src/examples/step.ts b/packages/web/destinations/plausible/src/examples/step.ts
index 5c1a73d55..7a3329666 100644
--- a/packages/web/destinations/plausible/src/examples/step.ts
+++ b/packages/web/destinations/plausible/src/examples/step.ts
@@ -39,7 +39,7 @@ export const purchase: Flow.StepExample = {
         revenue: {
           map: {
             currency: { value: 'EUR' },
-            amount: 'data.total',
+            amount: 'data.value',
           },
         },
       },
diff --git a/packages/web/destinations/posthog/src/examples/step.ts b/packages/web/destinations/posthog/src/examples/step.ts
index ffe3b5679..38398a133 100644
--- a/packages/web/destinations/posthog/src/examples/step.ts
+++ b/packages/web/destinations/posthog/src/examples/step.ts
@@ -1,5 +1,5 @@
 import type { Flow, WalkerOS } from '@walkeros/core';
-import { getEvent } from '@walkeros/core';
+import { createEvent, getEvent } from '@walkeros/core';
 import type { Settings } from '../types';
 
 /**
@@ -33,7 +33,7 @@ export const defaultEventForwarding: PostHogStepExample = {
  */
 export const wildcardIgnored: PostHogStepExample = {
   public: false,
-  in: getEvent('debug noise', { timestamp: 1700000101 }),
+  in: createEvent({ name: 'debug noise', timestamp: 1700000101 }),
   mapping: { ignore: true },
   out: [],
 };
@@ -128,7 +128,8 @@ export const userLoginIdentify: PostHogStepExample = {
   title: 'User login identify',
   description:
     'A user login fires PostHog identify with $set and $set_once person properties, skipping the capture.',
-  in: getEvent('user login', {
+  in: createEvent({
+    name: 'user login',
     timestamp: 1700000105000,
     data: {
       user_id: 'new-user-123',
@@ -191,7 +192,8 @@ export const profileUpdateSetPersonProperties: PostHogStepExample = {
   title: 'Set person properties',
   description:
     'A profile update calls setPersonProperties to enrich the PostHog profile without changing identity.',
-  in: getEvent('profile update', {
+  in: createEvent({
+    name: 'profile update',
     timestamp: 1700000106,
     data: {
       name: 'Jane Doe',
@@ -234,7 +236,7 @@ export const userLogoutReset: PostHogStepExample = {
   title: 'User logout reset',
   description:
     'A user logout calls posthog.reset to clear the distinct id and generate a new anonymous one.',
-  in: getEvent('user logout', { timestamp: 1700000107 }),
+  in: createEvent({ name: 'user logout', timestamp: 1700000107 }),
   mapping: {
     silent: true,
     settings: {
@@ -255,7 +257,8 @@ export const groupAssignmentWithProperties: PostHogStepExample = {
   title: 'Group assignment',
   description:
     'A company update assigns the user to a PostHog group and sets group properties via posthog.group.',
-  in: getEvent('company update', {
+  in: createEvent({
+    name: 'company update',
     timestamp: 1700000108,
     data: {
       company_id: 'company_123',
@@ -320,7 +323,7 @@ export const orderCompleteWithInclude: PostHogStepExample = {
         data_currency: 'EUR',
         data_shipping: 5.22,
         data_taxes: 73.76,
-        data_total: 555,
+        data_value: 555,
         globals_pagegroup: 'shop',
       },
     ],
diff --git a/packages/web/destinations/segment/src/examples/step.ts b/packages/web/destinations/segment/src/examples/step.ts
index fee70a9e6..7927a60f7 100644
--- a/packages/web/destinations/segment/src/examples/step.ts
+++ b/packages/web/destinations/segment/src/examples/step.ts
@@ -1,5 +1,5 @@
 import type { Flow, WalkerOS } from '@walkeros/core';
-import { getEvent } from '@walkeros/core';
+import { createEvent, getEvent } from '@walkeros/core';
 import type { Settings } from '../types';
 
 /**
@@ -54,7 +54,7 @@ export const defaultEventForwarding: SegmentStepExample = {
  */
 export const wildcardIgnored: SegmentStepExample = {
   public: false,
-  in: getEvent('debug noise', { timestamp: 1700000101 }),
+  in: createEvent({ name: 'debug noise', timestamp: 1700000101 }),
   mapping: { ignore: true },
   out: [],
 };
@@ -143,7 +143,8 @@ export const userLoginIdentify: SegmentStepExample = {
   title: 'User login identify',
   description:
     'A user login fires Segment analytics.identify with userId and reserved traits such as email, name, and plan.',
-  in: getEvent('user login', {
+  in: createEvent({
+    name: 'user login',
     timestamp: 1700000105,
     data: {
       user_id: 'new-user-123',
@@ -199,7 +200,8 @@ export const profileUpdateTraitsOnly: SegmentStepExample = {
   title: 'Profile update',
   description:
     'A profile update calls Segment analytics.identify with traits and no userId to merge traits into the current profile.',
-  in: getEvent('profile update', {
+  in: createEvent({
+    name: 'profile update',
     timestamp: 1700000106,
     data: {
       name: 'Jane Q. Doe',
@@ -245,7 +247,7 @@ export const userLogoutReset: SegmentStepExample = {
   title: 'User logout reset',
   description:
     'A user logout calls analytics.reset to clear userId, anonymousId, and traits then generate a new anonymous id.',
-  in: getEvent('user logout', { timestamp: 1700000107 }),
+  in: createEvent({ name: 'user logout', timestamp: 1700000107 }),
   mapping: {
     silent: true,
     settings: {
@@ -263,7 +265,8 @@ export const companyUpdateGroup: SegmentStepExample = {
   title: 'Group company',
   description:
     'A company update fires Segment analytics.group with a groupId and traits for account-level tracking.',
-  in: getEvent('company update', {
+  in: createEvent({
+    name: 'company update',
     timestamp: 1700000108,
     data: {
       company_id: 'comp-456',
@@ -387,7 +390,7 @@ export const orderCompletedEcommerce: SegmentStepExample = {
         currency: { key: 'data.currency', value: 'EUR' },
         shipping: 'data.shipping',
         tax: 'data.taxes',
-        total: 'data.total',
+        total: 'data.value',
         products: {
           loop: [
             'nested',
diff --git a/packages/web/destinations/snowplow/src/examples/step.ts b/packages/web/destinations/snowplow/src/examples/step.ts
index 9149fa707..87cc3a264 100644
--- a/packages/web/destinations/snowplow/src/examples/step.ts
+++ b/packages/web/destinations/snowplow/src/examples/step.ts
@@ -202,7 +202,7 @@ export const transaction: Flow.StepExample = {
           schema: SCHEMAS.TRANSACTION,
           data: {
             transaction_id: 'data.id',
-            revenue: 'data.total',
+            revenue: 'data.value',
             currency: { key: 'data.currency', value: 'USD' },
             payment_method: { value: 'credit_card' },
             tax: 'data.taxes',
@@ -387,7 +387,7 @@ export const contextLoop: Flow.StepExample = {
           schema: SCHEMAS.TRANSACTION,
           data: {
             transaction_id: 'data.id',
-            revenue: 'data.total',
+            revenue: 'data.value',
             currency: 'data.currency',
           },
         },
diff --git a/packages/web/destinations/snowplow/src/index.test.ts b/packages/web/destinations/snowplow/src/index.test.ts
index 8605c0da8..20e897e9e 100644
--- a/packages/web/destinations/snowplow/src/index.test.ts
+++ b/packages/web/destinations/snowplow/src/index.test.ts
@@ -2,7 +2,7 @@ import type { WalkerOS, Mapping } from '@walkeros/core';
 import type { DestinationSnowplow } from '.';
 import type { DestinationWeb } from '@walkeros/web-core';
 import { startFlow } from '@walkeros/collector';
-import { getEvent, mockEnv } from '@walkeros/core';
+import { createEvent, getEvent, mockEnv } from '@walkeros/core';
 import {
   clearUserData,
   enableAnonymousTracking,
@@ -247,7 +247,7 @@ describe('destination snowplow', () => {
 
   test('event without mapping is skipped', async () => {
     // Test that events without mapping configuration are silently skipped
-    const event = getEvent('custom unmapped');
+    const event = createEvent({ name: 'custom unmapped' });
 
     const destinationWithEnv = {
       ...destination,
@@ -1238,7 +1238,7 @@ describe('destination snowplow', () => {
         ...getEvent('order complete'),
         data: {
           id: 'ORD-123',
-          total: 1048,
+          value: 1048,
           currency: 'USD',
         },
         nested: [
@@ -1269,7 +1269,7 @@ describe('destination snowplow', () => {
                         'iglu:com.snowplowanalytics.snowplow.ecommerce/transaction/jsonschema/1-0-0',
                       data: {
                         transaction_id: 'data.id',
-                        revenue: 'data.total',
+                        revenue: 'data.value',
                         currency: 'data.currency',
                       },
                     },
diff --git a/packages/web/destinations/tiktok/src/examples/step.ts b/packages/web/destinations/tiktok/src/examples/step.ts
index 5c474487b..d3dc70d50 100644
--- a/packages/web/destinations/tiktok/src/examples/step.ts
+++ b/packages/web/destinations/tiktok/src/examples/step.ts
@@ -1,5 +1,5 @@
 import type { Flow, WalkerOS } from '@walkeros/core';
-import { getEvent } from '@walkeros/core';
+import { createEvent, getEvent } from '@walkeros/core';
 import type { Settings } from '../types';
 
 /**
@@ -196,7 +196,8 @@ export const userRegisterCompleteRegistration: TikTokStepExample = {
   title: 'Complete registration',
   description:
     'A user register fires ttq.identify for advanced matching and then tracks CompleteRegistration.',
-  in: getEvent('user register', {
+  in: createEvent({
+    name: 'user register',
     id: '55555555eeeeeeee',
     data: {
       email: 'new@acme.com',
@@ -265,7 +266,7 @@ export const orderCompleteCompletePayment: TikTokStepExample = {
     data: {
       map: {
         content_type: { value: 'product' },
-        value: 'data.total',
+        value: 'data.value',
         currency: { key: 'data.currency', value: 'EUR' },
         order_id: 'data.id',
         contents: {
@@ -329,7 +330,8 @@ export const searchSubmitSearch: TikTokStepExample = {
   title: 'Search',
   description:
     'A search submit fires TikTok Search with the query field mapped from event data.',
-  in: getEvent('search submit', {
+  in: createEvent({
+    name: 'search submit',
     id: '777777770000aaaa',
     data: {
       term: 'hiking backpack',
diff --git a/packages/web/sources/dataLayer/src/examples/mapping.ts b/packages/web/sources/dataLayer/src/examples/mapping.ts
index d32a27ea1..e4b623025 100644
--- a/packages/web/sources/dataLayer/src/examples/mapping.ts
+++ b/packages/web/sources/dataLayer/src/examples/mapping.ts
@@ -46,7 +46,7 @@ export const purchase: Mapping.Rule = {
   data: {
     map: {
       id: 'transaction_id',
-      total: 'value',
+      value: 'value',
       currency: 'currency',
       nested: {
         loop: [
diff --git a/skills/walkeros-create-destination/SKILL.md b/skills/walkeros-create-destination/SKILL.md
index 03fd08545..4699134eb 100644
--- a/skills/walkeros-create-destination/SKILL.md
+++ b/skills/walkeros-create-destination/SKILL.md
@@ -246,7 +246,7 @@ export const purchase: Flow.StepExample = {
     data: {
       map: {
         transaction_id: 'data.id',
-        value: 'data.total',
+        value: 'data.value',
         currency: { key: 'data.currency', value: 'EUR' },
       },
     },
diff --git a/skills/walkeros-create-destination/examples/events.ts b/skills/walkeros-create-destination/examples/events.ts
index b768a9822..e84fc7f3b 100644
--- a/skills/walkeros-create-destination/examples/events.ts
+++ b/skills/walkeros-create-destination/examples/events.ts
@@ -33,7 +33,7 @@ export const events: Record = {
   // Maps to purchaseCall in outputs.ts
   purchase: {
     name: 'order complete',
-    data: { id: 'T-123', total: 99.99 },
+    data: { id: 'T-123', value: 99.99 },
     context: {},
     globals: {},
     custom: {},
diff --git a/skills/walkeros-create-destination/examples/mapping.ts b/skills/walkeros-create-destination/examples/mapping.ts
index bc7336d62..be3d3d0d3 100644
--- a/skills/walkeros-create-destination/examples/mapping.ts
+++ b/skills/walkeros-create-destination/examples/mapping.ts
@@ -23,7 +23,7 @@ export const defaultMapping: Mapping.Rules = {
       data: {
         map: {
           transaction_id: 'data.id',
-          value: 'data.total',
+          value: 'data.value',
           currency: { value: 'USD' },
         },
       },
diff --git a/skills/walkeros-create-source/SKILL.md b/skills/walkeros-create-source/SKILL.md
index 966cb3101..16b11f8d9 100644
--- a/skills/walkeros-create-source/SKILL.md
+++ b/skills/walkeros-create-source/SKILL.md
@@ -362,11 +362,11 @@ export const orderComplete: Flow.StepExample = {
   in: {
     method: 'POST',
     path: '/collect',
-    body: { name: 'order complete', data: { id: 'ORD-123', total: 149.97 } },
+    body: { name: 'order complete', data: { id: 'ORD-123', value: 149.97 } },
   },
   out: {
     name: 'order complete',
-    data: { id: 'ORD-123', total: 149.97 },
+    data: { id: 'ORD-123', value: 149.97 },
     entity: 'order',
     action: 'complete',
   },
diff --git a/skills/walkeros-create-source/examples/mapping.ts b/skills/walkeros-create-source/examples/mapping.ts
index fb0a3a99a..62f105667 100644
--- a/skills/walkeros-create-source/examples/mapping.ts
+++ b/skills/walkeros-create-source/examples/mapping.ts
@@ -29,7 +29,7 @@ export const defaultMapping: Mapping.Rules = {
       data: {
         map: {
           id: 'properties.transaction_id',
-          total: 'properties.value',
+          value: 'properties.value',
           currency: 'properties.currency',
         },
       },
diff --git a/skills/walkeros-create-source/examples/outputs.ts b/skills/walkeros-create-source/examples/outputs.ts
index c7fee77d3..72f9a32e9 100644
--- a/skills/walkeros-create-source/examples/outputs.ts
+++ b/skills/walkeros-create-source/examples/outputs.ts
@@ -23,7 +23,7 @@ export const purchaseEvent: Partial = {
   event: 'order complete',
   data: {
     id: 'T-123',
-    total: 99.99,
+    value: 99.99,
     currency: 'USD',
   },
 };
diff --git a/skills/walkeros-debugging/SKILL.md b/skills/walkeros-debugging/SKILL.md
index d86e81544..581c7163f 100644
--- a/skills/walkeros-debugging/SKILL.md
+++ b/skills/walkeros-debugging/SKILL.md
@@ -125,7 +125,7 @@ mapping: {
 // Event structure
 {
   name: 'order complete',
-  data: { total: 100 },
+  data: { value: 100 },
   nested: [
     { entity: 'product', data: { id: 'P1' } }
   ]
diff --git a/skills/walkeros-mapping-configuration/SKILL.md b/skills/walkeros-mapping-configuration/SKILL.md
index a9577e06b..07530cfed 100644
--- a/skills/walkeros-mapping-configuration/SKILL.md
+++ b/skills/walkeros-mapping-configuration/SKILL.md
@@ -70,7 +70,7 @@ order: {
     data: {
       map: {
         transaction_id: 'data.orderId',
-        value: 'data.total',
+        value: 'data.value',
         currency: 'data.currency',
         items: {
           loop: [
@@ -122,7 +122,7 @@ order: {
       map: {
         content_ids: { fn: (e) => e.nested?.map((n) => n.data.id) ?? [] },
         content_type: { value: 'product' },
-        value: 'data.total',
+        value: 'data.value',
         currency: 'data.currency',
         num_items: { fn: (e) => e.nested?.length ?? 0 },
       },
@@ -167,11 +167,11 @@ order: {
   complete: [
     // High-value orders get extra tracking
     {
-      condition: (e) => (e.data?.total ?? 0) > 500,
+      condition: (e) => (e.data?.value ?? 0) > 500,
       name: 'high_value_purchase',
       data: {
         map: {
-          value: 'data.total',
+          value: 'data.value',
           priority: { value: 'high' },
           notify: { value: true },
         },
@@ -180,7 +180,7 @@ order: {
     // Standard orders
     {
       name: 'purchase',
-      data: { map: { value: 'data.total' } },
+      data: { map: { value: 'data.value' } },
     },
   ],
 }
diff --git a/skills/walkeros-understanding-flow/SKILL.md b/skills/walkeros-understanding-flow/SKILL.md
index 202f99afc..cc21faf0d 100644
--- a/skills/walkeros-understanding-flow/SKILL.md
+++ b/skills/walkeros-understanding-flow/SKILL.md
@@ -473,7 +473,7 @@ ASCII diagram and detailed explanation.
         "purchase": {
           "in": {
             "name": "order complete",
-            "data": { "id": "ORD-123", "total": 149.97 }
+            "data": { "id": "ORD-123", "value": 149.97 }
           },
           "out": [
             "event",
diff --git a/skills/walkeros-understanding-mapping/SKILL.md b/skills/walkeros-understanding-mapping/SKILL.md
index 156b83b2f..fb9ad27f0 100644
--- a/skills/walkeros-understanding-mapping/SKILL.md
+++ b/skills/walkeros-understanding-mapping/SKILL.md
@@ -308,7 +308,7 @@ mapping: {
     complete: {
       policy: {
         'enriched.total_cents': {
-          fn: (event) => Math.round(event.data.total * 100)
+          fn: (event) => Math.round(event.data.value * 100)
         }
       },
       name: 'purchase',
diff --git a/skills/walkeros-using-cli/SKILL.md b/skills/walkeros-using-cli/SKILL.md
index 6a9a9c672..08cbec4d5 100644
--- a/skills/walkeros-using-cli/SKILL.md
+++ b/skills/walkeros-using-cli/SKILL.md
@@ -192,7 +192,7 @@ Target a specific step and provide input as `SourceInput`
 walkeros push flow.json --simulate source.browser --event '{"content":"...","trigger":{"type":"click"}}'
 
 # Simulate a destination step with an event
-walkeros push flow.json --simulate destination.gtag -e '{"entity":"order","action":"complete","data":{"total":149.97}}'
+walkeros push flow.json --simulate destination.gtag -e '{"entity":"order","action":"complete","data":{"value":149.97}}'
 ```
 
 Example output (a destination step prints its matched mapping key and every
diff --git a/skills/walkeros-using-cli/flow-configuration.md b/skills/walkeros-using-cli/flow-configuration.md
index 398b06d2b..532673cab 100644
--- a/skills/walkeros-using-cli/flow-configuration.md
+++ b/skills/walkeros-using-cli/flow-configuration.md
@@ -513,7 +513,7 @@ walkeros bundle config.json --all -o dist/all
                 "data": {
                   "map": {
                     "transaction_id": "data.orderId",
-                    "value": "data.total"
+                    "value": "data.value"
                   }
                 }
               }
diff --git a/skills/walkeros-using-step-examples/SKILL.md b/skills/walkeros-using-step-examples/SKILL.md
index 288c467ac..9ebfb6a60 100644
--- a/skills/walkeros-using-step-examples/SKILL.md
+++ b/skills/walkeros-using-step-examples/SKILL.md
@@ -95,10 +95,10 @@ export const checkoutPost: Flow.StepExample = {
   in: {
     method: 'POST',
     path: '/collect',
-    body: { name: 'order complete', data: { id: 'ORD-123', total: 149.97 } },
+    body: { name: 'order complete', data: { id: 'ORD-123', value: 149.97 } },
   },
   out: [
-    ['elb', { name: 'order complete', data: { id: 'ORD-123', total: 149.97 } }],
+    ['elb', { name: 'order complete', data: { id: 'ORD-123', value: 149.97 } }],
   ],
 };
 ```
@@ -190,7 +190,7 @@ export const purchase: Flow.StepExample = {
   in: getEvent('order complete', { timestamp: 1700000000 }),
   mapping: {
     name: 'Purchase',
-    data: { map: { value: 'data.total', currency: { value: 'EUR' } } },
+    data: { map: { value: 'data.value', currency: { value: 'EUR' } } },
   },
   out: [
     [
@@ -284,7 +284,7 @@ import { getEvent } from '@walkeros/core';
 
 export const purchase: Flow.StepExample = {
   in: getEvent('order complete', { timestamp: 1700000000 }),
-  mapping: { name: 'Purchase', data: { map: { value: 'data.total' } } },
+  mapping: { name: 'Purchase', data: { map: { value: 'data.value' } } },
   out: ['track', 'Purchase', { value: 555 }, { eventID: '1700000000-gr0up-1' }],
 };
 
@@ -424,7 +424,7 @@ export const purchase: Flow.StepExample = {
   description:
     'A completed order mapped to the Meta Pixel Purchase standard event.',
   in: getEvent('order complete', { timestamp: 1700000000 }),
-  mapping: { name: 'Purchase', data: { map: { value: 'data.total' } } },
+  mapping: { name: 'Purchase', data: { map: { value: 'data.value' } } },
   out: [['fbq', 'track', 'Purchase', { value: 555, currency: 'EUR' }]],
 };
 
diff --git a/skills/walkeros-using-transformer-ga4/SKILL.md b/skills/walkeros-using-transformer-ga4/SKILL.md
index 391331f9a..45536bb79 100644
--- a/skills/walkeros-using-transformer-ga4/SKILL.md
+++ b/skills/walkeros-using-transformer-ga4/SKILL.md
@@ -147,7 +147,7 @@ source must populate) is also surfaced as a package hint, so `package_get` on
 Use `extend` to add or override individual fields of a shipped default rule
 without replacing it in full. Use `remove` to strip fields from the output.
 
-The `purchase` default ships `id`, `currency`, `total`, `tax`, `shipping`, and
+The `purchase` default ships `id`, `currency`, `value`, `tax`, `shipping`, and
 `coupon`. To add `affiliation` from a GA4 event parameter and drop `currency`:
 
 ```json
@@ -171,7 +171,7 @@ The `purchase` default ships `id`, `currency`, `total`, `tax`, `shipping`, and
 ```
 
 `extend.data.map` is deep-merged onto the default `data.map`, leaving `id`,
-`total`, `tax`, `shipping`, and `coupon` intact. `remove: ["currency"]` strips
+`value`, `tax`, `shipping`, and `coupon` intact. `remove: ["currency"]` strips
 that field from the final payload. A `null` value in `extend` clears an
 inherited field, for example `"extend": { "data": { "map": { "tax": null } } }`
 drops `tax` from the default `data.map`.
@@ -188,7 +188,7 @@ keyword keeps the existing replace behavior.
       "data": {
         "map": {
           "id": "params.ep.transaction_id",
-          "total": "params.epn.value",
+          "value": "params.epn.value",
           "currency": "params.ep.currency",
           "coupon": "params.ep.promo_code"
         }
diff --git a/website/docs/apps/cli.mdx b/website/docs/apps/cli.mdx
index 526623089..71b16ccaf 100644
--- a/website/docs/apps/cli.mdx
+++ b/website/docs/apps/cli.mdx
@@ -620,7 +620,7 @@ Create `event.json`:
   "name": "order complete",
   "data": {
     "id": "ORD-12345",
-    "total": 299.99,
+    "value": 299.99,
     "currency": "USD"
   },
   "user": {
@@ -729,7 +729,7 @@ The `--event` parameter accepts three formats:
 A simulated destination runs with the mock env its package ships (`examples.env.push`), so the vendor SDK or HTTP call is recorded instead of sent. The output lists, per simulated step, the mapping rule that matched and every recorded vendor call:
 
 
@@ -738,7 +738,7 @@ A simulated destination runs with the mock env its package ships (`examples.env.
 success: true
   destination.pubsub
     mapping: none
-    call PubSub.topic.publishMessage({"data":"{\"name\":\"order complete\",\"data\":{\"id\":\"ORD-1\",\"total\":42},...)
+    call PubSub.topic.publishMessage({"data":"{\"name\":\"order complete\",\"data\":{\"id\":\"ORD-1\",\"value\":42},...)
   Duration: 44779ms
 ```
 
diff --git a/website/docs/destinations/server/aws.mdx b/website/docs/destinations/server/aws.mdx
index 70ffa1af2..627d8c864 100644
--- a/website/docs/destinations/server/aws.mdx
+++ b/website/docs/destinations/server/aws.mdx
@@ -191,7 +191,7 @@ A mapping that keeps the id and a few fields:
           "id": "id",
           "name": "name",
           "order_id": "data.id",
-          "total": "data.total"
+          "total": "data.value"
         }
       }
     }
diff --git a/website/docs/destinations/server/bing.mdx b/website/docs/destinations/server/bing.mdx
index cce924562..46f1f2851 100644
--- a/website/docs/destinations/server/bing.mdx
+++ b/website/docs/destinations/server/bing.mdx
@@ -109,7 +109,7 @@ Bing UET distinguishes two event types. Set the type via `mapping.settings.event
         "map": {
           "customData": {
             "map": {
-              "value": "data.total",
+              "value": "data.value",
               "currency": "data.currency",
               "transactionId": "data.id",
               "pageType": { "value": "purchase" }
diff --git a/website/docs/destinations/server/clickhouse.mdx b/website/docs/destinations/server/clickhouse.mdx
index 890422c71..38323a5c4 100644
--- a/website/docs/destinations/server/clickhouse.mdx
+++ b/website/docs/destinations/server/clickhouse.mdx
@@ -417,7 +417,7 @@ configured in the flow rather than in the destination:
                   "event_id": "id",
                   "event_name": "name",
                   "order_id": "data.id",
-                  "revenue": "data.total",
+                  "revenue": "data.value",
                   "currency": { "key": "data.currency", "value": "EUR" }
                 }
               }
diff --git a/website/docs/destinations/server/datamanager.mdx b/website/docs/destinations/server/datamanager.mdx
index 05c7b0fb3..da2c1cc64 100644
--- a/website/docs/destinations/server/datamanager.mdx
+++ b/website/docs/destinations/server/datamanager.mdx
@@ -146,7 +146,7 @@ API's required `eventLocation.storeId` shape.
       data: {
         map: {
           transactionId: 'data.id',
-          conversionValue: 'data.total',
+          conversionValue: 'data.value',
           currency: 'data.currency',
           storeId: 'data.storeId',
           email: 'user.email', // hashed PII still applies
diff --git a/website/docs/destinations/server/linkedin.mdx b/website/docs/destinations/server/linkedin.mdx
index 172275659..f59fdd6a6 100644
--- a/website/docs/destinations/server/linkedin.mdx
+++ b/website/docs/destinations/server/linkedin.mdx
@@ -113,7 +113,7 @@ The default `conversionRuleId` from settings applies to all events. Override it
         "conversion": {
           "map": {
             "ruleId": { "value": "99887766" },
-            "value": "data.total",
+            "value": "data.value",
             "currency": { "key": "data.currency", "value": "EUR" }
           }
         }
diff --git a/website/docs/destinations/server/piwikpro.mdx b/website/docs/destinations/server/piwikpro.mdx
index 92f94a0e0..8355db8ba 100644
--- a/website/docs/destinations/server/piwikpro.mdx
+++ b/website/docs/destinations/server/piwikpro.mdx
@@ -197,7 +197,7 @@ A page view, a custom event with a goal and an ecommerce order:
           {
             "map": {
               "orderId": "data.id",
-              "grandTotal": "data.total",
+              "grandTotal": "data.value",
               "tax": "data.taxes",
               "shipping": "data.shipping"
             }
@@ -212,7 +212,7 @@ A page view, a custom event with a goal and an ecommerce order:
 
 ### Goals
 
-A rule's `settings.goalId` adds a second hit, `trackGoal`, to the same request. `settings.goalValue` is a mapping value for its revenue, such as `"data.total"`. A page view rule with goal settings still sends its page view. With `silent: true`, the rule sends only the goal hit.
+A rule's `settings.goalId` adds a second hit, `trackGoal`, to the same request. `settings.goalValue` is a mapping value for its revenue, such as `"data.value"`. A page view rule with goal settings still sends its page view. With `silent: true`, the rule sends only the goal hit.
 
 A rule without `name` on an event that is not a page view sends only the goal when it has a `goalId`:
 
@@ -221,7 +221,7 @@ A rule without `name` on an event that is not a page view sends only the goal wh
   "complete": {
     "settings": {
       "goalId": "6a0f4c1e-3b2d-4e8f-a7c9-1d2e3f4a5b6c",
-      "goalValue": "data.total"
+      "goalValue": "data.value"
     }
   }
 }`}
diff --git a/website/docs/destinations/server/reddit.mdx b/website/docs/destinations/server/reddit.mdx
index 0dbd8a916..69b4f14e1 100644
--- a/website/docs/destinations/server/reddit.mdx
+++ b/website/docs/destinations/server/reddit.mdx
@@ -112,7 +112,7 @@ Any non-standard name becomes `{ tracking_type: 'Custom', custom_event_name: ' act
           "complete": {
             "settings": {
               "channel": "#sales",
-              "text": "New order \${data.id}: \${data.total} \${data.currency}"
+              "text": "New order \${data.id}: \${data.value} \${data.currency}"
             }
           }
         },
diff --git a/website/docs/destinations/server/snapchat.mdx b/website/docs/destinations/server/snapchat.mdx
index 18c3a0ecb..e49b0b7aa 100644
--- a/website/docs/destinations/server/snapchat.mdx
+++ b/website/docs/destinations/server/snapchat.mdx
@@ -99,7 +99,7 @@ Map walkerOS events to Snapchat standard event names via each rule's `name` fiel
         "map": {
           "custom_data": {
             "map": {
-              "value": "data.total",
+              "value": "data.value",
               "currency": "data.currency",
               "transaction_id": "data.id"
             }
diff --git a/website/docs/destinations/server/twitter.mdx b/website/docs/destinations/server/twitter.mdx
index bf07535d3..fc9bcc8b4 100644
--- a/website/docs/destinations/server/twitter.mdx
+++ b/website/docs/destinations/server/twitter.mdx
@@ -127,7 +127,7 @@ The default `eventId` from settings applies to all events. Override it per event
     "complete": {
       "settings": {
         "eventId": { "value": "tw-o8z6j-purchase01" },
-        "value": "data.total",
+        "value": "data.value",
         "currency": { "key": "data.currency", "value": "EUR" },
         "number_items": "data.count"
       }
diff --git a/website/docs/destinations/web/d8a.mdx b/website/docs/destinations/web/d8a.mdx
index 2f13bc407..b6e6951ee 100644
--- a/website/docs/destinations/web/d8a.mdx
+++ b/website/docs/destinations/web/d8a.mdx
@@ -100,7 +100,7 @@ same way you would for GA4:
       data: {
         map: {
           transaction_id: 'data.id',
-          value: 'data.total',
+          value: 'data.value',
           currency: { key: 'data.currency', value: 'EUR' },
         },
       },
diff --git a/website/docs/destinations/web/linkedin.mdx b/website/docs/destinations/web/linkedin.mdx
index fe3ae4e7c..d72049014 100644
--- a/website/docs/destinations/web/linkedin.mdx
+++ b/website/docs/destinations/web/linkedin.mdx
@@ -66,7 +66,7 @@ await startFlow({
                 conversion: {
                   map: {
                     id: { value: 67890 },
-                    value: 'data.total',
+                    value: 'data.value',
                     currency: { key: 'data.currency', value: 'EUR' },
                     eventId: 'id',
                   },
diff --git a/website/docs/destinations/web/piwikpro.mdx b/website/docs/destinations/web/piwikpro.mdx
index 2ad7ead7a..252c6038d 100644
--- a/website/docs/destinations/web/piwikpro.mdx
+++ b/website/docs/destinations/web/piwikpro.mdx
@@ -150,7 +150,7 @@ a wildcard and map what you need:
 
 A rule's `settings.goalId` adds a `trackGoal` call after the method hit.
 `settings.goalValue` is a mapping value for its conversion value, such as
-`"data.total"`. A page view rule with goal settings still tracks its page
+`"data.value"`. A page view rule with goal settings still tracks its page
 view. With `silent: true`, the rule sends only the goal, with its dimensions.
 A `silent` rule without a goal sends nothing and logs a skip.
 
@@ -162,7 +162,7 @@ when it has a `goalId`:
   "complete": {
     "settings": {
       "goalId": "6a0f4c1e-3b2d-4e8f-a7c9-1d2e3f4a5b6c",
-      "goalValue": "data.total"
+      "goalValue": "data.value"
     }
   }
 }`}
diff --git a/website/docs/getting-started/flow/step-examples.mdx b/website/docs/getting-started/flow/step-examples.mdx
index 47174e6b0..bdf6ebbca 100644
--- a/website/docs/getting-started/flow/step-examples.mdx
+++ b/website/docs/getting-started/flow/step-examples.mdx
@@ -11,8 +11,8 @@ Add `examples` to any step in your flow config to document what it expects and p
 ```json
 "examples": {
   "purchase": {
-    "in": { "name": "order complete", "data": { "id": "ORD-123", "total": 149.97 } },
-    "mapping": { "name": "purchase", "data": { "map": { "transaction_id": "data.id", "value": "data.total" } } },
+    "in": { "name": "order complete", "data": { "id": "ORD-123", "value": 149.97 } },
+    "mapping": { "name": "purchase", "data": { "map": { "transaction_id": "data.id", "value": "data.value" } } },
     "out": [["gtag", "event", "purchase", { "transaction_id": "ORD-123", "value": 149.97, "send_to": "G-DEMO123456" }]]
   }
 }
@@ -50,14 +50,14 @@ Each example has a human-readable name and two fields:
     "purchase": {
       "in": {
         "name": "order complete",
-        "data": { "id": "ORD-123", "total": 149.97, "currency": "EUR" }
+        "data": { "id": "ORD-123", "value": 149.97, "currency": "EUR" }
       },
       "mapping": {
         "name": "purchase",
         "data": {
           "map": {
             "transaction_id": "data.id",
-            "value": "data.total",
+            "value": "data.value",
             "currency": "data.currency"
           }
         }
diff --git a/website/docs/getting-started/flow/validate.mdx b/website/docs/getting-started/flow/validate.mdx
index 9f2ccb90f..964a7b3af 100644
--- a/website/docs/getting-started/flow/validate.mdx
+++ b/website/docs/getting-started/flow/validate.mdx
@@ -16,7 +16,7 @@ Event shapes live in the top-level `contract` block of flow.json (a sibling of `
 {
   "version": 4,
   "contract": {
-    "web": { "events": { "order": { "complete": { "properties": { "data": { "required": ["total", "currency"] } } } } } }
+    "web": { "events": { "order": { "complete": { "properties": { "data": { "required": ["value", "currency"] } } } } } }
   }
 }
 ```
@@ -35,7 +35,7 @@ Add a `@walkeros/transformer-validate` step, point its `contract` setting at the
       "events": {
         "order": {
           "complete": {
-            "properties": { "data": { "required": ["total", "currency"] } }
+            "properties": { "data": { "required": ["value", "currency"] } }
           }
         }
       }
@@ -108,7 +108,7 @@ The top-level `contract` block is the reference for setting up mappings, `$contr
 
 ## Break it on purpose
 
-Contracts are easiest to trust after watching one fail. Take the contract above (it requires `total` and `currency` on `order complete` data), run the validate step with `mode: "strict"`, and give it an example whose `in` misses `currency` while its `out` still passes the event on. The validator names the exact path and what the step really does:
+Contracts are easiest to trust after watching one fail. Take the contract above (it requires `value` and `currency` on `order complete` data), run the validate step with `mode: "strict"`, and give it an example whose `in` misses `currency` while its `out` still passes the event on. The validator names the exact path and what the step really does:
 
 ) => {
     event.preventDefault();
     window.elb({
       name: 'checkout complete',
-      data: { total, items: itemCount },
+      data: { value, items: itemCount },
     });
   };
 
diff --git a/website/docs/getting-started/quickstart/walkerjs.mdx b/website/docs/getting-started/quickstart/walkerjs.mdx
index 027e29d6f..fe78d7388 100644
--- a/website/docs/getting-started/quickstart/walkerjs.mdx
+++ b/website/docs/getting-started/quickstart/walkerjs.mdx
@@ -297,7 +297,7 @@ Variables for its values, a GA4 Event tag.
 | walkerOS event   | GA4 event       | Parameters                                                                 |
 | ---------------- | --------------- | -------------------------------------------------------------------------- |
 | `product view`   | `view_item`     | `currency`, `value` from `data.price`, `items`                              |
-| `order complete` | `purchase`      | `transaction_id` from `data.id`, `value` from `data.total`, `currency`, `items` |
+| `order complete` | `purchase`      | `transaction_id` from `data.id`, `value` from `data.value`, `currency`, `items` |
 | `session start`  | `session start` | optional: `data.source`, `data.medium`, `data.campaign` under names of your own |
 
 An order confirmation is tagged like any other element, with its products
@@ -306,7 +306,7 @@ inside, and the same items variable reads them:
 
   
"`. The "data": { "map": { "id": "transaction_id", - "total": "value", + "value": "value", "currency": "currency" } } @@ -219,7 +219,7 @@ dataLayer.push({ event: 'purchase', value: 99.99 }); code={`// Old dataLayer.push({ event: 'purchase', value: 99.99 }); // New -elb({ name: 'order complete', data: { total: 99.99 } });`} +elb({ name: 'order complete', data: { value: 99.99 } });`} language="typescript" /> diff --git a/website/docs/transformers/file.mdx b/website/docs/transformers/file.mdx index 3bc131ed6..05f06188c 100644 --- a/website/docs/transformers/file.mdx +++ b/website/docs/transformers/file.mdx @@ -181,8 +181,6 @@ script request never becomes an event: - - ## Next steps - **[Cache](../collector/cache)** - Add response caching before file serving (integrated collector mode) diff --git a/website/docs/transformers/ga4.mdx b/website/docs/transformers/ga4.mdx index 3996eced6..23bd4d3c3 100644 --- a/website/docs/transformers/ga4.mdx +++ b/website/docs/transformers/ga4.mdx @@ -262,8 +262,8 @@ Because decoded events now carry `source.url`, the [fingerprint transformer](/do | `begin_checkout` | `order start` | `currency`, `value`, `coupon` | | `add_shipping_info` | `order shipping` | `currency`, `value`, `tier` | | `add_payment_info` | `order payment` | `currency`, `value`, `type` | -| `purchase` | `order complete` | `id`, `currency`, `total`, `tax`, `shipping`, `coupon` | -| `refund` | `order refund` | `id`, `currency`, `total` | +| `purchase` | `order complete` | `id`, `currency`, `value`, `tax`, `shipping`, `coupon` | +| `refund` | `order refund` | `id`, `currency`, `value` | | `add_to_wishlist` | `wishlist add` | `currency`, `value` | ### List / promotion @@ -344,7 +344,7 @@ User config applies per event name; other events keep their defaults. A rule wit } ``` -`id`, `total`, `tax`, `shipping`, and `coupon` stay as the default defines them. A `null` value in `extend` clears an inherited field, for example `"extend": { "data": { "map": { "tax": null } } }`. +`id`, `value`, `tax`, `shipping`, and `coupon` stay as the default defines them. A `null` value in `extend` clears an inherited field, for example `"extend": { "data": { "map": { "tax": null } } }`. ### Replace a default rule @@ -363,7 +363,7 @@ Without `extend` or `remove`, the whole rule is taken from your config: "data": { "map": { "id": "params.ep.transaction_id", - "total": "params.epn.value", + "value": "params.epn.value", "currency": "params.ep.currency", "coupon": "params.ep.promo_code" } diff --git a/website/scripts/flow-snippets.test.mjs b/website/scripts/flow-snippets.test.mjs index 9fd58f307..25db83276 100644 --- a/website/scripts/flow-snippets.test.mjs +++ b/website/scripts/flow-snippets.test.mjs @@ -285,10 +285,13 @@ describe('docs transform: node shapes', () => { }); test('FlowExample whose example has no out: key omitted', () => { - const e = entry('file-transformer'); - const expected = findStepExample(FLOW, e.pointer, e.example); - assert.equal(expected.out, undefined, 'fixture assumption: no out'); - const tree = transform('\n'); + const e = entry('ga4-purchase'); + const flow = structuredClone(FLOW); + delete findStepExample(flow, e.pointer, e.example).out; + const flowFile = flowFileOf('no-out', flow); + const tree = transform('\n', { + flowFile, + }); const [step] = findAll(tree, isJsx('StepExample')); const parsed = JSON.parse(attr(step.node, 'example').value.value); assert.equal('out' in parsed, false); From c226c2e8419ebf1158e7aa358c9737d462644efc Mon Sep 17 00:00:00 2001 From: alexanderkirtzel Date: Wed, 21 Oct 2026 02:28:11 +0000 Subject: [PATCH 04/41] guards transports --- package-lock.json | 29 +- .../src/examples/flow-complete.manifest.ts | 2 + .../src/telemetry/__tests__/emitter.test.ts | 4 +- packages/collector/src/dev.ts | 11 + .../src/dev/__tests__/conformance.test.ts | 306 ++++ .../__tests__/fixture-package/src/index.ts | 129 ++ .../__tests__/fixture-package/src/source.ts | 28 + .../__tests__/fixture-package/src/store.ts | 50 + .../fixture-package/src/transformer.ts | 38 + .../__tests__/fixture-package/src/vendor.ts | 46 + .../collector/src/dev/__tests__/fixtures.ts | 37 + .../src/dev/__tests__/runExamples.test.ts | 373 +++++ packages/collector/src/dev/boundary.ts | 396 +++++ packages/collector/src/dev/conformance.ts | 1326 +++++++++++++++++ packages/collector/src/dev/runExamples.ts | 930 ++++++++++++ packages/collector/src/timeout.ts | 21 +- packages/collector/tsconfig.json | 3 +- .../eslint/__tests__/await-send.test.mjs | 55 + .../__tests__/no-silent-return.test.mjs | 52 + packages/config/eslint/index.mjs | 20 + packages/config/eslint/rules/await-send.mjs | 148 ++ .../config/eslint/rules/no-silent-return.mjs | 79 + packages/config/jest.config.mjs | 5 +- packages/config/package.json | 8 +- .../tsup/__tests__/serverBanner.test.mjs | 42 + packages/config/tsup/index.mjs | 29 + packages/core/src/__tests__/envelope.test.ts | 77 + packages/core/src/__tests__/stepError.test.ts | 33 +- packages/core/src/envelope.ts | 32 +- packages/core/src/index.ts | 6 +- packages/core/src/schemas/flow.ts | 21 + packages/core/src/stepError.ts | 18 + packages/core/src/timeout.ts | 19 + packages/core/src/types/flow.ts | 25 + packages/core/src/types/send.ts | 7 +- .../core/src/__tests__/sendServer.test.ts | 140 ++ packages/server/core/src/sendServer.ts | 125 +- packages/server/destinations/api/package.json | 4 +- .../api/src/__tests__/stepExamples.test.ts | 2 +- .../destinations/api/src/examples/env.ts | 3 +- .../server/destinations/api/src/index.test.ts | 10 - packages/server/destinations/api/src/index.ts | 11 +- packages/server/destinations/aws/package.json | 4 +- .../bing/src/__tests__/index.test.ts | 10 +- .../bing/src/__tests__/stepExamples.test.ts | 1 - .../destinations/bing/src/examples/env.ts | 1 - packages/server/destinations/bing/src/push.ts | 10 +- .../destinations/clickhouse/package.json | 3 + .../criteo/src/__tests__/hash.test.ts | 2 +- .../criteo/src/__tests__/index.test.ts | 10 +- .../criteo/src/__tests__/stepExamples.test.ts | 1 - .../destinations/criteo/src/examples/env.ts | 1 - .../server/destinations/criteo/src/push.ts | 10 +- .../linkedin/src/__tests__/index.test.ts | 12 +- .../src/__tests__/stepExamples.test.ts | 1 - .../destinations/linkedin/src/examples/env.ts | 1 - .../server/destinations/linkedin/src/push.ts | 10 +- .../meta/src/__tests__/index.test.ts | 19 +- .../meta/src/__tests__/stepExamples.test.ts | 1 - .../destinations/meta/src/examples/env.ts | 1 - packages/server/destinations/meta/src/push.ts | 10 +- .../mparticle/src/__tests__/push.test.ts | 11 +- .../src/__tests__/stepExamples.test.ts | 2 +- .../mparticle/src/examples/env.ts | 1 - .../server/destinations/mparticle/src/push.ts | 10 +- .../pinterest/src/__tests__/index.test.ts | 15 +- .../src/__tests__/stepExamples.test.ts | 1 - .../pinterest/src/examples/env.ts | 1 - .../server/destinations/pinterest/src/push.ts | 10 +- .../piwikpro/src/__tests__/push.test.ts | 41 +- .../src/__tests__/stepExamples.test.ts | 2 +- .../destinations/piwikpro/src/examples/env.ts | 4 +- .../server/destinations/piwikpro/src/send.ts | 16 +- .../reddit/src/__tests__/index.test.ts | 12 +- .../reddit/src/__tests__/stepExamples.test.ts | 1 - .../destinations/reddit/src/examples/env.ts | 1 - .../server/destinations/reddit/src/push.ts | 10 +- .../slack/src/__tests__/stepExamples.test.ts | 2 +- .../destinations/slack/src/examples/env.ts | 3 +- .../server/destinations/slack/src/push.ts | 10 +- .../snapchat/src/__tests__/index.test.ts | 10 +- .../src/__tests__/stepExamples.test.ts | 1 - .../destinations/snapchat/src/examples/env.ts | 1 - .../server/destinations/snapchat/src/push.ts | 10 +- .../tiktok/src/__tests__/index.test.ts | 10 +- .../tiktok/src/__tests__/stepExamples.test.ts | 1 - .../destinations/tiktok/src/examples/env.ts | 1 - .../server/destinations/tiktok/src/push.ts | 10 +- .../twitter/src/__tests__/index.test.ts | 12 +- .../src/__tests__/stepExamples.test.ts | 1 - .../destinations/twitter/src/examples/env.ts | 1 - .../server/destinations/twitter/src/push.ts | 10 +- packages/server/stores/fs/package.json | 1 + packages/server/stores/gcs/package.json | 4 +- packages/server/stores/s3/package.json | 4 +- packages/server/stores/sheets/package.json | 4 +- .../transformers/fingerprint/package.json | 1 + packages/transformers/ga4/package.json | 1 + .../web/core/src/__tests__/loadVendor.test.ts | 132 ++ .../web/core/src/__tests__/sendWeb.test.ts | 339 +++-- packages/web/core/src/index.ts | 1 + packages/web/core/src/loadVendor.ts | 136 ++ packages/web/core/src/sendWeb.ts | 177 ++- packages/web/destinations/gtag/package.json | 3 + 104 files changed, 5275 insertions(+), 557 deletions(-) create mode 100644 packages/collector/src/dev/__tests__/conformance.test.ts create mode 100644 packages/collector/src/dev/__tests__/fixture-package/src/index.ts create mode 100644 packages/collector/src/dev/__tests__/fixture-package/src/source.ts create mode 100644 packages/collector/src/dev/__tests__/fixture-package/src/store.ts create mode 100644 packages/collector/src/dev/__tests__/fixture-package/src/transformer.ts create mode 100644 packages/collector/src/dev/__tests__/fixture-package/src/vendor.ts create mode 100644 packages/collector/src/dev/__tests__/fixtures.ts create mode 100644 packages/collector/src/dev/__tests__/runExamples.test.ts create mode 100644 packages/collector/src/dev/boundary.ts create mode 100644 packages/collector/src/dev/conformance.ts create mode 100644 packages/collector/src/dev/runExamples.ts create mode 100644 packages/config/eslint/__tests__/await-send.test.mjs create mode 100644 packages/config/eslint/__tests__/no-silent-return.test.mjs create mode 100644 packages/config/eslint/rules/await-send.mjs create mode 100644 packages/config/eslint/rules/no-silent-return.mjs create mode 100644 packages/config/tsup/__tests__/serverBanner.test.mjs create mode 100644 packages/core/src/timeout.ts create mode 100644 packages/server/core/src/__tests__/sendServer.test.ts create mode 100644 packages/web/core/src/__tests__/loadVendor.test.ts create mode 100644 packages/web/core/src/loadVendor.ts diff --git a/package-lock.json b/package-lock.json index 8b43da433..613aec29c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -49276,7 +49276,9 @@ "@walkeros/core": "4.8.1", "@walkeros/server-core": "4.8.1" }, - "devDependencies": {} + "devDependencies": { + "@walkeros/collector": "4.8.1" + } }, "packages/server/destinations/aws": { "name": "@walkeros/server-destination-aws", @@ -49296,7 +49298,9 @@ "@walkeros/core": "4.8.1", "@walkeros/server-core": "4.8.1" }, - "devDependencies": {} + "devDependencies": { + "@walkeros/collector": "4.8.1" + } }, "packages/server/destinations/bing": { "name": "@walkeros/server-destination-bing", @@ -49330,6 +49334,9 @@ "@clickhouse/client": "1.23.1", "@walkeros/core": "4.8.1", "@walkeros/server-core": "4.8.1" + }, + "devDependencies": { + "@walkeros/collector": "4.8.1" } }, "packages/server/destinations/criteo": { @@ -50183,6 +50190,7 @@ "@walkeros/core": "4.8.1" }, "devDependencies": { + "@walkeros/collector": "4.8.1", "@walkeros/core": "4.8.1" } }, @@ -50199,7 +50207,9 @@ "dependencies": { "@walkeros/core": "4.8.1" }, - "devDependencies": {} + "devDependencies": { + "@walkeros/collector": "4.8.1" + } }, "packages/server/stores/s3": { "name": "@walkeros/server-store-s3", @@ -50215,7 +50225,9 @@ "@walkeros/core": "4.8.1", "s3mini": "^0.9.1" }, - "devDependencies": {} + "devDependencies": { + "@walkeros/collector": "4.8.1" + } }, "packages/server/stores/sheets": { "name": "@walkeros/server-store-sheets", @@ -50230,7 +50242,9 @@ "dependencies": { "@walkeros/core": "4.8.1" }, - "devDependencies": {} + "devDependencies": { + "@walkeros/collector": "4.8.1" + } }, "packages/server/transformers/bot": { "name": "@walkeros/server-transformer-bot", @@ -50279,6 +50293,7 @@ "@walkeros/server-core": "4.8.1" }, "devDependencies": { + "@walkeros/collector": "4.8.1", "@walkeros/core": "4.8.1", "@walkeros/server-core": "4.8.1" } @@ -50315,6 +50330,7 @@ "@walkeros/core": "4.8.1" }, "devDependencies": { + "@walkeros/collector": "4.8.1", "@walkeros/core": "4.8.1" } }, @@ -50455,6 +50471,9 @@ "dependencies": { "@walkeros/core": "4.8.1", "@walkeros/web-core": "4.8.1" + }, + "devDependencies": { + "@walkeros/collector": "4.8.1" } }, "packages/web/destinations/heap": { diff --git a/packages/cli/src/examples/flow-complete.manifest.ts b/packages/cli/src/examples/flow-complete.manifest.ts index cb8e781cc..9ee3fd6b1 100644 --- a/packages/cli/src/examples/flow-complete.manifest.ts +++ b/packages/cli/src/examples/flow-complete.manifest.ts @@ -1962,6 +1962,8 @@ export const flowCompleteCoverage = { mapping: 'example-mapping', out: 'example-out', command: 'example-command', + outcome: RUNTIME('outcome (a failure example, public: false)'), + respond: RUNTIME('respond (a failure example, public: false)'), } satisfies Coverage, sourceConfig: { settings: 'step-settings', diff --git a/packages/cli/src/telemetry/__tests__/emitter.test.ts b/packages/cli/src/telemetry/__tests__/emitter.test.ts index cfbcd3e6c..df8390cf2 100644 --- a/packages/cli/src/telemetry/__tests__/emitter.test.ts +++ b/packages/cli/src/telemetry/__tests__/emitter.test.ts @@ -16,7 +16,7 @@ const mockSendServer = jest.fn( _url: string, _data?: SendDataValue, _options?: SendServerOptions, - ): Promise => Promise.resolve({ ok: true }), + ): Promise => Promise.resolve({}), ); jest.mock('@walkeros/server-core', () => { @@ -51,7 +51,7 @@ describe('emitter', () => { const p = getConfigPath(); if (existsSync(p)) rmSync(p, { force: true }); mockSendServer.mockClear(); - mockSendServer.mockResolvedValue({ ok: true }); + mockSendServer.mockResolvedValue({}); }); afterEach(() => { diff --git a/packages/collector/src/dev.ts b/packages/collector/src/dev.ts index 2b4ebf896..fef5517dc 100644 --- a/packages/collector/src/dev.ts +++ b/packages/collector/src/dev.ts @@ -3,3 +3,14 @@ import { schemas as coreSchemas, type JSONSchema } from '@walkeros/core/dev'; export const schemas: { settings: JSONSchema } = { settings: coreSchemas.CollectorSchemas.initConfigJsonSchema, }; + +export { runExamples } from './dev/runExamples'; +export type { + DestinationRunOptions, + RunOptions, + SourceRunOptions, + StoreRunOptions, + TransformerRunOptions, +} from './dev/runExamples'; +export { conformance, ConformanceError } from './dev/conformance'; +export type { ConformanceOptions, Profile } from './dev/conformance'; diff --git a/packages/collector/src/dev/__tests__/conformance.test.ts b/packages/collector/src/dev/__tests__/conformance.test.ts new file mode 100644 index 000000000..9c12f23bd --- /dev/null +++ b/packages/collector/src/dev/__tests__/conformance.test.ts @@ -0,0 +1,306 @@ +import type { Flow, Store, Trigger, WalkerOS } from '@walkeros/core'; +import { isObject } from '@walkeros/core'; +import { startFlow } from '../../flow'; +import { conformance } from '../conformance'; +import { fixtures } from './fixtures'; +import { + batching, + conditional, + echoing, + floating, + good, + rawHttps, + swallowing, +} from './fixture-package/src'; +import { dropping, hashing } from './fixture-package/src/transformer'; +import type { Client } from './fixture-package/src/store'; +import { missing, remote } from './fixture-package/src/store'; +import { chatty, forwarding } from './fixture-package/src/source'; +import type { Vendor } from './fixture-package/src/vendor'; +import { vendorGlobal } from './fixture-package/src/vendor'; + +describe('conformance kit', () => { + beforeEach(() => { + jest.useRealTimers(); + }); + + it('passes a destination that reports everything', async () => { + await expect( + conformance('destination', good, { profile: 'http', ...fixtures }), + ).resolves.toBeUndefined(); + }); + + it('fails a conditional send on item 10 only (it passes items 1 to 9)', async () => { + await expect( + conformance('destination', conditional, { profile: 'http', ...fixtures }), + ).rejects.toMatchObject({ + items: [10], + }); + }); + + it('fails a floating send, listing every failed item in one error, item 3 among them', async () => { + await expect( + conformance('destination', floating, { profile: 'http', ...fixtures }), + ).rejects.toThrow(/item 3\b/); + }); + + it('fails a destination that copies the vendor response into its message (item 7)', async () => { + await expect( + conformance('destination', echoing, { profile: 'http', ...fixtures }), + ).rejects.toMatchObject({ + items: [7], + }); + }); + + it('fails a destination whose request escapes the fetch boundary (item 1)', async () => { + await expect( + conformance('destination', rawHttps, { profile: 'http', ...fixtures }), + ).rejects.toMatchObject({ + items: expect.arrayContaining([1]), + }); + }); + + it('passes a batching destination that reports the failed entries (item 9)', async () => { + await expect( + conformance('destination', batching, { profile: 'http', ...fixtures }), + ).resolves.toBeUndefined(); + }); + + it('fails a batching destination that swallows a failed chunk (item 9)', async () => { + await expect( + conformance('destination', swallowing, { profile: 'http', ...fixtures }), + ).rejects.toMatchObject({ items: [9] }); + }); + + it('fails a destination without a silent example on item 11 only', async () => { + const { silent, ...examples } = fixtures.examples; + await expect( + conformance('destination', good, { + profile: 'http', + ...fixtures, + examples, + }), + ).rejects.toMatchObject({ items: [11] }); + }); + + it('fails a destination whose silent example makes its default call (item 11)', async () => { + await expect( + conformance( + 'destination', + { + ...good, + push: async (event, context) => { + await good.push(event, { ...context, rule: undefined }); + }, + }, + { profile: 'http', ...fixtures }, + ), + ).rejects.toMatchObject({ items: [11] }); + }); + + it('fails a silent example that declares no out on item 11 only', async () => { + const { out, ...silent } = fixtures.examples.silent; + await expect( + conformance('destination', good, { + profile: 'http', + ...fixtures, + examples: { ...fixtures.examples, silent }, + }), + ).rejects.toMatchObject({ items: [11] }); + }); + + it('refuses fake timers with a clear message', async () => { + jest.useFakeTimers(); + await expect( + conformance('destination', good, { profile: 'http', ...fixtures }), + ).rejects.toThrow(/real timers/); + jest.useRealTimers(); + }); +}); + +describe('conformance kit, other step kinds', () => { + beforeEach(() => { + jest.useRealTimers(); + }); + + const root = fixtures.root; + const schemaOf = (key: string) => ({ + type: 'object', + properties: { [key]: { type: 'string' } }, + required: [key], + }); + + describe('transformer', () => { + const options = { + profile: 'transformer' as const, + root, + schema: schemaOf('salt'), + settings: { salt: 's' }, + examples: { + hashed: { in: { name: 'page view', user: { id: 'u1' } } }, + missingId: { + public: false, + description: 'An event without a user id cannot be hashed.', + in: { name: 'page view' }, + out: [], + outcome: { status: 'failed' as const, code: 'missing-input' }, + }, + }, + }; + + it('passes a transformer that reports a missing input', async () => { + await expect( + conformance('transformer', hashing, options), + ).resolves.toBeUndefined(); + }); + + it('fails a transformer that drops a stripped event silently (item 10; item 2: its missing-input example does not hold)', async () => { + await expect( + conformance('transformer', dropping, options), + ).rejects.toMatchObject({ items: [2, 10] }); + }); + }); + + describe('store (socket profile)', () => { + const values = new Map(); + const client: Client = { + read: async (key) => values.get(key), + write: async (key, value) => { + values.set(key, value); + }, + remove: async (key) => { + values.delete(key); + }, + }; + const options = { + profile: 'socket' as const, + root, + schema: schemaOf('bucket'), + settings: { bucket: 'b' }, + env: { + push: { client }, + simulation: ['call:client.read', 'call:client.write'], + }, + examples: { + write: { in: { operation: 'set', key: 'a', value: 'v' } }, + read: { in: { operation: 'get', key: 'a' } }, + }, + }; + + it('passes a store that reports a failed read', async () => { + await expect( + conformance('store', remote, options), + ).resolves.toBeUndefined(); + }); + + it('fails a store that turns a failed read into a miss (item 2)', async () => { + await expect( + conformance('store', missing, options), + ).rejects.toMatchObject({ items: [2] }); + }); + }); + + describe('source', () => { + const isEvent = (value: unknown): value is WalkerOS.DeepPartialEvent => + isObject(value) && typeof value.name === 'string'; + const createTrigger: Trigger.CreateFn = async (config) => { + let flow: Trigger.FlowHandle | undefined; + return { + get flow() { + return flow; + }, + trigger: () => async (content) => { + if (!flow) { + const started = await startFlow(config); + flow = { collector: started.collector, elb: started.elb }; + } + if (isEvent(content)) await flow.collector.sources.step.push(content); + }, + }; + }; + const options = { + profile: 'source' as const, + root, + schema: schemaOf('path'), + settings: { path: '/collect' }, + createTrigger, + examples: { + event: { in: { name: 'page view', data: { title: 'Home' } } }, + }, + }; + + it('passes a source that keeps its input out of the logs', async () => { + await expect( + conformance('source', forwarding, options), + ).resolves.toBeUndefined(); + }); + + it('fails a source that logs its input (item 7)', async () => { + await expect( + conformance('source', chatty, options), + ).rejects.toMatchObject({ items: [7] }); + }); + }); +}); + +describe('conformance kit, vendor-global profile', () => { + beforeEach(() => { + jest.useRealTimers(); + }); + + const vendor: Vendor = { + track: () => undefined, + consent: () => undefined, + }; + const event = { name: 'page view', data: { title: 'Home' } }; + const examples: Flow.StepExamples = { + pageView: { in: event, out: [['vendor.track', 'page view', 'docs']] }, + silent: { + public: false, + description: 'A silent rule calls nothing.', + in: event, + mapping: { silent: true }, + out: [], + }, + unreachable: { + public: false, + description: 'The vendor call fails.', + in: event, + respond: { error: 'network' }, + out: [['vendor.track', 'page view', 'docs']], + outcome: { status: 'failed', code: 'transport' }, + }, + revoke: { + command: 'consent', + in: { marketing: false }, + out: [['vendor.consent', false]], + }, + }; + const options = { + profile: 'vendor-global' as const, + root: fixtures.root, + schema: { type: 'object', properties: { site: { type: 'string' } } }, + settings: { site: 'docs' }, + env: { + push: { vendor }, + simulation: ['call:vendor.track', 'call:vendor.consent'], + }, + examples, + }; + + it('passes a vendor destination that passes consent to the vendor', async () => { + await expect( + conformance('destination', vendorGlobal, options), + ).resolves.toBeUndefined(); + }); + + it('fails a consent revoke example that declares no out on item 8 only', async () => { + const { out, ...revoke } = examples.revoke; + await expect( + conformance('destination', vendorGlobal, { + ...options, + examples: { ...examples, revoke }, + }), + ).rejects.toMatchObject({ items: [8] }); + }); +}); diff --git a/packages/collector/src/dev/__tests__/fixture-package/src/index.ts b/packages/collector/src/dev/__tests__/fixture-package/src/index.ts new file mode 100644 index 000000000..2c5459a90 --- /dev/null +++ b/packages/collector/src/dev/__tests__/fixture-package/src/index.ts @@ -0,0 +1,129 @@ +import * as https from 'https'; +import { StepError } from '@walkeros/core'; +import type { Destination } from '@walkeros/core'; + +type Settings = { url?: string }; +type Types = Destination.Types; + +function requireUrl(config: Destination.Config): string { + const url = config.settings?.url; + if (!url) + throw new StepError('Config settings url missing', { code: 'config' }); + return url; +} + +async function post(url: string, body: string): Promise { + const res = await fetch(url, { method: 'POST', body }); + if (!res.ok) + throw new StepError(`HTTP ${res.status}`, { + code: 'transport', + status: res.status, + body: await res.text(), + }); +} + +/** Reports everything: passes every check of the http profile. */ +export const good: Destination.Instance = { + type: 'good', + config: {}, + init: ({ config }) => { + requireUrl(config); + }, + push: async (event, { config, rule }) => { + if (rule?.silent) return; + await post(requireUrl(config), JSON.stringify({ name: event.name })); + }, +}; + +/** The challenge's lens-1 destination: sends identified events only, so a stripped event is lost (item 10). */ +export const conditional: Destination.Instance = { + ...good, + type: 'conditional', + push: async (event, { config, rule }) => { + if (rule?.silent) return; + if (event.user.id) + await post(requireUrl(config), JSON.stringify({ name: event.name })); + }, +}; + +/** Floats its send: the push settles before the request does (items 2 and 3). */ +export const floating: Destination.Instance = { + ...good, + type: 'floating', + push: (event, { config, rule }) => { + if (rule?.silent) return; + post(requireUrl(config), JSON.stringify({ name: event.name })).catch( + () => undefined, + ); + }, +}; + +/** Copies the vendor's response text into its error message (item 7). */ +export const echoing: Destination.Instance = { + ...good, + type: 'echoing', + push: async (event, { config, rule }) => { + if (rule?.silent) return; + const res = await fetch(requireUrl(config), { + method: 'POST', + body: JSON.stringify({ name: event.name }), + }); + if (!res.ok) + throw new StepError(`HTTP ${res.status}: ${await res.text()}`, { + code: 'transport', + status: res.status, + }); + }, +}; + +/** Bypasses the helpers and the global fetch: its request escapes the kit's boundary (item 1). */ +export const rawHttps: Destination.Instance = { + ...good, + type: 'raw-https', + push: async (event, { config, rule }) => { + if (rule?.silent) return; + const body = JSON.stringify({ name: event.name }); + await new Promise((resolve, reject) => { + const req = https.request( + requireUrl(config), + { method: 'POST' }, + (res) => { + res.resume(); + res.on('end', () => resolve()); + }, + ); + req.on('error', (err) => + reject(new StepError(err.message, { code: 'transport' })), + ); + req.end(body); + }); + }, +}; + +/** Sends each batch entry on its own and reports the entries that failed (item 9). */ +export const batching: Destination.Instance = { + ...good, + type: 'batching', + pushBatch: async (batch, { config }) => { + const url = requireUrl(config); + const failed: Destination.BatchFailure[] = []; + for (const [index, entry] of batch.entries.entries()) + await post(url, JSON.stringify({ name: entry.event.name })).catch( + (error: unknown) => failed.push({ index, error }), + ); + if (failed.length) return { failed }; + }, +}; + +/** Swallows a failed batch: every entry counts as delivered (item 9). */ +export const swallowing: Destination.Instance = { + ...good, + type: 'swallowing', + pushBatch: async (batch, { config }) => { + const url = requireUrl(config); + for (const entry of batch.entries) + await post(url, JSON.stringify({ name: entry.event.name })).catch( + () => undefined, + ); + }, +}; diff --git a/packages/collector/src/dev/__tests__/fixture-package/src/source.ts b/packages/collector/src/dev/__tests__/fixture-package/src/source.ts new file mode 100644 index 000000000..5bb16db89 --- /dev/null +++ b/packages/collector/src/dev/__tests__/fixture-package/src/source.ts @@ -0,0 +1,28 @@ +import type { Elb, Source, WalkerOS } from '@walkeros/core'; +import { StepError } from '@walkeros/core'; + +type Push = (content: WalkerOS.DeepPartialEvent) => Promise; +type Types = Source.Types<{ path?: string }, unknown, Push>; + +/** Forwards what it receives to the collector. */ +export const forwarding: Source.Init = async ({ config, env }) => { + if (!config.settings?.path) + throw new StepError('Config settings path missing', { code: 'config' }); + return { + type: 'forwarding', + config, + push: (content: WalkerOS.DeepPartialEvent) => env.push(content), + }; +}; + +/** Logs the input it receives: personal data in a log line (item 7). */ +export const chatty: Source.Init = async (context) => { + const source = await forwarding(context); + return { + ...source, + push: (content: WalkerOS.DeepPartialEvent) => { + context.logger.debug('received', { content }); + return context.env.push(content); + }, + }; +}; diff --git a/packages/collector/src/dev/__tests__/fixture-package/src/store.ts b/packages/collector/src/dev/__tests__/fixture-package/src/store.ts new file mode 100644 index 000000000..3e1d7095f --- /dev/null +++ b/packages/collector/src/dev/__tests__/fixture-package/src/store.ts @@ -0,0 +1,50 @@ +import { StepError } from '@walkeros/core'; +import type { Store } from '@walkeros/core'; + +export interface Client { + read(key: string): Promise; + write(key: string, value: Store.StoreValue): Promise; + remove(key: string): Promise; +} + +type Types = Store.Types<{ bucket?: string }, { client?: Client }>; + +function clientOf(env: { client?: Client }): Client { + if (!env.client) + throw new StepError('Store client missing', { code: 'config' }); + return env.client; +} + +/** Reports a failed read as a StepError, never as a miss. */ +export const remote: Store.Init = ({ config, env }) => { + if (!config.settings?.bucket) + throw new StepError('Config settings bucket missing', { code: 'config' }); + const fail = (error: unknown): never => { + throw new StepError('Store request failed', { + code: 'transport', + cause: error, + }); + }; + return { + type: 'remote', + config, + get: (key) => clientOf(env).read(key).catch(fail), + set: (key, value) => clientOf(env).write(key, value).catch(fail), + delete: (key) => clientOf(env).remove(key).catch(fail), + }; +}; + +/** Turns a failed read into a miss: the failure is silent (item 2). */ +export const missing: Store.Init = async (context) => { + const store = await remote(context); + return { + ...store, + get: async (key) => { + try { + return await store.get(key); + } catch { + return undefined; + } + }, + }; +}; diff --git a/packages/collector/src/dev/__tests__/fixture-package/src/transformer.ts b/packages/collector/src/dev/__tests__/fixture-package/src/transformer.ts new file mode 100644 index 000000000..226d8d846 --- /dev/null +++ b/packages/collector/src/dev/__tests__/fixture-package/src/transformer.ts @@ -0,0 +1,38 @@ +import { StepError } from '@walkeros/core'; +import type { Transformer } from '@walkeros/core'; + +type Types = Transformer.Types<{ salt?: string }>; + +/** Hashes the user id with a required salt; a missing id is reported. */ +export const hashing: Transformer.Init = ({ config }) => { + const salt = config.settings?.salt; + if (!salt) + throw new StepError('Config settings salt missing', { code: 'config' }); + return { + type: 'hashing', + config, + push: (event) => { + const id = event.user?.id; + if (!id) + throw new StepError('Event has no user id to hash', { + code: 'missing-input', + }); + return { + event: { + ...event, + user: { ...event.user, hash: `${salt}:${id.length}` }, + }, + }; + }, + }; +}; + +/** Drops an event without a user id: the drop is silent (item 10). */ +export const dropping: Transformer.Init = async (context) => { + const instance = await hashing(context); + return { + ...instance, + push: (event, pushContext) => + event.user?.id ? instance.push(event, pushContext) : false, + }; +}; diff --git a/packages/collector/src/dev/__tests__/fixture-package/src/vendor.ts b/packages/collector/src/dev/__tests__/fixture-package/src/vendor.ts new file mode 100644 index 000000000..89b77555d --- /dev/null +++ b/packages/collector/src/dev/__tests__/fixture-package/src/vendor.ts @@ -0,0 +1,46 @@ +import { isObject, StepError } from '@walkeros/core'; +import type { Destination } from '@walkeros/core'; + +export interface Vendor { + track(name: string, site?: string): void | Promise; + consent(granted: boolean): void; +} + +type Types = Destination.Types<{ site?: string }, unknown, { vendor?: Vendor }>; + +function isVendor(value: unknown): value is Vendor { + return ( + isObject(value) && + typeof value.track === 'function' && + typeof value.consent === 'function' + ); +} + +function vendorOf(env: { vendor?: Vendor }): Vendor { + if (!env.vendor) + throw new StepError('Vendor not ready', { code: 'vendor-not-ready' }); + return env.vendor; +} + +/** Calls a vendor global; passes consent to it and honours `silent`. */ +export const vendorGlobal: Destination.Instance = { + type: 'vendor', + config: {}, + push: async (event, { config, env, rule }) => { + if (rule?.silent) return; + try { + await vendorOf(env).track(event.name, config.settings?.site); + } catch (error) { + if (error instanceof StepError) throw error; + throw new StepError('Vendor call failed', { + code: 'transport', + cause: error, + }); + } + }, + on: (type, { env, data }) => { + const vendor = env.vendor; + if (type !== 'consent' || !isVendor(vendor) || !isObject(data)) return; + vendor.consent(data.marketing === true); + }, +}; diff --git a/packages/collector/src/dev/__tests__/fixtures.ts b/packages/collector/src/dev/__tests__/fixtures.ts new file mode 100644 index 000000000..4efb1230c --- /dev/null +++ b/packages/collector/src/dev/__tests__/fixtures.ts @@ -0,0 +1,37 @@ +import path from 'path'; +import type { Flow } from '@walkeros/core'; +import type { JSONSchema } from '@walkeros/core/dev'; + +export const schema: JSONSchema = { + type: 'object', + properties: { url: { type: 'string' } }, + required: ['url'], + additionalProperties: false, +}; + +export const examples: Flow.StepExamples = { + pageView: { + in: { name: 'page view', data: { title: 'Home' }, user: { id: 'user-1' } }, + out: [ + [ + 'fetch', + 'https://127.0.0.1:9/e', + { method: 'POST', body: '{"name":"page view"}' }, + ], + ], + }, + silent: { + public: false, + description: 'A silent rule sends nothing: the rule settings say so.', + in: { name: 'page view', data: { title: 'Home' }, user: { id: 'user-1' } }, + mapping: { silent: true }, + out: [], + }, +}; + +export const fixtures = { + schema, + examples, + settings: { url: 'https://127.0.0.1:9/e' }, + root: path.join(__dirname, 'fixture-package'), +}; diff --git a/packages/collector/src/dev/__tests__/runExamples.test.ts b/packages/collector/src/dev/__tests__/runExamples.test.ts new file mode 100644 index 000000000..fa9281d6a --- /dev/null +++ b/packages/collector/src/dev/__tests__/runExamples.test.ts @@ -0,0 +1,373 @@ +import type { + Collector, + Destination, + Elb, + Flow, + Source, + Store, + Transformer, + Trigger, + WalkerOS, +} from '@walkeros/core'; +import { createEvent, getEvent, isObject, StepError } from '@walkeros/core'; +import { startFlow } from '../../flow'; +import { runExamples } from '../runExamples'; +import { fixtures } from './fixtures'; +import { good } from './fixture-package/src'; + +const URL = 'https://127.0.0.1:9/e'; +const fetchOut = (name: string): Flow.StepOut => [ + ['fetch', URL, { method: 'POST', body: JSON.stringify({ name }) }], +]; +const settings = fixtures.settings; + +describe('runExamples', () => { + beforeEach(() => { + jest.useRealTimers(); + }); + + describe('destination', () => { + it("resolves the fixture's pageView example for a destination that sends it", async () => { + await expect( + runExamples('destination', good, fixtures.examples, { settings }), + ).resolves.toBeUndefined(); + }); + + it('rejects an example whose out differs, naming the example', async () => { + await expect( + runExamples( + 'destination', + good, + { + pageView: { ...fixtures.examples.pageView, out: fetchOut('other') }, + }, + { settings }, + ), + ).rejects.toThrow(/example pageView: out differs/); + }); + + it('compares out as a literal (R4): a key whose value is undefined counts as absent, an extra effect does not', async () => { + const event = { name: 'page view', user: { id: 'user-1' } }; + await expect( + runExamples( + 'destination', + good, + { + undefinedKey: { + in: event, + out: [ + [ + 'fetch', + URL, + { + method: 'POST', + body: '{"name":"page view"}', + headers: undefined, + }, + ], + ], + }, + }, + { settings }, + ), + ).resolves.toBeUndefined(); + await expect( + runExamples( + 'destination', + good, + { + twice: { + in: event, + out: [...fetchOut('page view'), ...fetchOut('page view')], + }, + }, + { settings }, + ), + ).rejects.toThrow(/example twice: out differs/); + }); + + it('runs mappings through the destination config: example.mapping ?? config.mapping (R2)', async () => { + const event = getEvent('page view', { user: { id: 'user-1' } }); + const config: Partial = { + mapping: { page: { view: { silent: true } } }, + }; + + // The configured rule applies when the example brings none. + await expect( + runExamples( + 'destination', + good, + { configured: { in: event, out: [] } }, + { settings, config }, + ), + ).resolves.toBeUndefined(); + + // The example's own rule replaces it. + await expect( + runExamples( + 'destination', + good, + { own: { in: event, mapping: {}, out: fetchOut('page view') } }, + { settings, config }, + ), + ).resolves.toBeUndefined(); + }); + + it('looks claimed examples up by their standard event name (R1, R3)', async () => { + const examples: Flow.StepExamples = { + anyKey: { + in: getEvent('page view', { user: { id: 'user-1' } }), + out: fetchOut('page view'), + }, + }; + await expect( + runExamples('destination', good, examples, { + settings, + claims: ['page view'], + }), + ).resolves.toBeUndefined(); + + await expect( + runExamples('destination', good, examples, { + settings, + claims: ['page view', 'order complete'], + }), + ).rejects.toThrow(/claim order complete: no public example/); + }); + + it('rejects a public event example that is not claimed (R3)', async () => { + const examples: Flow.StepExamples = { + pageView: { + in: getEvent('page view', { user: { id: 'user-1' } }), + out: fetchOut('page view'), + }, + productView: { + in: getEvent('product view', { user: { id: 'user-1' } }), + out: fetchOut('product view'), + }, + }; + await expect( + runExamples('destination', good, examples, { + settings, + claims: ['page view'], + }), + ).rejects.toThrow( + /example productView: product view is public but not claimed/, + ); + }); + + it('requires a claimed example to start from the pinned getEvent (R6)', async () => { + await expect( + runExamples( + 'destination', + good, + { + pageView: { + in: createEvent({ name: 'page view', user: { id: 'user-1' } }), + out: fetchOut('page view'), + }, + }, + { settings, claims: ['page view'] }, + ), + ).rejects.toThrow(/does not start from getEvent\('page view'\)/); + }); + + it('rejects an example whose in is neither an event nor a command (R5)', async () => { + await expect( + runExamples( + 'destination', + good, + { odd: { in: 'page view', out: [] } }, + { settings }, + ), + ).rejects.toThrow(/example odd: its in is neither an event/); + }); + + it('answers respond at the boundary and compares the reported outcome', async () => { + const failure: Flow.StepExample = { + public: false, + in: fixtures.examples.pageView.in, + respond: { status: 503, body: { error: 'busy' } }, + out: fetchOut('page view'), + outcome: { status: 'failed', code: 'transport', retryable: true }, + }; + await expect( + runExamples('destination', good, { failure }, { settings }), + ).resolves.toBeUndefined(); + + await expect( + runExamples( + 'destination', + good, + { + failure: { + ...failure, + outcome: { status: 'failed', code: 'config' }, + }, + }, + { settings }, + ), + ).rejects.toThrow( + /example failure: expected failed config, got failed transport/, + ); + + const { outcome, ...undeclared } = failure; + await expect( + runExamples('destination', good, { undeclared }, { settings }), + ).rejects.toThrow( + /example undeclared: failed with transport but declares no outcome/, + ); + }); + + it("records the calls on an examples env's simulation paths", async () => { + type Env = { send: (url: string, body: string) => Promise }; + const viaEnv: Destination.Instance< + Destination.Types<{ url?: string }, unknown, Env> + > = { + type: 'via-env', + config: {}, + push: async (event, { config, env }) => { + await env.send(config.settings?.url ?? '', event.name); + }, + }; + const env = { + push: { send: () => Promise.resolve() }, + simulation: ['call:send'], + }; + await expect( + runExamples( + 'destination', + viaEnv, + { + pageView: { + in: { name: 'page view' }, + out: [['send', URL, 'page view']], + }, + }, + { settings, env }, + ), + ).resolves.toBeUndefined(); + }); + }); + + describe('transformer', () => { + type Types = Transformer.Types<{ drop?: boolean }>; + const dropping: Transformer.Init = ({ config }) => ({ + type: 'dropping', + config, + push: () => { + if (config.settings?.drop) return false; + throw new StepError('Event has no consent id', { + code: 'missing-input', + }); + }, + }); + + it('records what reaches the next step and the reported outcome', async () => { + await expect( + runExamples( + 'transformer', + dropping, + { dropped: { in: { name: 'page view' }, out: [] } }, + { settings: { drop: true } }, + ), + ).resolves.toBeUndefined(); + + await expect( + runExamples( + 'transformer', + dropping, + { + missing: { + public: false, + in: { name: 'page view' }, + out: [], + outcome: { status: 'failed', code: 'missing-input' }, + }, + }, + { settings: {} }, + ), + ).resolves.toBeUndefined(); + }); + }); + + describe('store', () => { + const memory: Store.Init = ({ config }) => { + const values = new Map(); + return { + type: 'memory', + config, + get: (key) => values.get(key), + set: (key, value) => { + values.set(key, value); + }, + delete: (key) => { + values.delete(key); + }, + }; + }; + + it('records each operation with its result', async () => { + await expect( + runExamples('store', memory, { + write: { + in: { operation: 'set', key: 'a', value: 'b' }, + out: [['set', 'a', 'b']], + }, + miss: { + in: { operation: 'get', key: 'missing' }, + out: [['get', 'missing', undefined]], + }, + }), + ).resolves.toBeUndefined(); + }); + }); + + describe('source', () => { + type Push = (content: WalkerOS.DeepPartialEvent) => Promise; + const forwarding: Source.Init< + Source.Types + > = async ({ config, env }) => ({ + type: 'forwarding', + config, + push: (content) => env.push(content), + }); + + const isEvent = (value: unknown): value is WalkerOS.DeepPartialEvent => + isObject(value) && typeof value.name === 'string'; + + const createTrigger: Trigger.CreateFn = async ( + config: Collector.InitConfig, + ) => { + let flow: Trigger.FlowHandle | undefined; + return { + get flow() { + return flow; + }, + trigger: () => async (content) => { + if (!flow) { + const started = await startFlow(config); + flow = { collector: started.collector, elb: started.elb }; + } + if (isEvent(content)) await flow.collector.sources.step.push(content); + }, + }; + }; + + it('feeds the input through the package createTrigger and records what the source emits', async () => { + await expect( + runExamples( + 'source', + forwarding, + { + event: { + in: { name: 'page view' }, + out: [['elb', { name: 'page view' }]], + }, + }, + { createTrigger }, + ), + ).resolves.toBeUndefined(); + }); + }); +}); diff --git a/packages/collector/src/dev/boundary.ts b/packages/collector/src/dev/boundary.ts new file mode 100644 index 000000000..e4d6ca3dc --- /dev/null +++ b/packages/collector/src/dev/boundary.ts @@ -0,0 +1,396 @@ +import type { Flow } from '@walkeros/core'; +import { isObject, isThenable, parseCallPath } from '@walkeros/core'; + +/** + * The network boundary the runner and the conformance kit control: the + * global `fetch`, `navigator.sendBeacon` and `XMLHttpRequest`, for web and + * server alike (`sendServer` and `sendWeb` run on them). Each is replaced for + * the duration of one run and restored after it. Every request is answered + * by the boundary and recorded, so a request that never shows up here + * escaped it (its own `https.request`). + */ + +/** How the boundary answers one request. */ +export interface Answer { + status?: number; + body?: unknown; + /** The status text, which failure answers use to echo a canary. */ + statusText?: string; + /** A network error, or no answer at all. */ + error?: 'network' | 'timeout'; + /** Answer one macrotask late (the floating-send check). */ + late?: boolean; +} + +/** One request the boundary answered. */ +export interface Request { + effect: Flow.StepEffect; + /** The run phase when it was made (`init` or `push`). */ + phase: string; + settled: boolean; +} + +export interface Boundary { + requests: Request[]; + /** The phase recorded with each request; the kit sets `init` during init. */ + phase: string; + /** Requests whose answer is still outstanding. */ + pending(): number; + restore(): void; +} + +type Answers = (index: number) => Answer; + +type BodyMethod = ReturnType< + Extract unknown> +>; + +/** A Response the boundary answers with; jsdom has no fetch API of its own. */ +class KitResponse implements Response { + readonly headers = new Headers(); + readonly redirected = false; + readonly type: ResponseType = 'basic'; + readonly url = ''; + readonly body = null; + readonly bodyUsed = false; + readonly ok: boolean; + + constructor( + private readonly content: string, + readonly status: number, + readonly statusText: string, + ) { + this.ok = status >= 200 && status < 300; + } + + clone(): Response { + return new KitResponse(this.content, this.status, this.statusText); + } + text(): Promise { + return Promise.resolve(this.content); + } + json(): Promise { + return Promise.resolve().then(() => JSON.parse(this.content)); + } + arrayBuffer(): BodyMethod<'arrayBuffer'> { + return Promise.reject(new Error('Not supported by the kit boundary')); + } + blob(): BodyMethod<'blob'> { + return Promise.reject(new Error('Not supported by the kit boundary')); + } + bytes(): BodyMethod<'bytes'> { + return Promise.reject(new Error('Not supported by the kit boundary')); + } + formData(): BodyMethod<'formData'> { + return Promise.reject(new Error('Not supported by the kit boundary')); + } +} + +function bodyText(body: unknown): string { + if (body === undefined) return ''; + return typeof body === 'string' ? body : JSON.stringify(body); +} + +function urlOf(input: string | URL | { url: string }): string { + if (typeof input === 'string') return input; + if (input instanceof URL) return input.href; + return input.url; +} + +/** The request init as recorded: without the abort signal, a transport detail. */ +function recordedInit(init?: RequestInit): Record | undefined { + if (!init) return undefined; + const recorded: Record = {}; + for (const [key, value] of Object.entries(init)) + if (key !== 'signal' && value !== undefined) recorded[key] = value; + return recorded; +} + +const macrotask = () => new Promise((resolve) => setTimeout(resolve, 0)); + +function abortError(): Error { + const error = new Error('The operation was aborted.'); + error.name = 'AbortError'; + return error; +} + +/** + * Replace one global property for a run; returns its restore. A property + * that did not exist is deleted again. + */ +function replace(target: object, key: string, value: unknown): () => void { + const original = Object.getOwnPropertyDescriptor(target, key); + Object.defineProperty(target, key, { + value, + configurable: true, + writable: true, + }); + return () => { + if (original) Object.defineProperty(target, key, original); + else Reflect.deleteProperty(target, key); + }; +} + +/** Install the boundary; `answers(n)` answers the n-th request (0-based). */ +export function installBoundary(answers: Answers = () => ({})): Boundary { + const requests: Request[] = []; + const closers: Array<() => void> = []; + const boundary: Boundary = { + requests, + phase: 'push', + pending: () => requests.filter((request) => !request.settled).length, + restore: () => { + closers.splice(0).forEach((close) => close()); + restores.splice(0).forEach((restore) => restore()); + }, + }; + + const record = (effect: Flow.StepEffect): [Request, Answer] => { + const request: Request = { effect, phase: boundary.phase, settled: false }; + const answer = answers(requests.length); + requests.push(request); + return [request, answer]; + }; + + const kitFetch = ( + input: string | URL | { url: string }, + init?: RequestInit, + ): Promise => { + const initRecord = recordedInit(init); + const [request, answer] = record( + initRecord + ? ['fetch', urlOf(input), initRecord] + : ['fetch', urlOf(input)], + ); + + return new Promise((resolve, reject) => { + const fail = (error: Error) => { + request.settled = true; + reject(error); + }; + init?.signal?.addEventListener('abort', () => fail(abortError())); + // At restore an unanswered request fails, so nothing waits forever. + closers.push(() => { + if (!request.settled) fail(abortError()); + }); + if (answer.error === 'timeout') return; + + const settle = () => { + if (request.settled) return; + if (answer.error === 'network') { + fail(new TypeError('fetch failed')); + return; + } + request.settled = true; + resolve( + new KitResponse( + bodyText(answer.body), + answer.status ?? 200, + answer.statusText ?? '', + ), + ); + }; + if (answer.late) macrotask().then(settle); + else settle(); + }); + }; + + const kitBeacon = (url: string, data?: unknown): boolean => { + const [request, answer] = record(['sendBeacon', url, data]); + request.settled = true; + return !answer.error && (answer.status ?? 200) < 400; + }; + + // Synchronous by default, as sendWeb opens it; the async form answers late. + class KitXMLHttpRequest { + status = 0; + statusText = ''; + response: unknown = ''; + responseText = ''; + readyState = 0; + onload: (() => void) | null = null; + onerror: (() => void) | null = null; + onreadystatechange: (() => void) | null = null; + private method = 'GET'; + private url = ''; + private async = true; + + open(method: string, url: string, async = true) { + this.method = method; + this.url = url; + this.async = async; + this.readyState = 1; + } + setRequestHeader() {} + addEventListener(type: string, listener: () => void) { + if (type === 'load') this.onload = listener; + if (type === 'error') this.onerror = listener; + } + send(body?: unknown) { + const [request, answer] = record([ + 'XMLHttpRequest', + this.method, + this.url, + body, + ]); + const settle = () => { + request.settled = true; + this.readyState = 4; + if (answer.error) { + if (!this.async) throw new Error('NetworkError'); + this.onerror?.(); + return; + } + this.status = answer.status ?? 200; + this.statusText = answer.statusText ?? ''; + this.responseText = bodyText(answer.body); + this.response = this.responseText; + this.onreadystatechange?.(); + this.onload?.(); + }; + if (this.async) macrotask().then(settle); + else settle(); + } + } + + const restores = [replace(globalThis, 'fetch', kitFetch)]; + restores.push(replace(globalThis, 'XMLHttpRequest', KitXMLHttpRequest)); + if (typeof navigator !== 'undefined') + restores.push(replace(navigator, 'sendBeacon', kitBeacon)); + + return boundary; +} + +/** One recorded call on the SDK boundary (an examples env path). */ +export interface EnvCall { + fn: string; + args: unknown[]; + phase: string; + settled: boolean; +} + +/** How the SDK boundary answers a call: pass it on, reject it or delay it. */ +export type EnvAnswer = + | { kind: 'pass' } + | { kind: 'reject'; error: Error } + | { kind: 'hang' } + | { kind: 'late' }; + +export interface EnvBoundary { + env: Record; + calls: EnvCall[]; + phase: string; + pending(): number; +} + +interface Tracked { + rest: string[]; + fn: string; +} + +type Callable = (...args: unknown[]) => unknown; + +function isCallable(value: unknown): value is Callable { + return typeof value === 'function'; +} + +/** + * A view of an examples env whose declared `simulation` paths are the SDK + * boundary: each call on one is recorded and answered as `answer` says (the + * original mock runs, rejects, never settles or settles one macrotask late). + * After a function, a path continues on its result or constructed instance, + * as `observeEnv` reads it. + */ +export function answerEnv( + env: Record, + paths: string[], + answer: (index: number) => EnvAnswer = () => ({ kind: 'pass' }), +): EnvBoundary { + const calls: EnvCall[] = []; + const boundary: EnvBoundary = { + env: {}, + calls, + phase: 'push', + pending: () => calls.filter((call) => !call.settled).length, + }; + + const tracked: Tracked[] = []; + for (const raw of paths) { + const segments = parseCallPath(raw); + if (segments.length) + tracked.push({ rest: segments, fn: segments.join('.') }); + } + + const select = (all: Tracked[], key: string) => { + const leafs: string[] = []; + const deeper: Tracked[] = []; + for (const t of all) { + if (t.rest[0] !== key) continue; + if (t.rest.length === 1) leafs.push(t.fn); + else deeper.push({ rest: t.rest.slice(1), fn: t.fn }); + } + return { leafs, deeper }; + }; + + const leaf = (fn: Callable, name: string): Callable => + function (this: unknown, ...args: unknown[]) { + const call: EnvCall = { + fn: name, + args, + phase: boundary.phase, + settled: false, + }; + const reply = answer(calls.length); + calls.push(call); + const settle = (value: V): V => { + call.settled = true; + return value; + }; + if (reply.kind === 'reject') { + call.settled = true; + return Promise.reject(reply.error); + } + if (reply.kind === 'hang') return new Promise(() => undefined); + if (reply.kind === 'late') + return macrotask().then(() => settle(Reflect.apply(fn, this, args))); + const result: unknown = Reflect.apply(fn, this, args); + if (isThenable(result)) + return Promise.resolve(result).then(settle, (error: unknown) => { + call.settled = true; + throw error; + }); + return settle(result); + }; + + const view = (value: unknown, next: Tracked[]): unknown => { + if (next.length === 0) return value; + if (isThenable(value)) + return Promise.resolve(value).then((resolved) => view(resolved, next)); + if (isCallable(value)) + return new Proxy(value, { + apply: (target, thisArg, args) => + view(Reflect.apply(target, thisArg, args), next), + construct: (target, args, newTarget) => { + const instance: unknown = Reflect.construct(target, args, newTarget); + const viewed = view(instance, next); + return isObject(viewed) ? viewed : {}; + }, + }); + if (isObject(value)) + return new Proxy(value, { + get: (target, key, receiver) => { + const property: unknown = Reflect.get(target, key, receiver); + if (typeof key !== 'string') return property; + const { leafs, deeper } = select(next, key); + if (leafs.length && isCallable(property)) + return leaf(property, leafs[0]); + return view(property, deeper); + }, + }); + return value; + }; + + const viewed = view(env, tracked); + boundary.env = isObject(viewed) ? viewed : env; + return boundary; +} diff --git a/packages/collector/src/dev/conformance.ts b/packages/collector/src/dev/conformance.ts new file mode 100644 index 000000000..87ec6b8cb --- /dev/null +++ b/packages/collector/src/dev/conformance.ts @@ -0,0 +1,1326 @@ +import type { + Collector, + Destination, + Flow, + Source, + Store, + Transformer, + Trigger, + WalkerOS, +} from '@walkeros/core'; +import type { JSONSchema } from '@walkeros/core/dev'; +import { getEvent, isObject, Level, StepError } from '@walkeros/core'; +import type { Answer, EnvAnswer } from './boundary'; +import type { + ExamplesEnv, + LogLine, + Session, + Setup, + Step, + StoreOperation, +} from './runExamples'; +import { + checkDestinationExample, + isEqual, + isEvent, + isStoreOperation, + mappingOf, + openDestination, + openSource, + openStore, + openTransformer, + outcomeOf, + readEnv, + RUN_TIMEOUT, + runExamples, + runStoreOperation, + show, + sourceConfig, + STEP_ID, + timersAreFake, + VENDOR_SCRIPT_MARKER, +} from './runExamples'; + +/** + * The conformance kit: the checks of the step contract (design Q9 plus item + * 11), run on a real collector with the boundary under the kit's control. + * Each package declares one profile; the profile, not the package, decides + * which checks run, so no package skips a check of its own. + */ + +/** The boundary a package's step talks to, which decides its checks. */ +export type Profile = + | 'http' + | 'sdk-env' + | 'vendor-global' + | 'socket' + | 'transformer' + | 'source'; + +export interface ConformanceOptions { + profile: Profile; + examples: Flow.StepExamples; + /** The settings schema (JSON Schema): its properties and `required`. */ + schema: JSONSchema; + /** Minimal valid settings, pure JSON. */ + settings: Record; + /** The package directory; item 6 scans `/src`. */ + root: string; + /** The examples env (`examples.env`) where the boundary is the SDK client. */ + env?: Record; + /** Settings forwarded into an SDK as a whole (read, never named in src). */ + passThrough?: string[]; + /** Sources: the package's `createTrigger`, which feeds the examples. */ + createTrigger?: Trigger.CreateFn; +} + +/** Every failed check of one conformance run: `items` lists their numbers. */ +export class ConformanceError extends Error { + items: number[]; + + constructor(failures: Array<[number, string]>) { + super( + failures.map(([item, problem]) => `item ${item}: ${problem}`).join('; '), + ); + this.name = 'ConformanceError'; + this.items = failures.map(([item]) => item); + Object.setPrototypeOf(this, new.target.prototype); + } +} + +/** Checks per profile (design Q9 table plus the two named profiles). */ +const PROFILE_ITEMS: Record = { + http: [1, 2, 3, 4, 5, 6, 7, 9, 10], + 'sdk-env': [1, 2, 3, 4, 5, 6, 7, 9, 10], + 'vendor-global': [1, 2, 4, 6, 7, 8, 10], + socket: [1, 2, 3, 4, 5, 6, 7, 9], + transformer: [1, 2, 4, 5, 6, 7, 10], + source: [1, 4, 5, 6, 7], +}; + +/** The value item 7 plants in an event and the boundary echoes on failure. */ +const CANARY = 'kit-canary-5f9e2c1a'; + +/** Failure codes that mean the step did not start with its settings. */ +const NOT_STARTED = ['config', 'not-ready']; + +type Problem = string | undefined; + +/** A success event example: an event, no declared outcome or failure answer. */ +interface EventExample { + name: string; + example: Flow.StepExample; + event: WalkerOS.DeepPartialEvent; +} + +interface Context { + profile: Profile; + options: ConformanceOptions; + env?: ExamplesEnv; + /** Env the run uses: the examples env except where the profile has none. */ + runEnv?: ExamplesEnv; + schemaKeys: string[]; + required: string[]; + events: EventExample[]; + /** The first success event example that sends something. */ + primary?: EventExample; +} + +/** Whether an example's rule skips the default call (`silent`, `ignore`). */ +const quiet = (example: Flow.StepExample) => + isObject(example.mapping) && + (example.mapping.silent === true || example.mapping.ignore === true); + +/** Whether an example sends something: its declared out, else its rule. */ +const sends = (example: Flow.StepExample) => + Array.isArray(example.out) ? example.out.length > 0 : !quiet(example); + +function contextOf(options: ConformanceOptions): Context { + const env = readEnv(options.env); + const properties = options.schema.properties; + const required = options.schema.required; + const events: EventExample[] = []; + for (const [name, example] of Object.entries(options.examples)) { + if (example.command || example.outcome || example.respond) continue; + if (isEvent(example.in)) events.push({ name, example, event: example.in }); + } + const primary = events.find(({ example }) => sends(example)); + return { + profile: options.profile, + options, + env, + runEnv: options.profile === 'http' ? undefined : env, + schemaKeys: isObject(properties) ? Object.keys(properties) : [], + required: Array.isArray(required) + ? required.filter((key): key is string => typeof key === 'string') + : [], + events, + primary: primary ?? events[0], + }; +} + +const errorsIn = (logs: LogLine[]) => + logs.filter((line) => line.level === Level.ERROR).length; + +function textOf(value: unknown): string { + if (typeof value === 'string') return value; + if (value instanceof Error) return value.message; + try { + return JSON.stringify(value) ?? ''; + } catch { + return String(value); + } +} + +/** Escape a key for a word-boundary match. */ +function wordIn(text: string, word: string): boolean { + const escaped = word.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + return new RegExp(`\\b${escaped}\\b`).test(text); +} + +function stepError(error: unknown): StepError | undefined { + return error instanceof StepError ? error : undefined; +} + +/** Problems of what init threw with `settings: {}` (item 4). */ +function notReadyProblem(ctx: Context, thrown: unknown[]): Problem { + if (!ctx.required.length) { + const failed = thrown.map(stepError).find((error) => error); + return failed && NOT_STARTED.includes(failed.code) + ? `fails ${failed.code} with settings {} although the schema requires nothing` + : undefined; + } + if (!thrown.length) + return `init does not throw with settings {}: it must throw a StepError naming ${ctx.required.join(', ')}`; + const error = stepError(thrown[0]); + if (!error) + return `init throws a plain error with settings {}: throw a StepError with code config naming ${ctx.required.join(', ')}`; + if (error.code !== 'config') + return `init throws code ${error.code} with settings {}: the code is config`; + const named = ctx.schemaKeys.filter((key) => wordIn(error.message, key)); + const missing = ctx.required.filter((key) => !named.includes(key)); + const extra = named.filter((key) => !ctx.required.includes(key)); + if (missing.length) + return `the init error does not name the required ${missing.join(', ')}`; + if (extra.length) + return `the init error names ${extra.join(', ')}, which the schema does not require`; + return undefined; +} + +/** The failure code a run reports for the step; a queued event has none. */ +function failedCode(step: Step): string | undefined { + const outcome = outcomeOf(step.result); + return outcome?.status === 'failed' ? outcome.code : undefined; +} + +/** A copy of the event with the canary in its data, user and globals. */ +function withCanary( + event: WalkerOS.DeepPartialEvent, +): WalkerOS.DeepPartialEvent { + return { + ...event, + data: { ...event.data, kitCanary: CANARY }, + user: { ...event.user, kitCanary: CANARY }, + globals: { ...event.globals, kitCanary: CANARY }, + }; +} + +/** Where the canary leaked: a log line or the reported error text. */ +function leaks(logs: LogLine[], step?: Step): string[] { + const found: string[] = []; + for (const line of logs) + if (line.message.includes(CANARY) || textOf(line.context).includes(CANARY)) + found.push(`the log line "${line.message.slice(0, 80)}"`); + if (step) { + const { result } = step; + const texts = [ + result.error, + result.reason?.message, + ...Object.values(result.failed ?? {}).flatMap((ref) => [ + ref.reason?.message, + textOf(ref.error), + ]), + ]; + if (texts.some((text) => typeof text === 'string' && text.includes(CANARY))) + found.push('the reported error message'); + } + return found; +} + +/** Item 6, static half: settings keys no source file mentions. */ +async function unreadKeys(ctx: Context): Promise { + const { readdir, readFile } = await import('node:fs/promises'); + const { join } = await import('node:path'); + const skipDirs = ['schemas', 'examples', '__tests__', '__mocks__']; + const skipFiles = ['schemas.ts', 'hints.ts', 'dev.ts']; + + async function text(dir: string): Promise { + let content = ''; + for (const entry of await readdir(dir, { withFileTypes: true })) { + const path = join(dir, entry.name); + if (entry.isDirectory()) { + if (!skipDirs.includes(entry.name)) content += await text(path); + } else if ( + entry.name.endsWith('.ts') && + !entry.name.endsWith('.test.ts') && + !skipFiles.includes(entry.name) + ) + content += await readFile(path, 'utf8'); + } + return content; + } + + const source = await text(join(ctx.options.root, 'src')); + const passThrough = ctx.options.passThrough ?? []; + return ctx.schemaKeys.filter( + (key) => !passThrough.includes(key) && !wordIn(source, key), + ); +} + +/** Item 6, both halves, from the keys the runs read. */ +async function settingsProblem( + ctx: Context, + reads: Set, +): Promise { + const problems: string[] = []; + const unread = await unreadKeys(ctx); + if (unread.length) + problems.push(`no source reads the schema key(s) ${unread.join(', ')}`); + const unknown = [...reads].filter((key) => !ctx.schemaKeys.includes(key)); + if (unknown.length) + problems.push(`reads ${unknown.join(', ')}, which the schema lacks`); + const forwarded = (ctx.options.passThrough ?? []).filter( + (key) => !reads.has(key), + ); + if (forwarded.length) + problems.push( + `passThrough ${forwarded.join(', ')} is never forwarded in an example`, + ); + return problems.length ? problems.join(', ') : undefined; +} + +function pureJson(settings: Record): boolean { + try { + return isEqual(JSON.parse(JSON.stringify(settings)), settings); + } catch { + return false; + } +} + +/** The failures item 2 answers at the fetch boundary, one run each. */ +const failureAnswers: Array<{ label: string; answer: Answer }> = [ + { label: 'HTTP 500', answer: { status: 500, body: { error: 'busy' } } }, + { label: 'a network error', answer: { error: 'network' } }, + { label: 'a timeout', answer: { error: 'timeout' } }, +]; + +/** The failures item 2 answers at the SDK boundary, one run each. */ +const failureEnvAnswers: Array<{ label: string; answer: EnvAnswer }> = [ + { + label: 'a rejected SDK call', + answer: { kind: 'reject', error: new Error('Connection refused') }, + }, + { label: 'an SDK call that never settles', answer: { kind: 'hang' } }, +]; + +/* ------------------------------------------------------------------------ */ +/* Destinations */ +/* ------------------------------------------------------------------------ */ + +type DestinationStep = Destination.Instance; + +interface DestinationRun extends Step { + initCalls: number; + scripts: { marked: number; unmarked: number }; + dlq: WalkerOS.Event[]; + status?: Collector.DestinationStatus; +} + +/** Open, run one event, collect what the kit reads, close. */ +async function runDestination( + step: DestinationStep, + ctx: Context, + target: EventExample, + setup: Setup, + event: WalkerOS.DeepPartialEvent = target.event, + ingest?: Record, +): Promise { + const mapping = mappingOf(target.example, { event: target.event }); + const session = await openDestination(step, { + env: ctx.runEnv, + ...setup, + config: { + timeout: RUN_TIMEOUT, + ...(mapping ? { mapping } : {}), + ...setup.config, + }, + }); + try { + const run = await session.run({ event, ingest }); + return { + ...run, + initCalls: session.initCalls(), + scripts: session.scripts(), + dlq: (session.collector.destinations[STEP_ID]?.dlq ?? []).map( + ([queued]) => queued, + ), + status: session.collector.status.destinations[STEP_ID], + }; + } finally { + await session.close(); + } +} + +/** One problem per failure answer that is not reported as the contract says. */ +function reportProblem(label: string, run: DestinationRun): Problem { + const reason = run.result.failed?.[STEP_ID]?.reason; + if (!reason || !('retryable' in reason)) + return `${label} is not reported (no failed reason)`; + if (label === 'HTTP 500' && reason.status !== 500) + return `${label} is reported without its status`; + if (!reason.retryable) return `${label} is reported as not retryable`; + if (!run.dlq.length) return `${label} leaves the DLQ empty`; + const errors = errorsIn(run.logs); + if (errors !== 1) + return `${label} writes ${errors} ERROR lines instead of the collector's one`; + return undefined; +} + +async function destinationItem( + item: number, + step: DestinationStep, + ctx: Context, +): Promise { + const { options, primary } = ctx; + const base: Setup = { settings: options.settings }; + + switch (item) { + case 1: { + if (!pureJson(options.settings)) return 'settings are not pure JSON'; + if (!ctx.events.length) return 'no success event example to run'; + const problems: string[] = []; + for (const target of ctx.events) { + const run = await runDestination(step, ctx, target, { + ...base, + env: undefined, + }); + const code = failedCode(run); + if (ctx.profile === 'http') { + if (code) + problems.push(`example ${target.name} fails ${code} without env`); + if (sends(target.example) && !run.effects.length) + problems.push( + `example ${target.name} sends no request through fetch, sendBeacon or XMLHttpRequest`, + ); + } else if (code && NOT_STARTED.includes(code)) + problems.push(`example ${target.name} fails ${code} without env`); + if (ctx.profile === 'vendor-global' && run.scripts.unmarked) + problems.push( + `example ${target.name} inserts a script without loadVendor`, + ); + } + return problems.length ? problems.join(', ') : undefined; + } + + case 2: { + if (!primary) return 'no success event example to fail'; + if (ctx.profile === 'vendor-global') + return vendorFailureProblem(step, ctx, primary); + if (ctx.profile === 'http') { + for (const { label, answer } of failureAnswers) { + const run = await runDestination(step, ctx, primary, { + ...base, + answers: () => answer, + }); + const problem = reportProblem(label, run); + if (problem) return problem; + } + return undefined; + } + if (!ctx.env) return declaredFailures(step, ctx); + for (const { label, answer } of failureEnvAnswers) { + const run = await runDestination(step, ctx, primary, { + ...base, + envAnswers: () => answer, + }); + const problem = reportProblem(label, run); + if (problem) return problem; + } + return undefined; + } + + case 3: { + if (!primary) return 'no success event example to run'; + const run = await runDestination(step, ctx, primary, { + ...base, + answers: () => ({ late: true }), + envAnswers: () => ({ kind: 'late' }), + }); + if (!run.effects.length && !run.pending) + return 'the example sent nothing to answer late'; + return run.pending + ? 'the push settled before its request did (a floating send)' + : undefined; + } + + case 4: { + const target = primary ?? { + name: 'page view', + example: {}, + event: getEvent('page view'), + }; + if (ctx.required.length && !step.init) + return `checks no settings in init: init must throw a StepError naming ${ctx.required.join(', ')}`; + const thrown: unknown[] = []; + await runDestination(step, ctx, target, { + settings: {}, + initError: (error) => thrown.push(error), + }); + return notReadyProblem(ctx, thrown); + } + + case 5: { + if (!primary) return 'no success event example to run'; + const run = await runDestination(step, ctx, primary, base); + return run.initCalls + ? `init makes ${run.initCalls} request(s) or SDK call(s); init stays offline` + : undefined; + } + + case 6: { + const reads = new Set(); + for (const target of ctx.events) + await runDestination(step, ctx, target, { ...base, reads }); + return settingsProblem(ctx, reads); + } + + case 7: { + if (!primary) return 'no success event example to run'; + const event = withCanary(primary.event); + const ingest = { kitCanary: CANARY }; + const found: string[] = []; + const runs = [ + await runDestination(step, ctx, primary, base, event, ingest), + await runDestination( + step, + ctx, + primary, + { + ...base, + answers: () => ({ + status: 500, + body: { error: CANARY }, + statusText: CANARY, + }), + envAnswers: () => ({ + kind: 'reject', + error: new Error(`Request failed: ${CANARY}`), + }), + }, + event, + ingest, + ), + ]; + for (const run of runs) found.push(...leaks(run.logs, run)); + return found.length + ? `personal data reaches ${[...new Set(found)].join(', ')}` + : undefined; + } + + case 8: { + const revokes = Object.entries(options.examples).filter( + ([, example]) => + example.command === 'consent' && + isObject(example.in) && + Object.values(example.in).some((value) => value === false), + ); + if (!revokes.length) + return 'no walker consent revoke example (its out is the vendor consent call, or [] with a description)'; + const problems: string[] = []; + for (const [name, example] of revokes) { + if (!Array.isArray(example.out)) { + problems.push( + `example ${name} declares no out (the vendor consent call, or [] with a description)`, + ); + continue; + } + if (example.out.length === 0 && !example.description) + problems.push( + `example ${name}: out [] needs a description saying why no consent call is needed`, + ); + for (const problem of await checkDestinationExample(step, example, { + settings: options.settings, + env: options.env, + })) + problems.push(`example ${name}: ${problem}`); + } + return problems.length ? problems.join(', ') : undefined; + } + + case 9: + return batchProblem(step, ctx); + + case 10: + return strippedProblem(ctx, async (target, event) => { + const run = await runDestination(step, ctx, target, base, event); + return { + step: run, + delivered: Boolean(run.result.done?.[STEP_ID]) && !run.effects.length, + }; + }); + + case 11: { + const silent = Object.entries(options.examples).filter( + ([, example]) => + example.public === false && + isObject(example.mapping) && + example.mapping.silent === true, + ); + if (!silent.length) + return 'no public: false example with a silent: true rule'; + const problems: string[] = []; + for (const [name, example] of silent) { + if (!Array.isArray(example.out)) { + problems.push( + `example ${name} declares no out (the calls the rule settings describe, usually [])`, + ); + continue; + } + for (const problem of await checkDestinationExample(step, example, { + settings: options.settings, + env: ctx.runEnv ? options.env : undefined, + })) + problems.push(`example ${name}: ${problem}`); + } + return problems.length ? problems.join(', ') : undefined; + } + } + return undefined; +} + +/** Item 2 without an injectable boundary: the declared failure examples. */ +async function declaredFailures( + step: DestinationStep, + ctx: Context, +): Promise { + const failures = Object.entries(ctx.options.examples).filter( + ([, example]) => example.outcome?.status === 'failed', + ); + if (!failures.length) + return 'no boundary to fail (pass env) and no failure example (outcome failed)'; + const problems: string[] = []; + for (const [name, example] of failures) + for (const problem of await checkDestinationExample(step, example, { + settings: ctx.options.settings, + env: ctx.options.env, + })) + problems.push(`example ${name}: ${problem}`); + return problems.length ? problems.join(', ') : undefined; +} + +/** Item 2 for vendor globals: an injected script that fires `error`. */ +async function vendorFailureProblem( + step: DestinationStep, + ctx: Context, + primary: EventExample, +): Promise { + const mapping = mappingOf(primary.example, { event: primary.event }); + const session = await openDestination(step, { + settings: ctx.options.settings, + env: ctx.env, + config: { + timeout: RUN_TIMEOUT, + loadScript: true, + ...(mapping ? { mapping } : {}), + }, + }); + try { + await session.run({ event: primary.event }); + if (!session.scripts().marked) return declaredFailures(step, ctx); + + document + .querySelectorAll(`script[${VENDOR_SCRIPT_MARKER}]`) + .forEach((script) => script.dispatchEvent(new Event('error'))); + const run = await session.run({ event: primary.event }); + const reason = run.result.failed?.[STEP_ID]?.reason; + if (reason?.code !== 'vendor-not-loaded') + return `a push after the vendor script failed is not reported vendor-not-loaded (${show(reason)})`; + const dlq = session.collector.destinations[STEP_ID]?.dlq ?? []; + if (!dlq.length) return 'the vendor-not-loaded event is not in the DLQ'; + const errors = errorsIn(run.logs); + return errors === 1 + ? undefined + : `vendor-not-loaded writes ${errors} ERROR lines instead of the collector's one`; + } finally { + await session.close(); + } +} + +/** Item 9: one chunk of a batch fails, the others land, the counts add up. */ +async function batchProblem( + step: DestinationStep, + ctx: Context, +): Promise { + const { primary } = ctx; + if (!primary) return 'no success event example to batch'; + const mapping = mappingOf(primary.example, { event: primary.event }); + const session = await openDestination(step, { + settings: ctx.options.settings, + env: ctx.runEnv, + config: { + timeout: RUN_TIMEOUT, + batch: { size: 1000, wait: 60_000 }, + ...(mapping ? { mapping } : {}), + }, + answers: (index) => (index === 0 ? { status: 500 } : {}), + envAnswers: (index) => + index === 0 + ? { kind: 'reject', error: new Error('Connection refused') } + : { kind: 'pass' }, + }); + try { + const events = [1, 2, 3].map((n) => ({ + ...primary.event, + id: `${primary.event.id ?? 'event'}-${n}`, + })); + for (const event of events) await session.run({ event }); + await session.collector.command('flush'); + + const status = session.collector.status.destinations[STEP_ID]; + const dlq = session.collector.destinations[STEP_ID]?.dlq ?? []; + const requests = + session.boundary.requests.length + + (session.envBoundary?.calls.length ?? 0); + if (!requests) return 'the batch sent nothing'; + if (!status?.failed) return 'a failed chunk is not reported (failed 0)'; + if (dlq.length !== status.failed) + return `the DLQ holds ${dlq.length} events for ${status.failed} failed`; + if (requests > 1 && !status.count) + return 'one chunk failed and no other chunk landed'; + return undefined; + } finally { + await session.close(); + } +} + +/** Item 10: an event without user, data, globals and context. */ +async function strippedProblem( + ctx: Context, + run: ( + target: EventExample, + event: WalkerOS.DeepPartialEvent, + ) => Promise<{ step: Step; delivered: boolean }>, +): Promise { + // An example whose own out is [] sends nothing by design (silent, ignore). + const sending = ctx.events.filter(({ example }) => sends(example)); + if (!sending.length) return 'no success event example that sends'; + const problems: string[] = []; + for (const target of sending) { + const { user, data, globals, context, ...stripped } = target.event; + const { step, delivered } = await run(target, stripped); + if (failedCode(step) === 'missing-input') { + if (!declaresMissingInput(ctx.options.examples, stripped)) + problems.push( + `example ${target.name} fails missing-input without an out: [] example for that shape with a description`, + ); + } else if (delivered) + problems.push( + `example ${target.name} without user, data, globals and context counts as delivered but nothing was sent`, + ); + } + return problems.length ? problems.join(', ') : undefined; +} + +const SHAPE_KEYS = ['user', 'data', 'globals', 'context']; + +function declaresMissingInput( + examples: Flow.StepExamples, + stripped: WalkerOS.DeepPartialEvent, +): boolean { + return Object.values(examples).some( + (example) => + isEvent(example.in) && + example.in.name === stripped.name && + example.out?.length === 0 && + Boolean(example.description) && + example.outcome?.code === 'missing-input' && + SHAPE_KEYS.every((key) => { + const value = isObject(example.in) ? example.in[key] : undefined; + return ( + value === undefined || (isObject(value) && !Object.keys(value).length) + ); + }), + ); +} + +/* ------------------------------------------------------------------------ */ +/* Transformers */ +/* ------------------------------------------------------------------------ */ + +type TransformerStep = Transformer.Init; + +interface TransformerRun extends Step { + initCalls: number; + /** What made the flow fail to start (a throwing transformer factory). */ + startError?: unknown; +} + +async function runTransformer( + step: TransformerStep, + ctx: Context, + setup: Setup, + event: WalkerOS.DeepPartialEvent, + ingest?: Record, +): Promise { + let session: Session; + try { + session = await openTransformer(step, { + env: ctx.runEnv, + ...setup, + config: { ...setup.config }, + }); + } catch (startError) { + return { + result: { ok: false }, + effects: [], + logs: [], + pending: 0, + initCalls: 0, + startError, + }; + } + try { + const run = await session.run({ event, ingest }); + return { ...run, initCalls: session.initCalls() }; + } finally { + await session.close(); + } +} + +/** A flow that did not start, as a problem; a StepError counts as reported. */ +function startProblem(run: TransformerRun): Problem { + if (run.startError === undefined) return undefined; + return `the flow does not start: ${textOf(run.startError)}`; +} + +async function transformerItem( + item: number, + step: TransformerStep, + ctx: Context, +): Promise { + const { options, primary } = ctx; + const base: Setup = { settings: options.settings }; + const event = primary?.event ?? getEvent('page view'); + + switch (item) { + case 1: { + if (!pureJson(options.settings)) return 'settings are not pure JSON'; + const problems: string[] = []; + for (const target of ctx.events) { + const run = await runTransformer( + step, + ctx, + { ...base, env: undefined }, + target.event, + ); + const started = startProblem(run); + if (started) problems.push(`example ${target.name}: ${started}`); + const code = failedCode(run); + if (code && NOT_STARTED.includes(code)) + problems.push(`example ${target.name} fails ${code} without env`); + } + return problems.length ? problems.join(', ') : undefined; + } + + case 2: { + const problems: string[] = []; + if (ctx.required.length) { + const run = await runTransformer(step, ctx, { settings: {} }, event); + // Refusing to start with a StepError is reported too, at the start. + if (run.startError !== undefined) { + if (!stepError(run.startError)) + problems.push( + 'a transformer that cannot work throws a plain error at the start: throw a StepError', + ); + } else if (run.result.ok || !run.result.reason) + problems.push( + 'a transformer that cannot work does not answer ok: false with a reason', + ); + if (run.effects.length) + problems.push('an event passes a transformer that cannot work'); + } + const failures = Object.fromEntries( + Object.entries(options.examples).filter( + ([, example]) => example.outcome?.status === 'failed', + ), + ); + if (Object.keys(failures).length) + await runExamples('transformer', step, failures, { + settings: options.settings, + env: options.env, + }).catch((error: unknown) => problems.push(textOf(error))); + return problems.length ? problems.join(', ') : undefined; + } + + case 4: { + const thrown: unknown[] = []; + await runTransformer( + step, + ctx, + { settings: {}, initError: (error) => thrown.push(error) }, + event, + ); + return notReadyProblem(ctx, thrown); + } + + case 5: { + const run = await runTransformer(step, ctx, base, event); + return ( + startProblem(run) ?? + (run.initCalls + ? `init makes ${run.initCalls} request(s) or SDK call(s); init stays offline` + : undefined) + ); + } + + case 6: { + const reads = new Set(); + for (const target of ctx.events) + await runTransformer(step, ctx, { ...base, reads }, target.event); + return settingsProblem(ctx, reads); + } + + case 7: { + const run = await runTransformer(step, ctx, base, withCanary(event), { + kitCanary: CANARY, + }); + const found = leaks(run.logs, run); + if (textOf(run.startError).includes(CANARY)) + found.push('the start error message'); + return ( + startProblem(run) ?? + (found.length + ? `personal data reaches ${[...new Set(found)].join(', ')}` + : undefined) + ); + } + + case 10: + return strippedProblem(ctx, async (_, stripped) => { + const run = await runTransformer(step, ctx, base, stripped); + return { + step: run, + delivered: run.result.ok && !run.effects.length, + }; + }); + } + return undefined; +} + +/* ------------------------------------------------------------------------ */ +/* Stores */ +/* ------------------------------------------------------------------------ */ + +type StoreStep = Store.Init; + +function operationsOf(examples: Flow.StepExamples): StoreOperation[] { + return Object.values(examples).flatMap((example) => + !example.outcome && isStoreOperation(example.in) ? [example.in] : [], + ); +} + +async function withStore( + step: StoreStep, + setup: Setup, + fn: (session: Session) => Promise, +): Promise { + const session = await openStore(step, setup); + try { + return await fn(session); + } finally { + await session.close(); + } +} + +async function storeItem( + item: number, + step: StoreStep, + ctx: Context, +): Promise { + const { options } = ctx; + const operations = operationsOf(options.examples); + const base: Setup = { settings: options.settings, env: ctx.env }; + + switch (item) { + case 1: { + if (!pureJson(options.settings)) return 'settings are not pure JSON'; + const thrown: unknown[] = []; + await withStore( + step, + { settings: options.settings, initError: (e) => thrown.push(e) }, + async () => undefined, + ).catch((error: unknown) => thrown.push(error)); + return thrown.length + ? `does not start without env: ${textOf(thrown[0])}` + : undefined; + } + + case 2: { + if (!ctx.env) return storeDeclaredFailures(step, ctx); + if (!operations.length) return 'no store operation example to fail'; + const problems: string[] = []; + for (const { label, answer } of failureEnvAnswers) { + // A store call has no collector timeout around it here: a call that + // never settles is item 3's late answer, not a failure to wait for. + if (answer.kind === 'hang') continue; + await withStore( + step, + { ...base, envAnswers: () => answer }, + async (session) => { + for (const operation of operations) { + const logsBefore = session.logs.length; + const error = await runStoreOperation(session, operation).then( + () => undefined, + (thrown: unknown) => thrown, + ); + if (!stepError(error)) + problems.push( + `${label} on ${operation.operation} ${error ? 'throws a plain error' : 'is not reported'}: throw a StepError`, + ); + if (errorsIn(session.logs.slice(logsBefore))) + problems.push( + `${label} on ${operation.operation} is logged by the store; the collector writes the one line`, + ); + } + }, + ); + } + return problems.length ? [...new Set(problems)].join(', ') : undefined; + } + + case 3: { + if (!ctx.env) return 'no SDK boundary to answer late (pass env)'; + const problems: string[] = []; + await withStore( + step, + { ...base, envAnswers: () => ({ kind: 'late' }) }, + async (session) => { + for (const operation of operations) { + await runStoreOperation(session, operation).catch(() => undefined); + if (session.envBoundary?.pending()) + problems.push( + `${operation.operation} settles before its SDK call did (a floating call)`, + ); + } + }, + ); + return problems.length ? problems.join(', ') : undefined; + } + + case 4: { + const thrown: unknown[] = []; + await withStore( + step, + { settings: {}, env: ctx.env, initError: (e) => thrown.push(e) }, + async () => undefined, + ).catch(() => undefined); + return notReadyProblem(ctx, thrown); + } + + case 5: { + const calls = await withStore(step, base, async (session) => + session.initCalls(), + ); + return calls + ? `init makes ${calls} request(s) or SDK call(s); init stays offline` + : undefined; + } + + case 6: { + const reads = new Set(); + await withStore(step, { ...base, reads }, async (session) => { + for (const operation of operations) + await runStoreOperation(session, operation).catch(() => undefined); + }); + return settingsProblem(ctx, reads); + } + + case 7: { + const found: string[] = []; + for (const envAnswer of [ + { kind: 'pass' } satisfies EnvAnswer, + { + kind: 'reject', + error: new Error(`Request failed: ${CANARY}`), + } satisfies EnvAnswer, + ]) + await withStore( + step, + { ...base, envAnswers: () => envAnswer }, + async (session) => { + for (const operation of operations) { + const planted = + operation.operation === 'set' + ? { ...operation, value: CANARY } + : operation; + const error = await runStoreOperation(session, planted).then( + () => undefined, + (thrown: unknown) => thrown, + ); + if (textOf(error).includes(CANARY)) + found.push('the thrown error message'); + } + found.push(...leaks(session.logs)); + }, + ); + return found.length + ? `personal data reaches ${[...new Set(found)].join(', ')}` + : undefined; + } + } + return undefined; +} + +async function storeDeclaredFailures( + step: StoreStep, + ctx: Context, +): Promise { + const failures = Object.fromEntries( + Object.entries(ctx.options.examples).filter( + ([, example]) => example.outcome?.status === 'failed', + ), + ); + if (!Object.keys(failures).length) + return 'no SDK boundary to fail (pass env) and no failure example (outcome failed)'; + return runExamples('store', step, failures, { + settings: ctx.options.settings, + }).then( + () => undefined, + (error: unknown) => textOf(error), + ); +} + +/* ------------------------------------------------------------------------ */ +/* Sources */ +/* ------------------------------------------------------------------------ */ + +type SourceStep = Source.Init; + +/** Add the canary to every plain object of an input (item 7 for sources). */ +function plantCanary(value: unknown, depth = 0): unknown { + if (depth > 5) return value; + if (Array.isArray(value)) + return value.map((item) => plantCanary(item, depth + 1)); + if (!isObject(value)) return value; + const planted: Record = { kitCanary: CANARY }; + for (const [key, inner] of Object.entries(value)) + planted[key] = plantCanary(inner, depth + 1); + return planted; +} + +function hasCanary(value: unknown): boolean { + return textOf(value).includes(CANARY); +} + +async function sourceItem( + item: number, + step: SourceStep, + ctx: Context, +): Promise { + const { options } = ctx; + + const start = async (setup: Setup) => { + const thrown: unknown[] = []; + const session = await openSource(step, { + ...setup, + initError: (error) => thrown.push(error), + }).catch((error: unknown) => { + thrown.push(error); + return undefined; + }); + const initCalls = session?.initCalls() ?? 0; + const logs = session?.logs ?? []; + await session?.close(); + return { thrown, initCalls, logs }; + }; + + switch (item) { + case 1: { + if (!pureJson(options.settings)) return 'settings are not pure JSON'; + const { thrown, logs } = await start({ settings: options.settings }); + if (thrown.length) + return `does not start without env: ${textOf(thrown[0])}`; + return errorsIn(logs) ? 'logs an ERROR when it starts' : undefined; + } + + case 4: { + const { thrown } = await start({ settings: {}, env: ctx.env }); + return notReadyProblem(ctx, thrown); + } + + case 5: { + const { initCalls } = await start({ + settings: options.settings, + env: ctx.env, + }); + return initCalls + ? `init makes ${initCalls} request(s) or SDK call(s); init stays offline` + : undefined; + } + + case 6: { + const reads = new Set(); + await start({ settings: options.settings, env: ctx.env, reads }); + return settingsProblem(ctx, reads); + } + + case 7: { + if (!options.createTrigger) + return 'sources run their examples through the package createTrigger: pass it'; + const found: string[] = []; + let planted = 0; + for (const example of Object.values(options.examples)) { + const input = plantCanary(example.in); + if (!hasCanary(input)) continue; + planted++; + const logs: LogLine[] = []; + const instance = await options.createTrigger( + sourceConfig( + step, + { env: options.env, settings: options.settings }, + [], + logs, + ), + ); + try { + await instance + .trigger( + example.trigger?.type, + example.trigger?.options, + )(input) + .then( + (result) => { + if (hasCanary(isObject(result) ? result.error : undefined)) + found.push('the answered error'); + }, + (error: unknown) => { + if (hasCanary(error)) found.push('the thrown error message'); + }, + ); + } finally { + await instance.flow?.collector.command('shutdown'); + } + found.push(...leaks(logs)); + } + if (!planted) + return 'no example input carries an object to plant the canary in'; + return found.length + ? `personal data reaches ${[...new Set(found)].join(', ')}` + : undefined; + } + } + return undefined; +} + +/* ------------------------------------------------------------------------ */ +/* Entry */ +/* ------------------------------------------------------------------------ */ + +type KitArgs = + | ['destination', DestinationStep] + | ['transformer', TransformerStep] + | ['store', StoreStep] + | ['source', SourceStep]; + +const KIND_PROFILES: Record = { + destination: ['http', 'sdk-env', 'vendor-global', 'socket'], + transformer: ['transformer'], + store: ['socket'], + source: ['source'], +}; + +type ConformanceArgs = + | ['destination', DestinationStep, ConformanceOptions] + | ['transformer', TransformerStep, ConformanceOptions] + | ['source', SourceStep, ConformanceOptions] + | ['store', StoreStep, ConformanceOptions]; + +function kitArgs(args: ConformanceArgs): KitArgs { + switch (args[0]) { + case 'destination': + return ['destination', args[1]]; + case 'transformer': + return ['transformer', args[1]]; + case 'store': + return ['store', args[1]]; + case 'source': + return ['source', args[1]]; + } +} + +/** + * Check a step against the contract. Resolves when every check its profile + * runs passes; otherwise rejects with one `ConformanceError` listing every + * failed item. Runs on real timers. + */ +export function conformance( + kind: 'destination', + step: Destination.Instance, + options: ConformanceOptions, +): Promise; +export function conformance( + kind: 'transformer', + step: Transformer.Init, + options: ConformanceOptions, +): Promise; +export function conformance( + kind: 'source', + step: Source.Init, + options: ConformanceOptions, +): Promise; +export function conformance( + kind: 'store', + step: Store.Init, + options: ConformanceOptions, +): Promise; +export async function conformance(...given: ConformanceArgs): Promise { + const [kind, , options] = given; + if (timersAreFake()) + throw new Error( + 'conformance runs on real timers: call jest.useRealTimers() first', + ); + if (!KIND_PROFILES[kind].includes(options.profile)) + throw new Error( + `profile ${options.profile} is not a ${kind} profile (${KIND_PROFILES[kind].join(', ')})`, + ); + + const args = kitArgs(given); + const ctx = contextOf(options); + const items = PROFILE_ITEMS[options.profile].filter( + (item) => + item !== 9 || (args[0] === 'destination' && Boolean(args[1].pushBatch)), + ); + if (args[0] === 'destination') items.push(11); + + const failures: Array<[number, string]> = []; + for (const item of items) { + const problem = await check(args, item, ctx).catch( + (error: unknown) => `the check threw: ${textOf(error)}`, + ); + if (problem) failures.push([item, problem]); + } + if (failures.length) throw new ConformanceError(failures); +} + +function check(args: KitArgs, item: number, ctx: Context): Promise { + switch (args[0]) { + case 'destination': + return destinationItem(item, args[1], ctx); + case 'transformer': + return transformerItem(item, args[1], ctx); + case 'store': + return storeItem(item, args[1], ctx); + case 'source': + return sourceItem(item, args[1], ctx); + } +} diff --git a/packages/collector/src/dev/runExamples.ts b/packages/collector/src/dev/runExamples.ts new file mode 100644 index 000000000..7fb0e7b9e --- /dev/null +++ b/packages/collector/src/dev/runExamples.ts @@ -0,0 +1,930 @@ +import type { + Collector, + Destination, + Elb, + Flow, + Logger, + Mapping, + StandardEventName, + Source, + Store, + Transformer, + Trigger, + WalkerOS, +} from '@walkeros/core'; +import { + createIngest, + getEvent, + isObject, + isStoreValue, + StepError, +} from '@walkeros/core'; +import { startFlow } from '../flow'; +import { wrapEnv } from '../wrapEnv'; +import type { Answer, Boundary, EnvAnswer, EnvBoundary } from './boundary'; +import { answerEnv, installBoundary } from './boundary'; + +/** + * The typed step-example runner: every example runs through a real collector + * (`startFlow`), mocked only at the boundary (the package's examples env, or + * the kit's fetch boundary), and its `out` and `outcome` are compared with + * what the step did. One implementation with a thin binding per step kind. + */ + +/** The id every run registers its step under. */ +export const STEP_ID = 'step'; + +/** The run phase a boundary records: requests made by `init` are `init`. */ +type Phase = 'init' | 'push'; + +export interface LogLine { + level: Logger.Level; + message: string; + context: Logger.LogContext; + scope: string[]; +} + +/** The package's examples env (`examples.env`): the mocks and their paths. */ +export interface ExamplesEnv { + push: Record; + simulation: string[]; +} + +/** `examples.env` read as mocks plus paths; a bare mock object has no paths. */ +export function readEnv( + env: Record | undefined, +): ExamplesEnv | undefined { + if (!env) return undefined; + const push = isObject(env.push) ? env.push : env; + const simulation = Array.isArray(env.simulation) + ? env.simulation.filter((path): path is string => typeof path === 'string') + : []; + return { push, simulation }; +} + +/** What one input is: an event to push or a walker command. */ +export interface Input { + event?: WalkerOS.DeepPartialEvent; + command?: Flow.StepCommand; + data?: unknown; + ingest?: Record; +} + +export function isEvent(value: unknown): value is WalkerOS.DeepPartialEvent { + return isObject(value) && typeof value.name === 'string'; +} + +function isRule(value: unknown): value is Mapping.Rule { + return isObject(value); +} + +/** The input an example describes, or a problem with it. */ +export function inputOf(example: Flow.StepExample): Input | string { + if (example.command) return { command: example.command, data: example.in }; + if (isEvent(example.in)) return { event: example.in }; + return 'its in is neither an event (an object with a name) nor a command'; +} + +/** Entity and action of an event name. */ +function keysOf(event: WalkerOS.DeepPartialEvent): [string, string] { + const [entity = '', action = ''] = (event.name ?? '').split(' '); + return [event.entity ?? entity, event.action ?? action]; +} + +/** + * R2: an example's own rule, as the destination's mapping for its event; + * without one the destination's configured mapping applies. + */ +export function mappingOf( + example: Flow.StepExample, + input: Input, +): Mapping.Rules | undefined { + if (!input.event || !isRule(example.mapping)) return undefined; + const [entity, action] = keysOf(input.event); + return { [entity]: { [action]: example.mapping } }; +} + +/** How the boundary answers an example's `respond`. */ +export function answerOf(respond?: Flow.StepRespond): Answer { + if (!respond) return {}; + return { status: respond.status, body: respond.body, error: respond.error }; +} + +/** The SDK boundary's answer to an example's `respond`. */ +export function envAnswerOf(respond?: Flow.StepRespond): EnvAnswer { + if (!respond) return { kind: 'pass' }; + if (respond.error === 'timeout') return { kind: 'hang' }; + if (respond.error === 'network') + return { kind: 'reject', error: new Error('Network error') }; + if (respond.status !== undefined && respond.status >= 400) + return { + kind: 'reject', + error: new StepError(`HTTP ${respond.status}`, { + code: 'transport', + status: respond.status, + body: respond.body, + }), + }; + return { kind: 'pass' }; +} + +/** Timeout of runs that wait for a collector timeout (ms). */ +export const RUN_TIMEOUT = 1000; + +/** Whether the test runs on fake timers (jest's modern or legacy fakes). */ +export function timersAreFake(): boolean { + return '_isMockFunction' in setTimeout || 'clock' in setTimeout; +} + +/** Setup of one run: what the step gets and what the boundaries answer. */ +export interface Setup { + settings?: unknown; + /** Extra step config (mapping, batch, timeout, consent, loadScript). */ + config?: Record; + /** Examples env; without it the step runs on its own env, at the boundary. */ + env?: ExamplesEnv; + answers?: (index: number) => Answer; + envAnswers?: (index: number) => EnvAnswer; + /** Records every settings key the step reads (kit item 6). */ + reads?: Set; + /** Receives what init threw (kit item 4). */ + initError?: (error: unknown) => void; +} + +/** One input's run: its result and what happened at the boundaries. */ +export interface Step { + result: Elb.PushResult; + effects: Flow.StepEffect[]; + logs: LogLine[]; + /** Requests or SDK calls still unanswered when the push settled. */ + pending: number; +} + +/** A started flow with one step and the boundaries around it. */ +export interface Session { + collector: Collector.Instance; + logs: LogLine[]; + boundary: Boundary; + envBoundary?: EnvBoundary; + /** Requests and SDK calls made while the step initialized. */ + initCalls(): number; + /** Scripts added to the document (web), with and without the marker. */ + scripts(): { marked: number; unmarked: number }; + run(input: Input): Promise; + close(): Promise; +} + +/** Keys a read of settings may touch without being a setting. */ +const IGNORED_READS = new Set([ + ...Object.getOwnPropertyNames(Object.prototype), + 'toJSON', + 'then', + '$$typeof', + 'asymmetricMatch', + 'nodeType', + 'tagName', +]); + +/** Settings as a Proxy that records every key read, else as given. */ +function settingsOf(setup: Setup): unknown { + const { settings, reads } = setup; + if (!reads || !isObject(settings)) return settings; + const record = (key: string | symbol) => { + if (typeof key === 'string' && !IGNORED_READS.has(key)) reads.add(key); + }; + return new Proxy(settings, { + get(target, key, receiver) { + record(key); + return Reflect.get(target, key, receiver); + }, + has(target, key) { + record(key); + return Reflect.has(target, key); + }, + }); +} + +/** + * The marker web-core's `loadVendor` sets on every script it injects (its + * `VENDOR_SCRIPT_MARKER`; the collector does not depend on web-core). A + * drift fails the vendor-global item 1 of every adopting destination. + */ +export const VENDOR_SCRIPT_MARKER = 'data-walkeros-vendor'; + +function documentScripts(): Element[] { + if (typeof document === 'undefined') return []; + return Array.from(document.querySelectorAll('script')); +} + +/** Runs `fn` as the init phase; its errors go to `initError`. */ +async function inInit( + setPhase: (phase: Phase) => void, + setup: Setup, + fn: () => R | Promise, +): Promise { + setPhase('init'); + try { + return await fn(); + } catch (error) { + setup.initError?.(error); + throw error; + } finally { + setPhase('push'); + } +} + +/** Open a session: install the boundaries, start the flow `bind` describes. */ +async function open( + setup: Setup, + bind: ( + setPhase: (phase: Phase) => void, + env: Record | undefined, + ) => Collector.InitConfig, + effectsOf?: () => Flow.StepEffect[], +): Promise { + const logs: LogLine[] = []; + const boundary = installBoundary(setup.answers); + const before = new Set(documentScripts()); + + let envBoundary: EnvBoundary | undefined; + let env: Record | undefined; + let recorded: Array<{ fn: string; args: unknown[] }> = []; + if (setup.env) { + envBoundary = answerEnv( + setup.env.push, + setup.env.simulation, + setup.envAnswers, + ); + if (setup.env.simulation.length) { + const wrapped = wrapEnv({ + ...envBoundary.env, + simulation: setup.env.simulation, + }); + env = wrapped.wrappedEnv; + recorded = wrapped.calls; + } else env = envBoundary.env; + } + + const setPhase = (phase: Phase) => { + boundary.phase = phase; + if (envBoundary) envBoundary.phase = phase; + }; + + const handler: Logger.Handler = (level, message, context, scope) => { + logs.push({ level, message, context, scope }); + }; + const config = bind(setPhase, env); + + let collector: Collector.Instance; + try { + ({ collector } = await startFlow({ + ...config, + logger: { level: 'DEBUG', handler }, + })); + } catch (error) { + boundary.restore(); + throw error; + } + + const effects = (): Flow.StepEffect[] => { + if (effectsOf) return effectsOf(); + if (setup.env) return recorded.map(({ fn, args }) => [fn, ...args]); + return boundary.requests.map((request) => request.effect); + }; + + return { + collector, + logs, + boundary, + envBoundary, + initCalls: () => + boundary.requests.filter((request) => request.phase === 'init').length + + (envBoundary?.calls.filter((call) => call.phase === 'init').length ?? 0), + scripts: () => { + const added = documentScripts().filter((script) => !before.has(script)); + const marked = added.filter((script) => + script.hasAttribute(VENDOR_SCRIPT_MARKER), + ).length; + return { marked, unmarked: added.length - marked }; + }, + run: async (input) => { + const effectsBefore = effects().length; + const logsBefore = logs.length; + const result = input.command + ? await collector.command(input.command, input.data) + : await collector.push( + input.event ?? {}, + input.ingest + ? { ingest: { ...createIngest('kit'), ...input.ingest } } + : undefined, + ); + return { + result, + effects: effects().slice(effectsBefore), + logs: logs.slice(logsBefore), + pending: boundary.pending() + (envBoundary?.pending() ?? 0), + }; + }, + close: async () => { + try { + await collector.command('shutdown'); + } finally { + boundary.restore(); + } + }, + }; +} + +/** A destination with its init run in the init phase. */ +function instrumentDestination( + step: Destination.Instance, + setPhase: (phase: Phase) => void, + setup: Setup, +): Destination.Instance { + const init = step.init; + if (!init) return step; + return { + ...step, + init: (context) => inInit(setPhase, setup, () => init.call(step, context)), + }; +} + +export function openDestination( + step: Destination.Instance, + setup: Setup, +): Promise { + return open(setup, (setPhase, env) => ({ + destinations: { + [STEP_ID]: { + code: instrumentDestination(step, setPhase, setup), + config: { + ...setup.config, + ...(setup.settings !== undefined + ? { settings: settingsOf(setup) } + : {}), + ...(env ? { env } : {}), + }, + }, + }, + })); +} + +/** What a destination's `before` chain delivered: the transformer's effects. */ +export function openTransformer( + step: Transformer.Init, + setup: Setup, +): Promise { + const delivered: Flow.StepEffect[] = []; + const capture: Destination.Instance = { + type: 'capture', + config: {}, + push: (event) => { + delivered.push(['return', { event }]); + }, + }; + return open( + setup, + (setPhase, env) => { + const code: Transformer.Init = (context) => + inInit(setPhase, setup, async () => { + const instance = await step(context); + const init = instance.init; + if (!init) return instance; + return { + ...instance, + init: (initContext) => + inInit(setPhase, setup, () => init.call(instance, initContext)), + }; + }); + return { + transformers: { + [STEP_ID]: { + code, + config: { + ...setup.config, + ...(setup.settings !== undefined + ? { settings: settingsOf(setup) } + : {}), + ...(env ? { env } : {}), + }, + }, + }, + // Under the step's id too, so its failure reads like a destination's. + destinations: { [STEP_ID]: { code: capture, before: STEP_ID } }, + }; + }, + () => delivered, + ); +} + +/** A store's operations on the collector's own instance. */ +export function openStore( + step: Store.Init, + setup: Setup, +): Promise { + return open(setup, (setPhase, env) => ({ + stores: { + [STEP_ID]: { + code: (context) => inInit(setPhase, setup, () => step(context)), + config: { + ...setup.config, + ...(setup.settings !== undefined + ? { settings: settingsOf(setup) } + : {}), + ...(env ? { env } : {}), + }, + }, + }, + })); +} + +/** A source started in the init phase; its events stop at the collector. */ +export function openSource( + step: Source.Init, + setup: Setup, +): Promise { + return open(setup, (setPhase, env) => ({ + sources: { + [STEP_ID]: { + code: (context) => inInit(setPhase, setup, () => step(context)), + config: { + ...setup.config, + ...(setup.settings !== undefined + ? { settings: settingsOf(setup) } + : {}), + }, + ...(env ? { env } : {}), + }, + }, + })); +} + +/** A store operation an example describes. */ +export interface StoreOperation { + operation: 'get' | 'set' | 'delete'; + key: string; + value?: unknown; +} + +export function isStoreOperation(value: unknown): value is StoreOperation { + return ( + isObject(value) && + typeof value.key === 'string' && + (value.operation === 'get' || + value.operation === 'set' || + value.operation === 'delete') + ); +} + +/** Run one store operation; its effect is the operation and its result. */ +export async function runStoreOperation( + session: Session, + operation: StoreOperation, +): Promise { + const store = session.collector.stores[STEP_ID]; + if (!store) throw new Error('The store did not start'); + if (operation.operation === 'get') + return ['get', operation.key, await store.get(operation.key)]; + if (operation.operation === 'set') { + if (!isStoreValue(operation.value)) + throw new Error(`The value to set ${operation.key} is not a store value`); + await store.set(operation.key, operation.value); + return ['set', operation.key, operation.value]; + } + await store.delete(operation.key); + return ['delete', operation.key]; +} + +/** Structural equality; a key whose value is undefined counts as absent. */ +export function isEqual(a: unknown, b: unknown): boolean { + if (Object.is(a, b)) return true; + if (Array.isArray(a) || Array.isArray(b)) { + if (!Array.isArray(a) || !Array.isArray(b) || a.length !== b.length) + return false; + return a.every((value, index) => isEqual(value, b[index])); + } + if (isObject(a) && isObject(b)) { + const keysOfA = Object.keys(a).filter((key) => a[key] !== undefined); + const keysOfB = Object.keys(b).filter((key) => b[key] !== undefined); + if (keysOfA.length !== keysOfB.length) return false; + return keysOfA.every((key) => isEqual(a[key], b[key])); + } + return false; +} + +/** A value as short readable text for a problem message. */ +export function show(value: unknown): string { + let text: string; + try { + text = + JSON.stringify(value, (_, inner: unknown) => + inner === undefined ? 'undefined' : inner, + ) ?? String(value); + } catch { + text = String(value); + } + return text.length > 400 ? `${text.slice(0, 400)}...` : text; +} + +/** The outcome a result reports for the step. */ +export function outcomeOf( + result: Elb.PushResult, +): Flow.StepOutcome | undefined { + const failed = result.failed?.[STEP_ID]?.reason; + if (failed) + return 'retryable' in failed + ? { status: 'failed', code: failed.code, retryable: failed.retryable } + : { status: 'failed', code: failed.code }; + const queued = result.queued?.[STEP_ID]?.reason; + if (queued) return { status: 'queued', code: queued.code }; + if (!result.ok && result.reason) + return { + status: 'failed', + code: result.reason.code, + retryable: result.reason.retryable, + }; + return undefined; +} + +/** Problems of an outcome against the example's declared one. */ +export function compareOutcome( + actual: Flow.StepOutcome | undefined, + expected: Flow.StepOutcome | undefined, +): string | undefined { + if (!expected) { + if (actual?.status === 'failed') + return `failed with ${actual.code} but declares no outcome`; + return undefined; + } + if (!actual) return `expected ${expected.status} ${expected.code}, got none`; + if (actual.status !== expected.status || actual.code !== expected.code) + return `expected ${expected.status} ${expected.code}, got ${actual.status} ${actual.code}`; + if ( + expected.retryable !== undefined && + actual.retryable !== expected.retryable + ) + return `expected retryable ${expected.retryable}, got ${actual.retryable}`; + return undefined; +} + +function compareOut( + effects: Flow.StepEffect[], + example: Flow.StepExample, +): string | undefined { + if (example.out === undefined || isEqual(effects, example.out)) + return undefined; + return `out differs: expected ${show(example.out)}, got ${show(effects)}`; +} + +/** Common options of every kind. */ +export interface RunOptions { + /** The step's settings, pure JSON. */ + settings?: Record; + /** The package's examples env (`examples.env`: `push` and `simulation`). */ + env?: Record; +} + +export interface DestinationRunOptions extends RunOptions { + /** Further destination config (consent, batch, mapping, loadScript). */ + config?: Partial; + /** R3: the standard events the destination maps. */ + claims?: StandardEventName[]; +} + +export interface TransformerRunOptions extends RunOptions { + config?: Partial; +} + +export interface StoreRunOptions extends RunOptions { + config?: Partial; +} + +export interface SourceRunOptions extends RunOptions { + config?: Partial; + /** The package's `createTrigger`, which feeds an example's `in`. */ + createTrigger?: Trigger.CreateFn; +} + +/** The setup of an example's run. */ +function setupOf( + example: Flow.StepExample, + options: RunOptions & { config?: object }, + mapping?: Mapping.Rules, +): Setup { + const respond = example.respond; + const config: Record = { ...options.config }; + if (mapping) config.mapping = mapping; + if (respond?.error === 'timeout' && config.timeout === undefined) + config.timeout = RUN_TIMEOUT; + return { + settings: options.settings, + config, + env: readEnv(options.env), + answers: () => answerOf(respond), + envAnswers: () => envAnswerOf(respond), + }; +} + +/** Run one destination example; its problems, empty when it holds. */ +export async function checkDestinationExample( + step: Destination.Instance, + example: Flow.StepExample, + options: DestinationRunOptions, +): Promise { + const input = inputOf(example); + if (typeof input === 'string') return [input]; + const session = await openDestination( + step, + setupOf(example, options, mappingOf(example, input)), + ); + try { + const { result, effects } = await session.run(input); + const problems: string[] = []; + const out = compareOut(effects, example); + if (out) problems.push(out); + const outcome = compareOutcome(outcomeOf(result), example.outcome); + if (outcome) problems.push(outcome); + return problems; + } finally { + await session.close(); + } +} + +async function checkTransformerExample( + step: Transformer.Init, + example: Flow.StepExample, + options: TransformerRunOptions, +): Promise { + if (!isEvent(example.in)) + return ['its in is not an event (an object with a name)']; + const session = await openTransformer(step, setupOf(example, options)); + try { + const { result, effects } = await session.run({ event: example.in }); + const problems: string[] = []; + const out = compareOut(effects, example); + if (out) problems.push(out); + const outcome = compareOutcome(outcomeOf(result), example.outcome); + if (outcome) problems.push(outcome); + return problems; + } finally { + await session.close(); + } +} + +async function checkStoreExample( + step: Store.Init, + example: Flow.StepExample, + options: StoreRunOptions, +): Promise { + if (!isStoreOperation(example.in)) + return ['its in is not a store operation ({ operation, key, value? })']; + const session = await openStore(step, setupOf(example, options)); + try { + let effect: Flow.StepEffect; + try { + effect = await runStoreOperation(session, example.in); + } catch (error) { + const code = error instanceof StepError ? error.code : 'transport'; + const outcome = compareOutcome( + { status: 'failed', code }, + example.outcome, + ); + return outcome ? [outcome] : []; + } + const problems: string[] = []; + const out = compareOut([effect], example); + if (out) problems.push(out); + const outcome = compareOutcome(undefined, example.outcome); + if (outcome) problems.push(outcome); + return problems; + } finally { + await session.close(); + } +} + +/** The config a source's createTrigger starts, events captured at the end. */ +export function sourceConfig( + step: Source.Init, + options: SourceRunOptions, + captured: Flow.StepEffect[], + logs: LogLine[], + settings: unknown = options.settings, +): Collector.InitConfig { + const env = readEnv(options.env); + const terminus: Collector.PushFn = async (event) => { + captured.push(['elb', event]); + return { ok: true }; + }; + return { + logger: { + level: 'DEBUG', + handler: (level, message, context, scope) => { + logs.push({ level, message, context, scope }); + }, + }, + sources: { + [STEP_ID]: { + code: step, + config: { + ...options.config, + ...(settings !== undefined ? { settings } : {}), + }, + ...(env ? { env: env.push } : {}), + terminus, + }, + }, + }; +} + +async function checkSourceExample( + step: Source.Init, + example: Flow.StepExample, + options: SourceRunOptions, +): Promise { + if (!options.createTrigger) + return ['sources run through the package createTrigger: pass it']; + const captured: Flow.StepEffect[] = []; + const instance = await options.createTrigger( + sourceConfig(step, options, captured, []), + ); + try { + await instance.trigger( + example.trigger?.type, + example.trigger?.options, + )(example.in); + const out = compareOut(captured, example); + return out ? [out] : []; + } finally { + await instance.flow?.collector.command('shutdown'); + } +} + +/** + * R1, R3, R6: each claimed standard event has a public example (looked up + * by its event name) that starts from the pinned `getEvent(name)`, and each + * public event example is claimed. + */ +export function checkClaims( + examples: Flow.StepExamples, + claims: StandardEventName[], +): string[] { + const problems: string[] = []; + const publicEvents = Object.entries(examples).filter( + ([, example]) => + example.public !== false && !example.command && isEvent(example.in), + ); + + for (const claim of claims) { + const found = publicEvents.find( + ([, example]) => isEvent(example.in) && example.in.name === claim, + ); + if (!found) { + problems.push(`claim ${claim}: no public example for it`); + continue; + } + const [name, example] = found; + if (isEvent(example.in) && example.in.id !== getEvent(claim).id) + problems.push( + `claim ${claim}: example ${name} does not start from getEvent('${claim}')`, + ); + } + + for (const [name, example] of publicEvents) { + const eventName = isEvent(example.in) ? example.in.name : undefined; + if (!claims.some((claim) => claim === eventName)) + problems.push(`example ${name}: ${eventName} is public but not claimed`); + } + + return problems; +} + +type RunArgs = + | [ + 'destination', + Destination.Instance, + Flow.StepExamples, + DestinationRunOptions, + ] + | [ + 'transformer', + Transformer.Init, + Flow.StepExamples, + TransformerRunOptions, + ] + | [ + 'store', + Store.Init, + Flow.StepExamples, + StoreRunOptions, + ] + | [ + 'source', + Source.Init, + Flow.StepExamples, + SourceRunOptions, + ]; + +/** The arguments as given: the options are optional. */ +type RunArgsGiven = + | [ + 'destination', + Destination.Instance, + Flow.StepExamples, + DestinationRunOptions?, + ] + | [ + 'transformer', + Transformer.Init, + Flow.StepExamples, + TransformerRunOptions?, + ] + | [ + 'store', + Store.Init, + Flow.StepExamples, + StoreRunOptions?, + ] + | [ + 'source', + Source.Init, + Flow.StepExamples, + SourceRunOptions?, + ]; + +function complete(args: RunArgsGiven): RunArgs { + switch (args[0]) { + case 'destination': + return ['destination', args[1], args[2], args[3] ?? {}]; + case 'transformer': + return ['transformer', args[1], args[2], args[3] ?? {}]; + case 'store': + return ['store', args[1], args[2], args[3] ?? {}]; + case 'source': + return ['source', args[1], args[2], args[3] ?? {}]; + } +} + +async function checkExample( + args: RunArgs, + example: Flow.StepExample, +): Promise { + switch (args[0]) { + case 'destination': + return checkDestinationExample(args[1], example, args[3]); + case 'transformer': + return checkTransformerExample(args[1], example, args[3]); + case 'store': + return checkStoreExample(args[1], example, args[3]); + case 'source': + return checkSourceExample(args[1], example, args[3]); + } +} + +/** + * Run every example of a step through a real collector and compare its `out` + * (R4, as a literal) and `outcome`. Resolves when every example holds; + * otherwise rejects with one error naming each failing example. + */ +export function runExamples( + kind: 'destination', + step: Destination.Instance, + examples: Flow.StepExamples, + options?: DestinationRunOptions, +): Promise; +export function runExamples( + kind: 'transformer', + step: Transformer.Init, + examples: Flow.StepExamples, + options?: TransformerRunOptions, +): Promise; +export function runExamples( + kind: 'store', + step: Store.Init, + examples: Flow.StepExamples, + options?: StoreRunOptions, +): Promise; +export function runExamples( + kind: 'source', + step: Source.Init, + examples: Flow.StepExamples, + options?: SourceRunOptions, +): Promise; +export async function runExamples(...given: RunArgsGiven): Promise { + const args = complete(given); + const examples = args[2]; + const problems: string[] = []; + + if (args[0] === 'destination' && args[3].claims) + problems.push(...checkClaims(examples, args[3].claims)); + + for (const [name, example] of Object.entries(examples)) { + if (example.respond?.error === 'timeout' && timersAreFake()) + throw new Error( + `example ${name} waits for a timeout: call jest.useRealTimers() first`, + ); + for (const problem of await checkExample(args, example)) + problems.push(`example ${name}: ${problem}`); + } + + if (problems.length) throw new Error(problems.join('\n')); +} diff --git a/packages/collector/src/timeout.ts b/packages/collector/src/timeout.ts index 36e6190aa..d21f609df 100644 --- a/packages/collector/src/timeout.ts +++ b/packages/collector/src/timeout.ts @@ -1,22 +1,9 @@ import { StepError } from '@walkeros/core'; -/** - * Default per-destination delivery timeout in ms. Applied when a destination's - * `config.timeout` is `0` or undefined. A hung delivery is converted into a - * counted DLQ failure after this window so one slow destination never wedges - * the collector push. - */ -export const DEFAULT_DESTINATION_TIMEOUT_MS = 10_000; - -/** - * Resolve the effective delivery timeout for a destination. A positive number - * wins; `0` or undefined falls back to {@link DEFAULT_DESTINATION_TIMEOUT_MS}. - */ -export function resolveDestinationTimeout(timeout?: number): number { - return typeof timeout === 'number' && timeout > 0 - ? timeout - : DEFAULT_DESTINATION_TIMEOUT_MS; -} +export { + DEFAULT_DESTINATION_TIMEOUT_MS, + resolveDestinationTimeout, +} from '@walkeros/core'; /** * Error thrown when a destination delivery does not settle within its timeout: diff --git a/packages/collector/tsconfig.json b/packages/collector/tsconfig.json index b1ac14360..b2f814cc9 100644 --- a/packages/collector/tsconfig.json +++ b/packages/collector/tsconfig.json @@ -1,7 +1,8 @@ { "extends": "@walkeros/config/tsconfig/base.json", "compilerOptions": { - "rootDir": "src" + "rootDir": "src", + "module": "ES2022" }, "include": ["src/**/*"], "exclude": ["node_modules", "dist"] diff --git a/packages/config/eslint/__tests__/await-send.test.mjs b/packages/config/eslint/__tests__/await-send.test.mjs new file mode 100644 index 000000000..a852caac4 --- /dev/null +++ b/packages/config/eslint/__tests__/await-send.test.mjs @@ -0,0 +1,55 @@ +import { RuleTester } from 'eslint'; +import * as tsParser from '@typescript-eslint/parser'; +import awaitSend from '../rules/await-send.mjs'; + +const ruleTester = new RuleTester({ languageOptions: { parser: tsParser } }); +const floating = (name) => ({ messageId: 'floating', data: { name } }); + +ruleTester.run('walkeros/await-send', awaitSend, { + valid: [ + 'async function push() { await sendServer(url, b); }', + 'function push() { return sendServer(url, b); }', + 'async function push() { const res = await sendWeb(url, b); }', + 'async function push() { await env.sendServer(url, b); }', + 'async function push() { await (env.sendWeb || sendWeb)(url, b); }', + 'function push() { return sendServer(url, b).then(() => undefined); }', + "function push() { sendWeb(url, b, { transport: 'beacon' }); }", + "function push() { sendWeb(url, b, { transport: 'xhr' }); }", + 'async function push() { await Promise.all([sendServer(a), sendServer(b)]); }', + 'function push() { send(url, b); }', + ], + invalid: [ + { + code: 'async function push() { sendServer(url, b); }', + errors: [floating('sendServer')], + }, + { + code: 'async function push() { env.sendServer(url, b); }', + errors: [floating('sendServer')], + }, + { + code: 'async function push() { (env.sendWeb || sendWeb)(url, b); }', + errors: [floating('sendWeb')], + }, + { + code: 'async function push() { const sendWebFn = env.sendWeb || sendWeb; sendWebFn(url, b); }', + errors: [floating('sendWeb')], + }, + { + code: 'function send(url: string, sendWebFn: typeof sendWeb) { sendWebFn(url, b, { transport }); }', + errors: [floating('sendWeb')], + }, + { + code: 'async function push() { sendServer(url, b).catch(() => undefined); }', + errors: [floating('sendServer')], + }, + { + code: 'async function push() { void sendWeb(url, b); }', + errors: [floating('sendWeb')], + }, + { + code: "async function push() { sendWeb(url, b, { transport: 'fetch' }); }", + errors: [floating('sendWeb')], + }, + ], +}); diff --git a/packages/config/eslint/__tests__/no-silent-return.test.mjs b/packages/config/eslint/__tests__/no-silent-return.test.mjs new file mode 100644 index 000000000..7ec8def52 --- /dev/null +++ b/packages/config/eslint/__tests__/no-silent-return.test.mjs @@ -0,0 +1,52 @@ +import { RuleTester } from 'eslint'; +import * as tsParser from '@typescript-eslint/parser'; +import noSilentReturn from '../rules/no-silent-return.mjs'; + +const ruleTester = new RuleTester({ languageOptions: { parser: tsParser } }); +const silent = { messageId: 'silent' }; + +ruleTester.run('walkeros/no-silent-return', noSilentReturn, { + valid: [ + 'const d = { on(type) { if (!type) return; }, push(e) { send(e); } };', + 'const d = { push(e) { if (!e) throw new StepError("no event", { code: "missing-input" }); } };', + 'const d = { push(e) { items.forEach((item) => { if (!item) return; use(item); }); } };', + 'const d = { push(e) { return send(e); } };', + 'const d = { on(type) { if (!type) return undefined; } };', + 'const d = { init(context) { if (!context) return; } };', + 'function helper() { return; }', + ], + invalid: [ + { + code: 'const d = { push(event) { if (!url) return; send(event); } };', + errors: [silent], + }, + { + code: 'const d = { async pushBatch(batch) { if (!url) return; } };', + errors: [silent], + }, + { + code: 'const d = { push: async (event) => { if (!url) return; } };', + errors: [silent], + }, + { + code: 'export const push: PushFn = async function (event, ctx) { if (!ctx) return; };', + errors: [silent], + }, + { + code: 'async function pushBatch(batch) { return; }', + errors: [silent], + }, + { + code: 'const d = { push(event) { if (!url) return undefined; send(event); } };', + errors: [silent], + }, + { + code: 'const d = { async push(event) { if (!url) return void log(event); } };', + errors: [silent], + }, + { + code: 'class D { push(event) { if (!event) return; } }', + errors: [silent], + }, + ], +}); diff --git a/packages/config/eslint/index.mjs b/packages/config/eslint/index.mjs index 7cef72546..4016c9b86 100644 --- a/packages/config/eslint/index.mjs +++ b/packages/config/eslint/index.mjs @@ -2,6 +2,17 @@ import typescriptEslint from '@typescript-eslint/eslint-plugin'; import jest from 'eslint-plugin-jest'; import globals from 'globals'; import tsParser from '@typescript-eslint/parser'; +import awaitSend from './rules/await-send.mjs'; +import noSilentReturn from './rules/no-silent-return.mjs'; + +// walkerOS rules for the delivery contract. At `warn` while the step packages +// migrate onto it; `error` once every package is migrated. +const walkeros = { + rules: { + 'await-send': awaitSend, + 'no-silent-return': noSilentReturn, + }, +}; export default [ { @@ -18,6 +29,7 @@ export default [ plugins: { '@typescript-eslint': typescriptEslint, jest, + walkeros, }, languageOptions: { parser: tsParser, @@ -30,6 +42,7 @@ export default [ 'jest/no-disabled-tests': 'warn', 'jest/no-focused-tests': 'error', '@typescript-eslint/no-explicit-any': 'error', + 'walkeros/await-send': 'warn', 'no-restricted-syntax': [ 'error', { @@ -49,12 +62,19 @@ export default [ files: ['**/src/dev.ts', '**/examples/**', '**/schemas/**'], rules: { 'no-restricted-syntax': 'off' }, }, + { + // A destination's push and pushBatch never end with a bare return. + files: ['**/destinations/**/*.{js,ts,tsx}'], + rules: { 'walkeros/no-silent-return': 'warn' }, + }, { // Relaxed rules for test files files: ['**/*.test.{js,ts,tsx}', '**/__tests__/**/*.{js,ts,tsx}'], rules: { '@typescript-eslint/no-explicit-any': 'off', 'no-console': 'off', + 'walkeros/await-send': 'off', + 'walkeros/no-silent-return': 'off', }, }, ]; diff --git a/packages/config/eslint/rules/await-send.mjs b/packages/config/eslint/rules/await-send.mjs new file mode 100644 index 000000000..76f965b95 --- /dev/null +++ b/packages/config/eslint/rules/await-send.mjs @@ -0,0 +1,148 @@ +/** + * walkeros/await-send: a call to a transport helper (`sendServer`, `sendWeb`, + * `sendWebAsFetch`) must be awaited, returned or otherwise used. A floating + * send lets the push settle before the request did, so its failure is lost + * and the event counts as delivered. + * + * AST only: the helper is recognized by name, as a member (`env.sendServer`), + * through a fallback (`env.sendWeb || sendWeb`) and through a local alias or + * a parameter typed `typeof sendWeb`. A `sendWeb` call whose options name the + * synchronous `beacon` or `xhr` transport returns its result and throws at + * once, so it may stand alone. + */ + +const HELPERS = ['sendServer', 'sendWeb', 'sendWebAsFetch']; +const SYNC_TRANSPORTS = ['beacon', 'xhr']; +const CHAIN_METHODS = ['then', 'catch', 'finally']; +const WRAPPERS = [ + 'ChainExpression', + 'TSNonNullExpression', + 'TSAsExpression', + 'TSSatisfiesExpression', +]; + +function propertyName(node) { + if (!node) return undefined; + if (node.type === 'Identifier') return node.name; + if (node.type === 'Literal' && typeof node.value === 'string') + return node.value; + return undefined; +} + +/** The helper name a callee refers to, or undefined. */ +function helperOf(node, aliases) { + if (!node) return undefined; + switch (node.type) { + case 'Identifier': + if (HELPERS.includes(node.name)) return node.name; + return aliases.get(node.name); + case 'MemberExpression': { + if (node.computed && node.property.type !== 'Literal') return undefined; + const name = propertyName(node.property); + return HELPERS.includes(name) ? name : undefined; + } + case 'LogicalExpression': + return helperOf(node.left, aliases) || helperOf(node.right, aliases); + case 'ConditionalExpression': + return ( + helperOf(node.consequent, aliases) || helperOf(node.alternate, aliases) + ); + default: + if (WRAPPERS.includes(node.type)) + return helperOf(node.expression, aliases); + return undefined; + } +} + +/** The helper a `typeof ` type annotation names, or undefined. */ +function typeofHelper(annotation) { + const type = annotation && annotation.typeAnnotation; + if (!type || type.type !== 'TSTypeQuery') return undefined; + const name = type.exprName && type.exprName.name; + return HELPERS.includes(name) ? name : undefined; +} + +/** Whether a sendWeb call's options select a synchronous transport. */ +function isSyncTransport(call) { + const options = call.arguments[2]; + if (!options || options.type !== 'ObjectExpression') return false; + return options.properties.some( + (property) => + property.type === 'Property' && + propertyName(property.key) === 'transport' && + property.value.type === 'Literal' && + SYNC_TRANSPORTS.includes(property.value.value), + ); +} + +/** Whether the value of an expression is thrown away (a floating promise). */ +function isDiscarded(node) { + let current = node; + let parent = current.parent; + while (parent && WRAPPERS.includes(parent.type)) { + current = parent; + parent = current.parent; + } + if (!parent) return false; + if (parent.type === 'ExpressionStatement') return true; + if (parent.type === 'UnaryExpression' && parent.operator === 'void') + return true; + // A `.then()`, `.catch()` or `.finally()` chain floats when its end does. + if ( + parent.type === 'MemberExpression' && + parent.object === current && + CHAIN_METHODS.includes(propertyName(parent.property)) && + parent.parent && + parent.parent.type === 'CallExpression' && + parent.parent.callee === parent + ) + return isDiscarded(parent.parent); + return false; +} + +export default { + meta: { + type: 'problem', + docs: { + description: + 'Require transport helper calls (sendServer, sendWeb) to be awaited or returned', + }, + schema: [], + messages: { + floating: + 'Await or return this {{name}} call: a floating send settles after the push, so its failure is lost and the event counts as delivered.', + }, + }, + create(context) { + const aliases = new Map(); + + function addParams(fn) { + for (const param of fn.params) { + const helper = + param.type === 'Identifier' + ? typeofHelper(param.typeAnnotation) + : undefined; + if (helper) aliases.set(param.name, helper); + } + } + + return { + VariableDeclarator(node) { + if (node.id.type !== 'Identifier' || !node.init) return; + const helper = + helperOf(node.init, aliases) || typeofHelper(node.id.typeAnnotation); + if (helper) aliases.set(node.id.name, helper); + }, + FunctionDeclaration: addParams, + FunctionExpression: addParams, + ArrowFunctionExpression: addParams, + CallExpression(node) { + const name = helperOf(node.callee, aliases); + if (!name) return; + if (name !== 'sendServer' && isSyncTransport(node)) return; + if (isDiscarded(node)) + context.report({ node, messageId: 'floating', data: { name } }); + }, + }; + }, +}; diff --git a/packages/config/eslint/rules/no-silent-return.mjs b/packages/config/eslint/rules/no-silent-return.mjs new file mode 100644 index 000000000..cd8204bf6 --- /dev/null +++ b/packages/config/eslint/rules/no-silent-return.mjs @@ -0,0 +1,79 @@ +/** + * walkeros/no-silent-return: a bare `return;` (or `return undefined;`, + * `return void x;`) inside a destination's `push` or `pushBatch` ends the + * push as if the event was delivered, so a skipped event is counted as sent + * and nothing is logged. Throw a `StepError` that + * names why (a missing setting is `config`, a missing input `missing-input`) + * instead. Other functions (`init`, `on`, helpers) are not checked. + * + * AST only: a push function is the value of a `push` or `pushBatch` + * property or method, a variable of that name, or a function declaration of + * that name. Returns inside functions nested in it belong to those. + */ + +const PUSH_NAMES = ['push', 'pushBatch']; + +function keyName(node) { + if (!node) return undefined; + if (node.type === 'Identifier') return node.name; + if (node.type === 'Literal' && typeof node.value === 'string') + return node.value; + return undefined; +} + +/** A return value that is nothing: none, `undefined` or a `void` expression. */ +function isSilent(argument) { + if (argument === null) return true; + if (argument.type === 'Identifier') return argument.name === 'undefined'; + return argument.type === 'UnaryExpression' && argument.operator === 'void'; +} + +/** Whether a function node is a destination's push or pushBatch. */ +function isPushFunction(fn) { + if (fn.type === 'FunctionDeclaration') + return PUSH_NAMES.includes(fn.id && fn.id.name); + const parent = fn.parent; + if (!parent) return false; + if ( + (parent.type === 'Property' || parent.type === 'MethodDefinition') && + parent.value === fn && + !parent.computed + ) + return PUSH_NAMES.includes(keyName(parent.key)); + if (parent.type === 'VariableDeclarator' && parent.init === fn) + return PUSH_NAMES.includes(keyName(parent.id)); + return false; +} + +export default { + meta: { + type: 'problem', + docs: { + description: + "Disallow a bare return in a destination's push or pushBatch", + }, + schema: [], + messages: { + silent: + 'A bare return ends this push as delivered: throw a StepError that names why the event is not sent (code config or missing-input).', + }, + }, + create(context) { + const stack = []; + const enter = (fn) => stack.push(isPushFunction(fn)); + const exit = () => stack.pop(); + + return { + FunctionDeclaration: enter, + FunctionExpression: enter, + ArrowFunctionExpression: enter, + 'FunctionDeclaration:exit': exit, + 'FunctionExpression:exit': exit, + 'ArrowFunctionExpression:exit': exit, + ReturnStatement(node) { + if (isSilent(node.argument) && stack[stack.length - 1] === true) + context.report({ node, messageId: 'silent' }); + }, + }; + }, +}; diff --git a/packages/config/jest.config.mjs b/packages/config/jest.config.mjs index b6aa29c58..d52a9dca8 100644 --- a/packages/config/jest.config.mjs +++ b/packages/config/jest.config.mjs @@ -3,5 +3,8 @@ import baseConfig from '@walkeros/config/jest'; export default { ...baseConfig, displayName: 'config', - testMatch: ['/tsup/__tests__/**/*.test.(ts|tsx|js|jsx|mjs)'], + testMatch: [ + '/tsup/__tests__/**/*.test.(ts|tsx|js|jsx|mjs)', + '/eslint/__tests__/**/*.test.mjs', + ], }; diff --git a/packages/config/package.json b/packages/config/package.json index 066ce7733..425d96872 100644 --- a/packages/config/package.json +++ b/packages/config/package.json @@ -6,9 +6,10 @@ "license": "MIT", "files": [ "tsconfig/**/*.json", - "eslint/**/*.mjs", + "eslint/*.mjs", + "eslint/rules/*.mjs", "jest/**/*.mjs", - "tsup/**/*.mjs", + "tsup/*.mjs", "tsup/**/*.d.ts", "CHANGELOG.md" ], @@ -26,6 +27,9 @@ "./jest/node.setup": "./jest/node.setup.mjs", "./tsup": "./tsup/index.mjs" }, + "scripts": { + "test": "jest" + }, "dependencies": { "@swc/core": "^1.11.13", "@swc/jest": "^0.2.37", diff --git a/packages/config/tsup/__tests__/serverBanner.test.mjs b/packages/config/tsup/__tests__/serverBanner.test.mjs new file mode 100644 index 000000000..54de331e5 --- /dev/null +++ b/packages/config/tsup/__tests__/serverBanner.test.mjs @@ -0,0 +1,42 @@ +import { mkdtempSync, rmSync, writeFileSync } from 'fs'; +import { tmpdir } from 'os'; +import { join } from 'path'; +import { buildModules } from '../index.mjs'; + +describe('buildModules createRequire banner', () => { + let dir; + + const configFor = (walkerOS) => { + dir = mkdtempSync(join(tmpdir(), 'walkeros-banner-')); + writeFileSync( + join(dir, 'package.json'), + JSON.stringify({ name: 'pkg', version: '1.0.0', walkerOS }), + ); + jest.spyOn(process, 'cwd').mockReturnValue(dir); + return buildModules(); + }; + + afterEach(() => { + jest.restoreAllMocks(); + rmSync(dir, { recursive: true, force: true }); + }); + + it('adds createRequire to the ESM build of a server package', () => { + const config = configFor({ type: 'destination', platform: 'server' }); + expect(config.banner({ format: 'esm' }).js).toContain('createRequire'); + expect(config.banner({ format: 'cjs' }).js).toBeUndefined(); + }); + + it('reads a platform list', () => { + const config = configFor({ type: 'destination', platform: ['server'] }); + expect(config.banner({ format: 'esm' }).js).toContain('createRequire'); + }); + + it.each([ + ['a web package', { type: 'destination', platform: ['web'] }], + ['a package for web and server', { platform: ['web', 'server'] }], + ['a package without a platform', undefined], + ])('adds no banner to %s', (_, walkerOS) => { + expect(configFor(walkerOS).banner).toBeUndefined(); + }); +}); diff --git a/packages/config/tsup/index.mjs b/packages/config/tsup/index.mjs index 1469de205..9a400b51c 100644 --- a/packages/config/tsup/index.mjs +++ b/packages/config/tsup/index.mjs @@ -30,6 +30,34 @@ const getVersion = () => { } }; +// The package's walkerOS.platform list (package.json), empty when unset +const getPlatforms = () => { + try { + const packagePath = resolve(process.cwd(), 'package.json'); + const platform = JSON.parse(readFileSync(packagePath, 'utf8')).walkerOS + ?.platform; + if (Array.isArray(platform)) return platform; + return platform ? [platform] : []; + } catch { + return []; + } +}; + +// ESM builds of server-only packages define `require`, so the CommonJS SDKs +// they load with require() work when the ESM build is imported directly. +// Never for a package a browser imports: `module` does not exist there. +const CREATE_REQUIRE_BANNER = + "import { createRequire } from 'module';const require = createRequire(import.meta.url);"; + +const serverBanner = () => { + const platforms = getPlatforms(); + if (!platforms.includes('server') || platforms.includes('web')) return {}; + return { + banner: ({ format }) => + format === 'esm' ? { js: CREATE_REQUIRE_BANNER } : {}, + }; +}; + // Modules const buildModules = (customConfig = {}) => { // Auto-inject package version @@ -46,6 +74,7 @@ const buildModules = (customConfig = {}) => { sourcemap: true, declaration: true, declarationMap: true, + ...serverBanner(), define: { __VERSION__: JSON.stringify(version), ...customConfig.define, diff --git a/packages/core/src/__tests__/envelope.test.ts b/packages/core/src/__tests__/envelope.test.ts index 84ac1b2d2..54e5f4bff 100644 --- a/packages/core/src/__tests__/envelope.test.ts +++ b/packages/core/src/__tests__/envelope.test.ts @@ -3,7 +3,9 @@ import { isBatchBody, batchResponse, pushResultToOutcome, + acknowledge, } from '../envelope'; +import type { Elb } from '../types'; describe('toEventList', () => { it.each([ @@ -120,3 +122,78 @@ describe('pushResultToOutcome', () => { expect(pushResultToOutcome(result)).toEqual(expected); }); }); + +describe('acknowledge', () => { + const ref = (code: string, retryable: boolean) => ({ + type: 'api', + reason: { code, message: code, retryable }, + }); + + it.each<[string, Elb.PushResult, 'ack' | 'retry' | 'reject']>([ + ['invalid input', { ok: false, invalid: true }, 'reject'], + ['a delivered event', { ok: true, done: { api: { type: 'api' } } }, 'ack'], + [ + 'a queued event', + { + ok: true, + queued: { + api: { type: 'api', reason: { code: 'batch', message: 'batched' } }, + }, + }, + 'ack', + ], + [ + 'deterministic failures only (ruling 1)', + { + ok: false, + failed: { a: ref('config', false), b: ref('transport', false) }, + }, + 'ack', + ], + [ + 'one retryable failure among deterministic ones', + { + ok: false, + failed: { a: ref('config', false), b: ref('timeout', true) }, + }, + 'retry', + ], + [ + 'a retryable reason without a failed entry', + { + ok: false, + reason: { code: 'queue-full', message: 'full', retryable: true }, + }, + 'retry', + ], + [ + 'a deterministic reason without a failed entry', + { + ok: false, + reason: { + code: 'mapping', + message: 'condition threw', + retryable: false, + }, + }, + 'ack', + ], + [ + 'an error text that names queue-full while the reason is mapping (strings are never read)', + { + ok: false, + error: 'queue-full', + reason: { code: 'mapping', message: 'queue-full', retryable: false }, + }, + 'ack', + ], + ['a failure without any reason', { ok: false }, 'retry'], + [ + 'a failed entry without a reason', + { ok: false, failed: { a: { type: 'api' } } }, + 'retry', + ], + ])('%s', (_, result, expected) => { + expect(acknowledge(result)).toBe(expected); + }); +}); diff --git a/packages/core/src/__tests__/stepError.test.ts b/packages/core/src/__tests__/stepError.test.ts index ef65cdfc3..d7cc19463 100644 --- a/packages/core/src/__tests__/stepError.test.ts +++ b/packages/core/src/__tests__/stepError.test.ts @@ -1,4 +1,4 @@ -import { StepError, toFailureReason } from '../stepError'; +import { networkErrorMessage, StepError, toFailureReason } from '../stepError'; describe('StepError', () => { it('carries code, status, retryable and the response body', () => { @@ -89,3 +89,34 @@ describe('StepError', () => { ); }); }); + +describe('networkErrorMessage', () => { + const withCause = (cause: unknown) => + new TypeError('fetch failed', { cause }); + + it.each<[string, unknown, string]>([ + [ + 'a cause code, never the cause message', + withCause( + Object.assign(new Error('connect ECONNREFUSED 127.0.0.1:9?key=s'), { + code: 'ECONNREFUSED', + }), + ), + 'fetch failed (ECONNREFUSED)', + ], + [ + 'a cause without a code', + withCause(new Error('socket hang up')), + 'fetch failed', + ], + [ + 'a cause code that is not a string', + withCause({ code: 13 }), + 'fetch failed', + ], + ['no cause', new TypeError('Failed to fetch'), 'Failed to fetch'], + ['a thrown value that is not an error', 'refused', 'Network error'], + ])('%s', (_, error, expected) => { + expect(networkErrorMessage(error)).toBe(expected); + }); +}); diff --git a/packages/core/src/envelope.ts b/packages/core/src/envelope.ts index f25b9f88a..ad958f5c5 100644 --- a/packages/core/src/envelope.ts +++ b/packages/core/src/envelope.ts @@ -1,4 +1,4 @@ -import type { WalkerOS } from './types'; +import type { Elb, WalkerOS } from './types'; import { isArray, isObject } from './is'; /** @@ -129,3 +129,33 @@ export function pushResultToOutcome(result: { } return { id: result?.event?.id }; } + +/** What a producer does with the message that carried a push. */ +export type Acknowledgement = 'ack' | 'retry' | 'reject'; + +/** + * Decides from a push result whether a producer acks, retries or rejects the + * message that carried it: invalid input is `reject`, a delivered or queued + * event `ack`. A failure is `retry` when any of its reasons is retryable and + * `ack` when every reason is deterministic (a redelivery cannot fix a missing + * setting or a 4xx; the event is in the DLQ and its cause line is logged). A + * failure without any reason is `retry`, so an unclassified failure never + * deletes a message. Only `retryable` is read, never an error text. + * + * @param result The push result. + * @returns `ack`, `retry` or `reject`. + */ +export function acknowledge(result: Elb.PushResult): Acknowledgement { + if (result.invalid === true) return 'reject'; + if (result.ok) return 'ack'; + + const retryable: boolean[] = []; + for (const ref of Object.values(result.failed ?? {})) { + const reason = ref.reason; + retryable.push(reason && 'retryable' in reason ? reason.retryable : true); + } + if (result.reason) retryable.push(result.reason.retryable); + + if (retryable.length === 0) return 'retry'; + return retryable.some(Boolean) ? 'retry' : 'ack'; +} diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index c9604fef4..d2ec3c4d3 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -36,7 +36,7 @@ export * from './trim'; export * from './tryCatch'; export * from './fatalError'; export * from './invalidEventError'; -export { StepError, toFailureReason } from './stepError'; +export { networkErrorMessage, StepError, toFailureReason } from './stepError'; export type { FailureReason, QueueReason, @@ -44,6 +44,10 @@ export type { StepErrorOptions, } from './stepError'; export { createCauseLog, reportCause } from './causeLog'; +export { + DEFAULT_DESTINATION_TIMEOUT_MS, + resolveDestinationTimeout, +} from './timeout'; export type { CauseKey, CauseLog, diff --git a/packages/core/src/schemas/flow.ts b/packages/core/src/schemas/flow.ts index fe9b0bf58..5d013ebee 100644 --- a/packages/core/src/schemas/flow.ts +++ b/packages/core/src/schemas/flow.ts @@ -240,6 +240,27 @@ export const StepExampleSchema = z .describe( "Invoke elb('walker ', in) instead of pushing in as an event", ), + outcome: z + .object({ + status: z.enum(['failed', 'queued']).describe('Status in the result'), + code: z.string().describe('Reason code, e.g. transport or config'), + retryable: z.boolean().optional().describe('Whether a retry can help'), + }) + .optional() + .describe( + 'Reported outcome of a failure or queue example (public: false)', + ), + respond: z + .object({ + status: z.number().optional().describe('HTTP status to answer'), + body: z.unknown().optional().describe('Response body to answer'), + error: z + .enum(['network', 'timeout']) + .optional() + .describe('A network error or a timeout instead of a response'), + }) + .optional() + .describe('What the boundary answers for this example'), }) .meta({ id: 'FlowStepExample', diff --git a/packages/core/src/stepError.ts b/packages/core/src/stepError.ts index 6570d189f..976eaaf6b 100644 --- a/packages/core/src/stepError.ts +++ b/packages/core/src/stepError.ts @@ -109,3 +109,21 @@ export function toFailureReason(err: unknown): FailureReason { retryable: true, }; } + +/** + * A network error's message with its cause code (`ECONNREFUSED`, `ENOTFOUND`), + * which runtimes keep on `cause` (Node's fetch says only `fetch failed`): the + * code only, never the cause's message, which carries addresses and URLs. + */ +export function networkErrorMessage(error: unknown): string { + const message = error instanceof Error ? error.message : 'Network error'; + const cause = error instanceof Error ? error.cause : undefined; + const code = + typeof cause === 'object' && + cause !== null && + 'code' in cause && + typeof cause.code === 'string' + ? cause.code + : undefined; + return code ? `${message} (${code})` : message; +} diff --git a/packages/core/src/timeout.ts b/packages/core/src/timeout.ts new file mode 100644 index 000000000..610bdaf12 --- /dev/null +++ b/packages/core/src/timeout.ts @@ -0,0 +1,19 @@ +/** + * Default per-destination delivery timeout in ms. Applied when a destination's + * `config.timeout` is `0` or undefined. A hung delivery is converted into a + * counted DLQ failure after this window so one slow destination never wedges + * the collector push. + */ +export const DEFAULT_DESTINATION_TIMEOUT_MS = 10_000; + +/** + * Resolve the effective delivery timeout for a destination. A positive number + * wins; `0` or undefined falls back to {@link DEFAULT_DESTINATION_TIMEOUT_MS}. + * The collector races deliveries with it and `loadVendor` measures its vendor + * deadline with it, so both use one definition. + */ +export function resolveDestinationTimeout(timeout?: number): number { + return typeof timeout === 'number' && timeout > 0 + ? timeout + : DEFAULT_DESTINATION_TIMEOUT_MS; +} diff --git a/packages/core/src/types/flow.ts b/packages/core/src/types/flow.ts index a756996f1..55b23c8b6 100644 --- a/packages/core/src/types/flow.ts +++ b/packages/core/src/types/flow.ts @@ -816,6 +816,31 @@ export namespace Flow { * When absent (default), `in` is pushed as a regular event. */ command?: StepCommand; + /** + * The reported outcome of a failure or queue example (`public: false`): + * the step's status in the push result and its reason code. Absent means + * the step must not fail. + */ + outcome?: StepOutcome; + /** What the boundary answers for this example (a failure fixture). */ + respond?: StepRespond; + } + + /** How a failure or queue example is reported in the push result. */ + export interface StepOutcome { + status: 'failed' | 'queued'; + code: string; + retryable?: boolean; + } + + /** + * The boundary's answer to the request an example makes: an HTTP status and + * body, or a network error or a timeout instead of a response. + */ + export interface StepRespond { + status?: number; + body?: unknown; + error?: 'network' | 'timeout'; } /** Named step examples keyed by scenario name. */ diff --git a/packages/core/src/types/send.ts b/packages/core/src/types/send.ts index 74537fb32..2d266fed7 100644 --- a/packages/core/src/types/send.ts +++ b/packages/core/src/types/send.ts @@ -3,8 +3,11 @@ import type { WalkerOS } from '.'; export type SendDataValue = WalkerOS.Property | WalkerOS.Properties; export type SendHeaders = { [key: string]: string }; +/** + * What a transport helper (`sendWeb`, `sendServer`) resolves on success: the + * response body, parsed as JSON where it parses. A failure throws a + * `StepError` (code, status, retryable, body) instead. + */ export interface SendResponse { - ok: boolean; data?: unknown; - error?: string; } diff --git a/packages/server/core/src/__tests__/sendServer.test.ts b/packages/server/core/src/__tests__/sendServer.test.ts new file mode 100644 index 000000000..4a41895d0 --- /dev/null +++ b/packages/server/core/src/__tests__/sendServer.test.ts @@ -0,0 +1,140 @@ +import { StepError } from '@walkeros/core'; +import { sendServer } from '../sendServer'; + +describe('sendServer on the global fetch', () => { + afterEach(() => jest.restoreAllMocks()); + + it('throws a StepError with status, retryable and body for a 503 from a stubbed global fetch', async () => { + jest + .spyOn(globalThis, 'fetch') + .mockResolvedValue(new Response('{"error":"busy"}', { status: 503 })); + const sent = sendServer('https://vendor.example/collect', { + name: 'page view', + }); + await expect(sent).rejects.toBeInstanceOf(StepError); + await expect(sent).rejects.toMatchObject({ + code: 'transport', + status: 503, + retryable: true, + body: { error: 'busy' }, + }); + }); + + it('throws a non-retryable StepError for a 400, the body never in the message', async () => { + jest.spyOn(globalThis, 'fetch').mockResolvedValue( + new Response('{"error":"invalid value user@example.com"}', { + status: 400, + statusText: 'Bad Request user@example.com', + }), + ); + const sent = sendServer('https://vendor.example/collect', {}); + await expect(sent).rejects.toMatchObject({ + code: 'transport', + status: 400, + retryable: false, + message: 'HTTP 400', + }); + }); + + it('keeps the cause code of a network error in its message, never the address', async () => { + const cause = Object.assign( + new Error('connect ECONNREFUSED 127.0.0.1:9?key=secret'), + { code: 'ECONNREFUSED' }, + ); + jest + .spyOn(globalThis, 'fetch') + .mockRejectedValue(new TypeError('fetch failed', { cause })); + await expect( + sendServer('https://vendor.example/collect', {}), + ).rejects.toMatchObject({ + code: 'transport', + message: 'fetch failed (ECONNREFUSED)', + }); + }); + + it('throws code transport for a network error', async () => { + jest + .spyOn(globalThis, 'fetch') + .mockRejectedValue(new TypeError('fetch failed')); + await expect( + sendServer('https://vendor.example/collect', {}), + ).rejects.toMatchObject({ code: 'transport', retryable: true }); + }); + + it('resolves with the parsed body on a 2xx', async () => { + jest + .spyOn(globalThis, 'fetch') + .mockResolvedValue(new Response('{"id":"e1"}', { status: 200 })); + await expect( + sendServer('https://vendor.example/collect', { name: 'page view' }), + ).resolves.toEqual({ data: { id: 'e1' } }); + }); + + it('resolves with the text of a body that is not JSON', async () => { + jest + .spyOn(globalThis, 'fetch') + .mockResolvedValue(new Response('accepted', { status: 202 })); + await expect( + sendServer('https://vendor.example/collect', {}), + ).resolves.toEqual({ data: 'accepted' }); + }); + + it('times out a fetch that never settles, driven by fake timers', async () => { + jest.useFakeTimers(); + jest + .spyOn(globalThis, 'fetch') + .mockReturnValue(new Promise(() => undefined)); + const sent = sendServer( + 'https://vendor.example/collect', + {}, + { timeout: 1000 }, + ); + jest.advanceTimersByTime(1000); + await expect(sent).rejects.toMatchObject({ + code: 'timeout', + retryable: true, + }); + jest.useRealTimers(); + }); + + it('reports an invalid URL as config, not retryable', async () => { + await expect(sendServer('not a url', {})).rejects.toMatchObject({ + code: 'config', + retryable: false, + }); + }); + + it('sends no body for GET', async () => { + const spy = jest + .spyOn(globalThis, 'fetch') + .mockResolvedValue(new Response('', { status: 200 })); + await sendServer( + 'https://vendor.example/collect', + { a: 1 }, + { + method: 'GET', + }, + ); + expect(spy.mock.calls[0][1]?.body).toBeUndefined(); + }); + + it('sends the JSON body with the method and headers on POST', async () => { + const spy = jest + .spyOn(globalThis, 'fetch') + .mockResolvedValue(new Response('', { status: 200 })); + await sendServer( + 'https://vendor.example/collect', + { a: 1 }, + { headers: { 'X-Key': 'k' } }, + ); + expect(spy.mock.calls[0][0]).toBe('https://vendor.example/collect'); + expect(spy.mock.calls[0][1]).toMatchObject({ + method: 'POST', + body: '{"a":1}', + headers: { + 'Content-Type': 'application/json; charset=utf-8', + 'X-Key': 'k', + }, + }); + }); +}); diff --git a/packages/server/core/src/sendServer.ts b/packages/server/core/src/sendServer.ts index ce9010653..d90b3790f 100644 --- a/packages/server/core/src/sendServer.ts +++ b/packages/server/core/src/sendServer.ts @@ -1,7 +1,11 @@ import type { SendDataValue, SendHeaders, SendResponse } from '@walkeros/core'; -import { getHeaders, transformData, tryCatch } from '@walkeros/core'; -import * as http from 'http'; -import * as https from 'https'; +import { + getHeaders, + networkErrorMessage, + StepError, + transformData, + tryCatch, +} from '@walkeros/core'; export interface SendServerOptions { headers?: SendHeaders; @@ -9,71 +13,84 @@ export interface SendServerOptions { timeout?: number; } -export function sendServer( +/** Methods whose request carries no body (fetch refuses one). */ +const NO_BODY_METHODS = ['GET', 'HEAD']; + +/** + * Send a request with the global `fetch`. Resolves `{ data }` (the body, + * parsed as JSON where it parses) for a 2xx answer and throws a `StepError` + * otherwise: `transport` with `status` and `body` for any other status or a + * network error, `timeout` when no answer arrives within `options.timeout` + * (default 5000 ms), `config` for an invalid URL. The body stays on the error, + * never in its message. + */ +export async function sendServer( url: string, data?: SendDataValue, options: SendServerOptions = {}, ): Promise { const headers = getHeaders(options.headers); - const body = transformData(data); const method = options.method || 'POST'; const timeout = options.timeout || 5000; + const body = NO_BODY_METHODS.includes(method.toUpperCase()) + ? undefined + : transformData(data); - return new Promise((resolve) => { - const urlObj = new URL(url); - const lib = urlObj.protocol === 'https:' ? https : http; - const options: http.RequestOptions | https.RequestOptions = { - method, - headers, - }; - - const req = lib.request(urlObj, options, (res) => { - const chunks: Uint8Array[] = []; - - res.on('data', (chunk) => { - chunks.push(chunk); - }); + try { + new URL(url); + } catch (err) { + throw new StepError('Invalid request URL', { + code: 'config', + retryable: false, + cause: err, + }); + } - res.on('end', () => { - const ok = !!( - res.statusCode && - res.statusCode >= 200 && - res.statusCode < 300 - ); + // Own timer, so a request that ignores the abort signal still settles. + const controller = new AbortController(); + let timer: ReturnType | undefined; + const timedOut = new Promise((_, reject) => { + timer = setTimeout(() => { + controller.abort(); + reject( + new StepError(`Request timed out after ${timeout}ms`, { + code: 'timeout', + }), + ); + }, timeout); + }); - const responseData = Buffer.concat(chunks).toString(); - const parsedData = tryCatch( - JSON.parse, - () => responseData, - )(responseData); + try { + return await Promise.race([ + request(url, { method, headers, body, signal: controller.signal }), + timedOut, + ]); + } finally { + clearTimeout(timer); + } +} - resolve({ - ok, - data: parsedData, - error: ok ? undefined : `${res.statusCode} ${res.statusMessage}`, - }); - }); +async function request(url: string, init: RequestInit): Promise { + let response: Response; + try { + // Read at call time, so a replaced global fetch is the one used. + response = await globalThis.fetch(url, init); + } catch (err) { + throw new StepError(networkErrorMessage(err), { + code: 'transport', + cause: err, }); + } - req.on('error', (error) => { - resolve({ - ok: false, - error: error.message, - }); - }); + const text = await response.text(); + const parsed: unknown = tryCatch(JSON.parse, () => text)(text); - req.on('timeout', () => { - req.destroy(); - resolve({ - ok: false, - error: 'Request timeout', - }); + if (!response.ok) + throw new StepError(`HTTP ${response.status}`, { + code: 'transport', + status: response.status, + body: parsed, }); - req.setTimeout(timeout); - - if (body) req.write(body); - - req.end(); - }); + return { data: parsed }; } diff --git a/packages/server/destinations/api/package.json b/packages/server/destinations/api/package.json index fa2edca47..6057525d3 100644 --- a/packages/server/destinations/api/package.json +++ b/packages/server/destinations/api/package.json @@ -41,7 +41,9 @@ "@walkeros/core": "4.8.1", "@walkeros/server-core": "4.8.1" }, - "devDependencies": {}, + "devDependencies": { + "@walkeros/collector": "4.8.1" + }, "repository": { "url": "git+https://github.com/elbwalker/walkerOS.git", "directory": "packages/server/destinations/api" diff --git a/packages/server/destinations/api/src/__tests__/stepExamples.test.ts b/packages/server/destinations/api/src/__tests__/stepExamples.test.ts index 837c454b9..597d94007 100644 --- a/packages/server/destinations/api/src/__tests__/stepExamples.test.ts +++ b/packages/server/destinations/api/src/__tests__/stepExamples.test.ts @@ -19,7 +19,7 @@ describe('Step Examples', () => { beforeEach(() => { jest.clearAllMocks(); - mockSendServer.mockResolvedValue({ ok: true }); + mockSendServer.mockResolvedValue({}); }); it.each(Object.entries(examples.step))('%s', async (name, example) => { diff --git a/packages/server/destinations/api/src/examples/env.ts b/packages/server/destinations/api/src/examples/env.ts index 4c15e2b69..ff0d4bebd 100644 --- a/packages/server/destinations/api/src/examples/env.ts +++ b/packages/server/destinations/api/src/examples/env.ts @@ -7,8 +7,7 @@ import type { Env } from '../types'; * and development without requiring external dependencies. */ -const sendServer: NonNullable = () => - Promise.resolve({ ok: true }); +const sendServer: NonNullable = () => Promise.resolve({}); export const init: Env | undefined = { sendServer: undefined, diff --git a/packages/server/destinations/api/src/index.test.ts b/packages/server/destinations/api/src/index.test.ts index 15074c396..5c5d32af7 100644 --- a/packages/server/destinations/api/src/index.test.ts +++ b/packages/server/destinations/api/src/index.test.ts @@ -207,11 +207,6 @@ describe('Destination API', () => { eventName: event.name, }), ); - - expect(testLogger.debug).toHaveBeenCalledWith( - 'API destination response', - expect.objectContaining({ ok: undefined }), - ); }); describe('pushBatch', () => { @@ -370,11 +365,6 @@ describe('Destination API', () => { 'API destination sending batch', expect.objectContaining({ url, method: 'PUT', events: 2 }), ); - - expect(testLogger.debug).toHaveBeenCalledWith( - 'API destination response', - expect.objectContaining({ ok: undefined }), - ); }); }); }); diff --git a/packages/server/destinations/api/src/index.ts b/packages/server/destinations/api/src/index.ts index b255bbedb..bbfc478bc 100644 --- a/packages/server/destinations/api/src/index.ts +++ b/packages/server/destinations/api/src/index.ts @@ -1,4 +1,4 @@ -import type { Logger, SendDataValue } from '@walkeros/core'; +import type { SendDataValue } from '@walkeros/core'; import type { Settings, Destination, Env } from './types'; import { isDefined } from '@walkeros/core'; import { sendServer } from '@walkeros/server-core'; @@ -10,14 +10,11 @@ async function send( body: SendDataValue, settings: Settings, env: Env | undefined, - logger?: Logger.Instance, ): Promise { const { url, headers, method, timeout } = settings; const sendServerFn = env?.sendServer || sendServer; - const response = await sendServerFn(url, body, { headers, method, timeout }); - - logger?.debug('API destination response', { ok: response?.ok }); + await sendServerFn(url, body, { headers, method, timeout }); } export const destinationAPI: Destination = { @@ -44,7 +41,7 @@ export const destinationAPI: Destination = { eventName: event.name, }); - await send(body, { ...settings, url }, env, logger); + await send(body, { ...settings, url }, env); }, async pushBatch(batch, { config, rule, env, logger }) { @@ -68,7 +65,7 @@ export const destinationAPI: Destination = { events: items.length, }); - await send(JSON.stringify(payload), { ...settings, url }, env, logger); + await send(JSON.stringify(payload), { ...settings, url }, env); }, }; diff --git a/packages/server/destinations/aws/package.json b/packages/server/destinations/aws/package.json index 5561ecb3b..9ebf94791 100644 --- a/packages/server/destinations/aws/package.json +++ b/packages/server/destinations/aws/package.json @@ -42,7 +42,9 @@ "@walkeros/core": "4.8.1", "@walkeros/server-core": "4.8.1" }, - "devDependencies": {}, + "devDependencies": { + "@walkeros/collector": "4.8.1" + }, "repository": { "url": "git+https://github.com/elbwalker/walkerOS.git", "directory": "packages/server/destinations/aws" diff --git a/packages/server/destinations/bing/src/__tests__/index.test.ts b/packages/server/destinations/bing/src/__tests__/index.test.ts index 61679d3e6..9e2594a29 100644 --- a/packages/server/destinations/bing/src/__tests__/index.test.ts +++ b/packages/server/destinations/bing/src/__tests__/index.test.ts @@ -7,6 +7,7 @@ import { createMockContext, createMockLogger, getEvent, + StepError, } from '@walkeros/core'; import { startFlow } from '@walkeros/collector'; import { examples } from '../dev'; @@ -28,7 +29,6 @@ describe('Server Destination Bing', () => { jest.clearAllMocks(); mockSendServer.mockResolvedValue({ - ok: true, data: JSON.stringify({ status: 'OK', requestId: 'req-1' }), }); @@ -367,7 +367,6 @@ describe('Server Destination Bing', () => { test('environment customization', async () => { const customSendServer = jest.fn(); customSendServer.mockResolvedValue({ - ok: true, data: JSON.stringify({ status: 'OK' }), }); @@ -391,10 +390,9 @@ describe('Server Destination Bing', () => { }); test('error response throws', async () => { - mockSendServer.mockResolvedValue({ - ok: false, - error: '401', - }); + mockSendServer.mockRejectedValue( + new StepError('HTTP 401', { code: 'transport', status: 401 }), + ); const event = getEvent(); const config: Config = { diff --git a/packages/server/destinations/bing/src/__tests__/stepExamples.test.ts b/packages/server/destinations/bing/src/__tests__/stepExamples.test.ts index 6a91ad70f..555b7f971 100644 --- a/packages/server/destinations/bing/src/__tests__/stepExamples.test.ts +++ b/packages/server/destinations/bing/src/__tests__/stepExamples.test.ts @@ -16,7 +16,6 @@ describe('Step Examples', () => { beforeEach(() => { jest.clearAllMocks(); mockSendServer.mockResolvedValue({ - ok: true, data: JSON.stringify({ status: 'OK', requestId: 'req-1' }), }); }); diff --git a/packages/server/destinations/bing/src/examples/env.ts b/packages/server/destinations/bing/src/examples/env.ts index 2a1c4195f..c474c5fad 100644 --- a/packages/server/destinations/bing/src/examples/env.ts +++ b/packages/server/destinations/bing/src/examples/env.ts @@ -16,7 +16,6 @@ type MockSendServer = ( ) => Promise; const mockSendServer: MockSendServer = async () => ({ - ok: true, data: { status: 'OK', requestId: 'mock-123', diff --git a/packages/server/destinations/bing/src/push.ts b/packages/server/destinations/bing/src/push.ts index b722faa8f..83269a2da 100644 --- a/packages/server/destinations/bing/src/push.ts +++ b/packages/server/destinations/bing/src/push.ts @@ -123,20 +123,12 @@ export const push: PushFn = async function ( }); const sendServerFn = (env as Env)?.sendServer || sendServer; - const result = await sendServerFn(endpoint, JSON.stringify(body), { + await sendServerFn(endpoint, JSON.stringify(body), { headers: { Authorization: `Bearer ${accessToken}`, 'Content-Type': 'application/json', }, }); - - logger.debug('Bing UET CAPI response', { - ok: isObject(result) ? result.ok : true, - }); - - if (isObject(result) && result.ok === false) { - logger.throw(`Bing UET CAPI error: ${JSON.stringify(result)}`); - } }; const DEFAULT_IP: WalkerOSMapping.Value = ['ingest.ip', 'event.user.ip']; diff --git a/packages/server/destinations/clickhouse/package.json b/packages/server/destinations/clickhouse/package.json index f540e4d43..83771a54a 100644 --- a/packages/server/destinations/clickhouse/package.json +++ b/packages/server/destinations/clickhouse/package.json @@ -39,6 +39,9 @@ "@walkeros/core": "4.8.1", "@walkeros/server-core": "4.8.1" }, + "devDependencies": { + "@walkeros/collector": "4.8.1" + }, "repository": { "url": "git+https://github.com/elbwalker/walkerOS.git", "directory": "packages/server/destinations/clickhouse" diff --git a/packages/server/destinations/criteo/src/__tests__/hash.test.ts b/packages/server/destinations/criteo/src/__tests__/hash.test.ts index 64f0c9b58..5465247b5 100644 --- a/packages/server/destinations/criteo/src/__tests__/hash.test.ts +++ b/packages/server/destinations/criteo/src/__tests__/hash.test.ts @@ -20,7 +20,7 @@ describe('Criteo email hashing', () => { beforeEach(() => { jest.clearAllMocks(); - mockSendServer.mockResolvedValue({ ok: true, data: 'OK' }); + mockSendServer.mockResolvedValue({ data: 'OK' }); }); async function pushEmail(email: string): Promise { diff --git a/packages/server/destinations/criteo/src/__tests__/index.test.ts b/packages/server/destinations/criteo/src/__tests__/index.test.ts index 9b904a3ec..73ec04bd5 100644 --- a/packages/server/destinations/criteo/src/__tests__/index.test.ts +++ b/packages/server/destinations/criteo/src/__tests__/index.test.ts @@ -7,6 +7,7 @@ import { createMockContext, createMockLogger, getEvent, + StepError, } from '@walkeros/core'; import { startFlow } from '@walkeros/collector'; import { examples } from '../dev'; @@ -28,7 +29,6 @@ describe('Server Destination Criteo', () => { jest.clearAllMocks(); mockSendServer.mockResolvedValue({ - ok: true, data: 'OK', }); @@ -184,7 +184,7 @@ describe('Server Destination Criteo', () => { expect(body.events[0].event).toBe('trackTransaction'); jest.clearAllMocks(); - mockSendServer.mockResolvedValue({ ok: true, data: 'OK' }); + mockSendServer.mockResolvedValue({ data: 'OK' }); await destination.push( event, @@ -319,7 +319,9 @@ describe('Server Destination Criteo', () => { }); test('error response throws', async () => { - mockSendServer.mockResolvedValue({ ok: false, error: '401' }); + mockSendServer.mockRejectedValue( + new StepError('HTTP 401', { code: 'transport', status: 401 }), + ); const event = getEvent(); const config: Config = { settings: { partnerId, callerId } }; @@ -334,7 +336,7 @@ describe('Server Destination Criteo', () => { test('environment customization via env.sendServer', async () => { const customSendServer = jest.fn(); - customSendServer.mockResolvedValue({ ok: true, data: 'OK' }); + customSendServer.mockResolvedValue({ data: 'OK' }); const customEnv = { sendServer: customSendServer }; const event = getEvent(); diff --git a/packages/server/destinations/criteo/src/__tests__/stepExamples.test.ts b/packages/server/destinations/criteo/src/__tests__/stepExamples.test.ts index 0ef6d4756..906b3cfc1 100644 --- a/packages/server/destinations/criteo/src/__tests__/stepExamples.test.ts +++ b/packages/server/destinations/criteo/src/__tests__/stepExamples.test.ts @@ -16,7 +16,6 @@ describe('Step Examples', () => { beforeEach(() => { jest.clearAllMocks(); mockSendServer.mockResolvedValue({ - ok: true, data: 'OK', }); }); diff --git a/packages/server/destinations/criteo/src/examples/env.ts b/packages/server/destinations/criteo/src/examples/env.ts index d0b98761e..2945510ef 100644 --- a/packages/server/destinations/criteo/src/examples/env.ts +++ b/packages/server/destinations/criteo/src/examples/env.ts @@ -16,7 +16,6 @@ type MockSendServer = ( ) => Promise; const mockSendServer: MockSendServer = async () => ({ - ok: true, data: 'OK', }); diff --git a/packages/server/destinations/criteo/src/push.ts b/packages/server/destinations/criteo/src/push.ts index 78ad9a075..be70b5a93 100644 --- a/packages/server/destinations/criteo/src/push.ts +++ b/packages/server/destinations/criteo/src/push.ts @@ -188,15 +188,7 @@ export const push: PushFn = async function ( }); const sendServerFn = (env as Env)?.sendServer || sendServer; - const result = await sendServerFn(url, JSON.stringify(body)); - - logger.debug('Criteo API response', { - ok: isObject(result) ? result.ok : true, - }); - - if (isObject(result) && result.ok === false) { - logger.throw(`Criteo API error: ${JSON.stringify(result)}`); - } + await sendServerFn(url, JSON.stringify(body)); }; const DEFAULT_IP: WalkerOSMapping.Value = ['ingest.ip', 'event.user.ip']; diff --git a/packages/server/destinations/linkedin/src/__tests__/index.test.ts b/packages/server/destinations/linkedin/src/__tests__/index.test.ts index 79b86d9ed..17fe146ee 100644 --- a/packages/server/destinations/linkedin/src/__tests__/index.test.ts +++ b/packages/server/destinations/linkedin/src/__tests__/index.test.ts @@ -6,6 +6,7 @@ import { createMockContext, createMockLogger, getEvent, + StepError, } from '@walkeros/core'; import { startFlow } from '@walkeros/collector'; import { examples } from '../dev'; @@ -26,7 +27,6 @@ describe('Server Destination LinkedIn', () => { jest.clearAllMocks(); mockSendServer.mockResolvedValue({ - ok: true, data: {}, }); @@ -97,7 +97,7 @@ describe('Server Destination LinkedIn', () => { test('environment customization', async () => { const customSendServer = jest.fn(); - customSendServer.mockResolvedValue({ ok: true, data: {} }); + customSendServer.mockResolvedValue({ data: {} }); const customEnv = { sendServer: customSendServer }; const event = createEvent({ @@ -122,11 +122,9 @@ describe('Server Destination LinkedIn', () => { }); test('error handling - non-ok response triggers throw', async () => { - mockSendServer.mockResolvedValue({ - ok: false, - data: { message: 'Invalid token' }, - error: '401 Unauthorized', - }); + mockSendServer.mockRejectedValue( + new StepError('HTTP 401', { code: 'transport', status: 401 }), + ); const event = createEvent({ name: 'form submit', diff --git a/packages/server/destinations/linkedin/src/__tests__/stepExamples.test.ts b/packages/server/destinations/linkedin/src/__tests__/stepExamples.test.ts index 14c96ed23..2b311eeee 100644 --- a/packages/server/destinations/linkedin/src/__tests__/stepExamples.test.ts +++ b/packages/server/destinations/linkedin/src/__tests__/stepExamples.test.ts @@ -16,7 +16,6 @@ describe('Step Examples', () => { beforeEach(() => { jest.clearAllMocks(); mockSendServer.mockResolvedValue({ - ok: true, data: {}, }); }); diff --git a/packages/server/destinations/linkedin/src/examples/env.ts b/packages/server/destinations/linkedin/src/examples/env.ts index 0bf724273..bb4bdfc53 100644 --- a/packages/server/destinations/linkedin/src/examples/env.ts +++ b/packages/server/destinations/linkedin/src/examples/env.ts @@ -18,7 +18,6 @@ async function mockSendServer( options?: SendServerOptions, ): Promise { return { - ok: true, data: {}, }; } diff --git a/packages/server/destinations/linkedin/src/push.ts b/packages/server/destinations/linkedin/src/push.ts index eb1a62685..112f2619a 100644 --- a/packages/server/destinations/linkedin/src/push.ts +++ b/packages/server/destinations/linkedin/src/push.ts @@ -149,7 +149,7 @@ export const push: PushFn = async function ( }); const sendServerFn = (env as Env)?.sendServer || sendServer; - const result = await sendServerFn(endpoint, JSON.stringify(body), { + await sendServerFn(endpoint, JSON.stringify(body), { headers: { Authorization: `Bearer ${accessToken}`, 'Content-Type': 'application/json', @@ -158,12 +158,4 @@ export const push: PushFn = async function ( 'Linkedin-Version': apiVersion, }, }); - - logger.debug('LinkedIn API response', { - ok: isObject(result) ? result.ok : true, - }); - - if (isObject(result) && result.ok === false) { - logger.throw(`LinkedIn API error: ${JSON.stringify(result)}`); - } }; diff --git a/packages/server/destinations/meta/src/__tests__/index.test.ts b/packages/server/destinations/meta/src/__tests__/index.test.ts index a03628292..c3cf1aa2e 100644 --- a/packages/server/destinations/meta/src/__tests__/index.test.ts +++ b/packages/server/destinations/meta/src/__tests__/index.test.ts @@ -7,6 +7,7 @@ import { createIngest, createMockContext, createMockLogger, + StepError, } from '@walkeros/core'; import { startFlow } from '@walkeros/collector'; import { examples } from '../dev'; @@ -29,7 +30,6 @@ describe('Server Destination Meta', () => { // Reset mockSendServer to default successful response mockSendServer.mockResolvedValue({ - ok: true, data: { events_received: 1, messages: [], @@ -289,7 +289,7 @@ describe('Server Destination Meta', () => { test('environment customization', async () => { const customSendServer = jest.fn(); - customSendServer.mockResolvedValue({ ok: true, data: {} }); + customSendServer.mockResolvedValue({ data: {} }); const customEnv = { sendServer: customSendServer }; const event = getEvent(); @@ -311,18 +311,9 @@ describe('Server Destination Meta', () => { }); test('error', async () => { - mockSendServer.mockResolvedValue({ - ok: false, - data: { - error: { - message: 'Invalid', - type: 'OAuthException', - code: 190, - fbtrace_id: 'abc', - }, - }, - error: '400 Bad Request', - }); + mockSendServer.mockRejectedValue( + new StepError('HTTP 400', { code: 'transport', status: 400 }), + ); const mockCollector = {} as Collector.Instance; const event = getEvent(); const config: Config = { diff --git a/packages/server/destinations/meta/src/__tests__/stepExamples.test.ts b/packages/server/destinations/meta/src/__tests__/stepExamples.test.ts index 40a9a8a5c..0a40af8e4 100644 --- a/packages/server/destinations/meta/src/__tests__/stepExamples.test.ts +++ b/packages/server/destinations/meta/src/__tests__/stepExamples.test.ts @@ -35,7 +35,6 @@ describe('Step Examples', () => { beforeEach(() => { jest.clearAllMocks(); mockSendServer.mockResolvedValue({ - ok: true, data: { events_received: 1, messages: [], fbtrace_id: 'abc' }, }); }); diff --git a/packages/server/destinations/meta/src/examples/env.ts b/packages/server/destinations/meta/src/examples/env.ts index 756ec99d1..6e93a3f54 100644 --- a/packages/server/destinations/meta/src/examples/env.ts +++ b/packages/server/destinations/meta/src/examples/env.ts @@ -19,7 +19,6 @@ async function mockSendServer( ): Promise { // Simulate successful Meta API response return { - ok: true, data: { events_received: 1, messages: [], diff --git a/packages/server/destinations/meta/src/push.ts b/packages/server/destinations/meta/src/push.ts index 87082c586..31905d0e1 100644 --- a/packages/server/destinations/meta/src/push.ts +++ b/packages/server/destinations/meta/src/push.ts @@ -109,19 +109,11 @@ export const push: PushFn = async function ( }); const sendServerFn = env?.sendServer || sendServer; - const result = await sendServerFn(endpoint, JSON.stringify(body), { + await sendServerFn(endpoint, JSON.stringify(body), { headers: { Authorization: `Bearer ${accessToken}`, }, }); - - logger.debug('Meta API response', { - ok: isObject(result) ? result.ok : true, - }); - - if (isObject(result) && result.ok === false) { - logger.throw(`Meta API error: ${JSON.stringify(result)}`); - } }; const DEFAULT_IP: WalkerOSMapping.Value = ['ingest.ip', 'event.user.ip']; diff --git a/packages/server/destinations/mparticle/src/__tests__/push.test.ts b/packages/server/destinations/mparticle/src/__tests__/push.test.ts index 700f45ce0..3e0533cbb 100644 --- a/packages/server/destinations/mparticle/src/__tests__/push.test.ts +++ b/packages/server/destinations/mparticle/src/__tests__/push.test.ts @@ -4,6 +4,7 @@ import { createMockContext, createMockLogger, getEvent, + StepError, } from '@walkeros/core'; import { startFlow } from '@walkeros/collector'; import { examples } from '../dev'; @@ -20,7 +21,7 @@ describe('push', () => { beforeEach(() => { jest.clearAllMocks(); - mockSendServer.mockResolvedValue({ ok: true, data: {} }); + mockSendServer.mockResolvedValue({ data: {} }); testEnv = clone(env.push); testEnv.sendServer = mockSendServer; destination = jest.requireActual('../').default; @@ -180,12 +181,14 @@ describe('push', () => { ); }); - test('throws when sendServer returns ok: false', async () => { - mockSendServer.mockResolvedValue({ ok: false, error: 'Bad Request' }); + test('throws when sendServer rejects', async () => { + mockSendServer.mockRejectedValue( + new StepError('HTTP 400', { code: 'transport', status: 400 }), + ); const event = getEvent('product view', { user: { id: 'u1' } }); await expect( pushWith(event, { settings: { apiKey, apiSecret } }), - ).rejects.toThrow(/mParticle API error/); + ).rejects.toMatchObject({ code: 'transport', status: 400 }); }); test('rule.name overrides event name for custom_event', async () => { diff --git a/packages/server/destinations/mparticle/src/__tests__/stepExamples.test.ts b/packages/server/destinations/mparticle/src/__tests__/stepExamples.test.ts index a1bb730f5..aea1d0fda 100644 --- a/packages/server/destinations/mparticle/src/__tests__/stepExamples.test.ts +++ b/packages/server/destinations/mparticle/src/__tests__/stepExamples.test.ts @@ -19,7 +19,7 @@ describe('Step Examples', () => { beforeEach(() => { jest.clearAllMocks(); - mockSendServer.mockResolvedValue({ ok: true, data: {} }); + mockSendServer.mockResolvedValue({ data: {} }); }); it.each(Object.entries(examples.step))('%s', async (_name, example) => { diff --git a/packages/server/destinations/mparticle/src/examples/env.ts b/packages/server/destinations/mparticle/src/examples/env.ts index 36d4c0973..9c3075731 100644 --- a/packages/server/destinations/mparticle/src/examples/env.ts +++ b/packages/server/destinations/mparticle/src/examples/env.ts @@ -18,7 +18,6 @@ type MockSendServer = ( const mockSendServer: MockSendServer = async () => { // Simulates mParticle's typical 202 Accepted response for the Events API. return { - ok: true, data: {}, }; }; diff --git a/packages/server/destinations/mparticle/src/push.ts b/packages/server/destinations/mparticle/src/push.ts index a285d5e74..70951098e 100644 --- a/packages/server/destinations/mparticle/src/push.ts +++ b/packages/server/destinations/mparticle/src/push.ts @@ -132,20 +132,12 @@ export const push: PushFn = async function ( }); const sendServerFn = env?.sendServer || sendServer; - const result = await sendServerFn(endpoint, JSON.stringify(batch), { + await sendServerFn(endpoint, JSON.stringify(batch), { headers: { Authorization: authHeader, 'Content-Type': 'application/json', }, }); - - logger.debug('mParticle API response', { - ok: isObject(result) ? result.ok : true, - }); - - if (isObject(result) && result.ok === false) { - logger.throw(`mParticle API error: ${JSON.stringify(result)}`); - } }; async function resolveIdentities( diff --git a/packages/server/destinations/pinterest/src/__tests__/index.test.ts b/packages/server/destinations/pinterest/src/__tests__/index.test.ts index 4adb8123e..cd1cee2ef 100644 --- a/packages/server/destinations/pinterest/src/__tests__/index.test.ts +++ b/packages/server/destinations/pinterest/src/__tests__/index.test.ts @@ -6,6 +6,7 @@ import { getEvent, createMockContext, createMockLogger, + StepError, } from '@walkeros/core'; import { startFlow } from '@walkeros/collector'; import { examples } from '../dev'; @@ -27,7 +28,6 @@ describe('Server Destination Pinterest', () => { jest.clearAllMocks(); mockSendServer.mockResolvedValue({ - ok: true, data: { num_events_received: 1, num_events_processed: 1, @@ -146,7 +146,7 @@ describe('Server Destination Pinterest', () => { test('environment customization', async () => { const customSendServer = jest.fn(); - customSendServer.mockResolvedValue({ ok: true, data: {} }); + customSendServer.mockResolvedValue({ data: {} }); const customEnv = { sendServer: customSendServer }; const event = getEvent(); @@ -168,14 +168,9 @@ describe('Server Destination Pinterest', () => { }); test('error', async () => { - mockSendServer.mockResolvedValue({ - ok: false, - data: { - code: 4196, - message: 'Invalid request', - }, - error: '400 Bad Request', - }); + mockSendServer.mockRejectedValue( + new StepError('HTTP 400', { code: 'transport', status: 400 }), + ); const event = getEvent(); const config: Config = { settings: { accessToken, adAccountId }, diff --git a/packages/server/destinations/pinterest/src/__tests__/stepExamples.test.ts b/packages/server/destinations/pinterest/src/__tests__/stepExamples.test.ts index 5422bf8f1..ab3a4f950 100644 --- a/packages/server/destinations/pinterest/src/__tests__/stepExamples.test.ts +++ b/packages/server/destinations/pinterest/src/__tests__/stepExamples.test.ts @@ -17,7 +17,6 @@ describe('Step Examples', () => { beforeEach(() => { jest.clearAllMocks(); mockSendServer.mockResolvedValue({ - ok: true, data: { num_events_received: 1, num_events_processed: 1, diff --git a/packages/server/destinations/pinterest/src/examples/env.ts b/packages/server/destinations/pinterest/src/examples/env.ts index 917db9d94..cd01e9063 100644 --- a/packages/server/destinations/pinterest/src/examples/env.ts +++ b/packages/server/destinations/pinterest/src/examples/env.ts @@ -19,7 +19,6 @@ async function mockSendServer( ): Promise { // Simulate successful Pinterest API response return { - ok: true, data: { num_events_received: 1, num_events_processed: 1, diff --git a/packages/server/destinations/pinterest/src/push.ts b/packages/server/destinations/pinterest/src/push.ts index fb3c81c1f..ee36d503c 100644 --- a/packages/server/destinations/pinterest/src/push.ts +++ b/packages/server/destinations/pinterest/src/push.ts @@ -102,17 +102,9 @@ export const push: PushFn = async function ( }); const sendServerFn = env?.sendServer || sendServer; - const result = await sendServerFn(endpoint, JSON.stringify(body), { + await sendServerFn(endpoint, JSON.stringify(body), { headers: { Authorization: `Bearer ${accessToken}` }, }); - - logger.debug('Pinterest API response', { - ok: isObject(result) ? result.ok : true, - }); - - if (isObject(result) && result.ok === false) { - logger.throw(`Pinterest API error: ${JSON.stringify(result)}`); - } }; const DEFAULT_IP: WalkerOSMapping.Value = ['ingest.ip', 'event.user.ip']; diff --git a/packages/server/destinations/piwikpro/src/__tests__/push.test.ts b/packages/server/destinations/piwikpro/src/__tests__/push.test.ts index f82620b8d..04aad172c 100644 --- a/packages/server/destinations/piwikpro/src/__tests__/push.test.ts +++ b/packages/server/destinations/piwikpro/src/__tests__/push.test.ts @@ -6,7 +6,13 @@ import type { WalkerOS, } from '@walkeros/core'; import type { PartialConfig } from '../types'; -import { createEvent, createIngest, getEvent, Level } from '@walkeros/core'; +import { + createEvent, + createIngest, + getEvent, + Level, + StepError, +} from '@walkeros/core'; import { startFlow } from '@walkeros/collector'; import { destinationPiwikPro } from '..'; @@ -20,7 +26,7 @@ describe('push', () => { beforeEach(() => { sendServer.mockReset(); - sendServer.mockResolvedValue({ ok: true, data: '' }); + sendServer.mockResolvedValue({ data: '' }); logs = []; }); @@ -153,10 +159,9 @@ describe('push', () => { it('fails a push on a transport error', async () => { await setup(); - sendServer.mockResolvedValue({ - ok: false, - error: '500 Internal Server Error', - }); + sendServer.mockRejectedValue( + new StepError('HTTP 500', { code: 'transport', status: 500 }), + ); const failed = collector.status.failed; await collector.push(event('page view')); expect(collector.status.failed).toBe(failed + 1); @@ -164,10 +169,9 @@ describe('push', () => { it('fails the whole batch on a transport error', async () => { await setup({ batch: { size: 2, wait: 60000 } }); - sendServer.mockResolvedValue({ - ok: false, - error: '500 Internal Server Error', - }); + sendServer.mockRejectedValue( + new StepError('HTTP 500', { code: 'transport', status: 500 }), + ); const failed = collector.status.failed; await collector.push(event('page view')); await collector.push(event('page view')); @@ -175,14 +179,17 @@ describe('push', () => { expect(collector.status.failed).toBe(failed + 2); }); - it('warns on a 429 before failing', async () => { + it('reports a 429 as a retryable failure with its status', async () => { await setup(); - sendServer.mockResolvedValue({ ok: false, error: '429 Too Many Requests' }); - await collector.push(event('page view')); - expect(logs).toContainEqual([ - Level.WARN, - 'Piwik PRO rate limited the request (429)', - ]); + sendServer.mockRejectedValue( + new StepError('HTTP 429', { code: 'transport', status: 429 }), + ); + const result = await collector.push(event('page view')); + expect(result.failed?.piwik?.reason).toMatchObject({ + code: 'transport', + status: 429, + retryable: true, + }); }); it('warns once per skip reason, then logs at debug', async () => { diff --git a/packages/server/destinations/piwikpro/src/__tests__/stepExamples.test.ts b/packages/server/destinations/piwikpro/src/__tests__/stepExamples.test.ts index a439e731e..53d2ae555 100644 --- a/packages/server/destinations/piwikpro/src/__tests__/stepExamples.test.ts +++ b/packages/server/destinations/piwikpro/src/__tests__/stepExamples.test.ts @@ -35,7 +35,7 @@ describe('Step Examples', () => { beforeEach(() => { sendServer.mockReset(); - sendServer.mockResolvedValue({ ok: true, data: '' }); + sendServer.mockResolvedValue({ data: '' }); }); async function run( diff --git a/packages/server/destinations/piwikpro/src/examples/env.ts b/packages/server/destinations/piwikpro/src/examples/env.ts index 1b2b1a74f..36b99af5a 100644 --- a/packages/server/destinations/piwikpro/src/examples/env.ts +++ b/packages/server/destinations/piwikpro/src/examples/env.ts @@ -5,10 +5,10 @@ import type { Env } from '../types'; * Example environment for the Piwik PRO destination. * * The Tracking API answers a bulk request with 202 and an empty body, so the - * mock resolves `{ ok: true, data: '' }` without an HTTP request. + * mock resolves `{ data: '' }` without an HTTP request. */ async function mockSendServer(): Promise { - return { ok: true, data: '' }; + return { data: '' }; } /** Standard mock environment for push and pushBatch. */ diff --git a/packages/server/destinations/piwikpro/src/send.ts b/packages/server/destinations/piwikpro/src/send.ts index 3af1e8e97..f6468a5d7 100644 --- a/packages/server/destinations/piwikpro/src/send.ts +++ b/packages/server/destinations/piwikpro/src/send.ts @@ -21,21 +21,9 @@ export async function sendHits( const url = `${settings.url}ppms.php`; const sendServerFn = env?.sendServer || sendServer; - const response = await sendServerFn(url, toBody(hits), { + await sendServerFn(url, toBody(hits), { timeout: settings.timeout ?? 5000, }); - if (response.ok) { - logger.debug('Piwik PRO request sent', { url, hits: hits.length }); - return; - } - - // sendServer exposes no response headers, so Retry-After is not readable - if (response.error?.startsWith('429')) - logger.warn('Piwik PRO rate limited the request (429)'); - - logger.throw(`Piwik PRO request failed: ${response.error}`, { - url, - data: response.data, - }); + logger.debug('Piwik PRO request sent', { url, hits: hits.length }); } diff --git a/packages/server/destinations/reddit/src/__tests__/index.test.ts b/packages/server/destinations/reddit/src/__tests__/index.test.ts index fcf5815f3..dbfa8e370 100644 --- a/packages/server/destinations/reddit/src/__tests__/index.test.ts +++ b/packages/server/destinations/reddit/src/__tests__/index.test.ts @@ -6,6 +6,7 @@ import { getEvent, createMockContext, createMockLogger, + StepError, } from '@walkeros/core'; import { startFlow } from '@walkeros/collector'; import { examples } from '../dev'; @@ -27,7 +28,6 @@ describe('Server Destination Reddit', () => { jest.clearAllMocks(); mockSendServer.mockResolvedValue({ - ok: true, data: { success: true }, }); @@ -144,7 +144,7 @@ describe('Server Destination Reddit', () => { test('environment customization', async () => { const customSendServer = jest.fn(); - customSendServer.mockResolvedValue({ ok: true, data: {} }); + customSendServer.mockResolvedValue({ data: {} }); const customEnv = { sendServer: customSendServer }; const event = getEvent(); @@ -166,11 +166,9 @@ describe('Server Destination Reddit', () => { }); test('error', async () => { - mockSendServer.mockResolvedValue({ - ok: false, - data: { success: false, message: 'Invalid request' }, - error: '400 Bad Request', - }); + mockSendServer.mockRejectedValue( + new StepError('HTTP 400', { code: 'transport', status: 400 }), + ); const event = getEvent(); const config: Config = { settings: { accessToken, pixelId }, diff --git a/packages/server/destinations/reddit/src/__tests__/stepExamples.test.ts b/packages/server/destinations/reddit/src/__tests__/stepExamples.test.ts index 2448d3aad..90577b696 100644 --- a/packages/server/destinations/reddit/src/__tests__/stepExamples.test.ts +++ b/packages/server/destinations/reddit/src/__tests__/stepExamples.test.ts @@ -16,7 +16,6 @@ describe('Step Examples', () => { beforeEach(() => { jest.clearAllMocks(); mockSendServer.mockResolvedValue({ - ok: true, data: { success: true }, }); }); diff --git a/packages/server/destinations/reddit/src/examples/env.ts b/packages/server/destinations/reddit/src/examples/env.ts index e7be84fa7..e35a10456 100644 --- a/packages/server/destinations/reddit/src/examples/env.ts +++ b/packages/server/destinations/reddit/src/examples/env.ts @@ -21,7 +21,6 @@ type MockSendServer = ( const mockSendServer: MockSendServer = async () => { // Simulate successful Reddit API response return { - ok: true, data: { success: true, }, diff --git a/packages/server/destinations/reddit/src/push.ts b/packages/server/destinations/reddit/src/push.ts index 802b87485..73f3b6cb8 100644 --- a/packages/server/destinations/reddit/src/push.ts +++ b/packages/server/destinations/reddit/src/push.ts @@ -125,17 +125,9 @@ export const push: PushFn = async function ( }); const sendServerFn = env?.sendServer || sendServer; - const result = await sendServerFn(endpoint, JSON.stringify(body), { + await sendServerFn(endpoint, JSON.stringify(body), { headers: { Authorization: `Bearer ${accessToken}` }, }); - - logger.debug('Reddit API response', { - ok: isObject(result) ? result.ok : true, - }); - - if (isObject(result) && result.ok === false) { - logger.throw(`Reddit API error: ${JSON.stringify(result)}`); - } }; const DEFAULT_IP: WalkerOSMapping.Value = ['ingest.ip', 'event.user.ip']; diff --git a/packages/server/destinations/slack/src/__tests__/stepExamples.test.ts b/packages/server/destinations/slack/src/__tests__/stepExamples.test.ts index 4ccfc9bf3..6dad8c033 100644 --- a/packages/server/destinations/slack/src/__tests__/stepExamples.test.ts +++ b/packages/server/destinations/slack/src/__tests__/stepExamples.test.ts @@ -49,7 +49,7 @@ function spyEnv(): { env: Env; collected: () => Captured[] } { const sendServer = ((...args: unknown[]) => { calls.push(['sendServer', ...args] as Captured); - return Promise.resolve({ ok: true }); + return Promise.resolve({}); }) as Env['sendServer']; return { diff --git a/packages/server/destinations/slack/src/examples/env.ts b/packages/server/destinations/slack/src/examples/env.ts index 99f634201..0a74910e6 100644 --- a/packages/server/destinations/slack/src/examples/env.ts +++ b/packages/server/destinations/slack/src/examples/env.ts @@ -27,8 +27,7 @@ function createMockClient(): SlackClientMock { export const push: Env = { slackClient: createMockClient(), - sendServer: (() => - Promise.resolve({ ok: true } as never)) as Env['sendServer'], + sendServer: () => Promise.resolve({}), }; export const simulation = [ diff --git a/packages/server/destinations/slack/src/push.ts b/packages/server/destinations/slack/src/push.ts index c3c755ee5..7ff808c23 100644 --- a/packages/server/destinations/slack/src/push.ts +++ b/packages/server/destinations/slack/src/push.ts @@ -1,4 +1,3 @@ -import { isObject } from '@walkeros/core'; import { sendServer } from '@walkeros/server-core'; import type { Mapping, PushFn, Settings, SlackClientMock } from './types'; import { buildMessage } from './message'; @@ -39,14 +38,7 @@ async function pushWebhook( }; const send = env?.sendServer || sendServer; - const result = await send( - settings.webhookUrl as string, - JSON.stringify(body), - ); - - if (isObject(result) && result.ok === false) { - logger.warn(`Slack webhook error: ${JSON.stringify(result)}`); - } + await send(settings.webhookUrl as string, JSON.stringify(body)); } async function pushWebApi( diff --git a/packages/server/destinations/snapchat/src/__tests__/index.test.ts b/packages/server/destinations/snapchat/src/__tests__/index.test.ts index 833e5ca7d..904f46e38 100644 --- a/packages/server/destinations/snapchat/src/__tests__/index.test.ts +++ b/packages/server/destinations/snapchat/src/__tests__/index.test.ts @@ -7,6 +7,7 @@ import { createMockContext, createMockLogger, getEvent, + StepError, } from '@walkeros/core'; import { startFlow } from '@walkeros/collector'; import { examples } from '../dev'; @@ -28,7 +29,6 @@ describe('Server Destination Snapchat', () => { jest.clearAllMocks(); mockSendServer.mockResolvedValue({ - ok: true, data: JSON.stringify({ status: 'OK', request_id: 'req-1' }), }); @@ -330,7 +330,6 @@ describe('Server Destination Snapchat', () => { test('environment customization', async () => { const customSendServer = jest.fn(); customSendServer.mockResolvedValue({ - ok: true, data: JSON.stringify({ status: 'OK' }), }); @@ -354,10 +353,9 @@ describe('Server Destination Snapchat', () => { }); test('error response throws', async () => { - mockSendServer.mockResolvedValue({ - ok: false, - error: '401', - }); + mockSendServer.mockRejectedValue( + new StepError('HTTP 401', { code: 'transport', status: 401 }), + ); const event = getEvent(); const config: Config = { diff --git a/packages/server/destinations/snapchat/src/__tests__/stepExamples.test.ts b/packages/server/destinations/snapchat/src/__tests__/stepExamples.test.ts index e0816c9e5..5982d86c2 100644 --- a/packages/server/destinations/snapchat/src/__tests__/stepExamples.test.ts +++ b/packages/server/destinations/snapchat/src/__tests__/stepExamples.test.ts @@ -17,7 +17,6 @@ describe('Step Examples', () => { beforeEach(() => { jest.clearAllMocks(); mockSendServer.mockResolvedValue({ - ok: true, data: JSON.stringify({ status: 'OK', request_id: 'req-1' }), }); }); diff --git a/packages/server/destinations/snapchat/src/examples/env.ts b/packages/server/destinations/snapchat/src/examples/env.ts index afad704b8..b492eb44d 100644 --- a/packages/server/destinations/snapchat/src/examples/env.ts +++ b/packages/server/destinations/snapchat/src/examples/env.ts @@ -16,7 +16,6 @@ type MockSendServer = ( ) => Promise; const mockSendServer: MockSendServer = async () => ({ - ok: true, data: { status: 'OK', request_id: 'mock-123', diff --git a/packages/server/destinations/snapchat/src/push.ts b/packages/server/destinations/snapchat/src/push.ts index ed1bd3e04..e23d0c201 100644 --- a/packages/server/destinations/snapchat/src/push.ts +++ b/packages/server/destinations/snapchat/src/push.ts @@ -107,15 +107,7 @@ export const push: PushFn = async function ( }); const sendServerFn = (env as Env)?.sendServer || sendServer; - const result = await sendServerFn(endpoint, JSON.stringify(body)); - - logger.debug('Snapchat API response', { - ok: isObject(result) ? result.ok : true, - }); - - if (isObject(result) && result.ok === false) { - logger.throw(`Snapchat API error: ${JSON.stringify(result)}`); - } + await sendServerFn(endpoint, JSON.stringify(body)); }; const DEFAULT_IP: WalkerOSMapping.Value = ['ingest.ip', 'event.user.ip']; diff --git a/packages/server/destinations/tiktok/src/__tests__/index.test.ts b/packages/server/destinations/tiktok/src/__tests__/index.test.ts index eda1d5aed..72d126652 100644 --- a/packages/server/destinations/tiktok/src/__tests__/index.test.ts +++ b/packages/server/destinations/tiktok/src/__tests__/index.test.ts @@ -7,6 +7,7 @@ import { createMockContext, createMockLogger, getEvent, + StepError, } from '@walkeros/core'; import { startFlow } from '@walkeros/collector'; import { examples } from '../dev'; @@ -29,7 +30,6 @@ describe('Server Destination TikTok', () => { // Reset mockSendServer to default successful response mockSendServer.mockResolvedValue({ - ok: true, data: JSON.stringify({ code: 0, message: 'OK' }), }); @@ -276,7 +276,6 @@ describe('Server Destination TikTok', () => { test('environment customization', async () => { const customSendServer = jest.fn(); customSendServer.mockResolvedValue({ - ok: true, data: JSON.stringify({ code: 0, message: 'OK' }), }); @@ -300,10 +299,9 @@ describe('Server Destination TikTok', () => { }); test('error response throws', async () => { - mockSendServer.mockResolvedValue({ - ok: false, - error: '401', - }); + mockSendServer.mockRejectedValue( + new StepError('HTTP 401', { code: 'transport', status: 401 }), + ); const event = getEvent(); const config: Config = { diff --git a/packages/server/destinations/tiktok/src/__tests__/stepExamples.test.ts b/packages/server/destinations/tiktok/src/__tests__/stepExamples.test.ts index 5823c2b55..08b882a38 100644 --- a/packages/server/destinations/tiktok/src/__tests__/stepExamples.test.ts +++ b/packages/server/destinations/tiktok/src/__tests__/stepExamples.test.ts @@ -17,7 +17,6 @@ describe('Step Examples', () => { beforeEach(() => { jest.clearAllMocks(); mockSendServer.mockResolvedValue({ - ok: true, data: JSON.stringify({ code: 0, message: 'OK' }), }); }); diff --git a/packages/server/destinations/tiktok/src/examples/env.ts b/packages/server/destinations/tiktok/src/examples/env.ts index 8bea815e4..552cea933 100644 --- a/packages/server/destinations/tiktok/src/examples/env.ts +++ b/packages/server/destinations/tiktok/src/examples/env.ts @@ -19,7 +19,6 @@ async function mockSendServer( ): Promise { // Simulate successful TikTok API response return { - ok: true, data: { code: 0, message: 'OK', diff --git a/packages/server/destinations/tiktok/src/push.ts b/packages/server/destinations/tiktok/src/push.ts index fed509a9a..bc32c2772 100644 --- a/packages/server/destinations/tiktok/src/push.ts +++ b/packages/server/destinations/tiktok/src/push.ts @@ -112,20 +112,12 @@ export const push: PushFn = async function ( }); const sendServerFn = (env as Env)?.sendServer || sendServer; - const result = await sendServerFn(url, JSON.stringify(body), { + await sendServerFn(url, JSON.stringify(body), { headers: { 'Access-Token': accessToken, 'Content-Type': 'application/json', }, }); - - logger.debug('TikTok API response', { - ok: isObject(result) ? result.ok : true, - }); - - if (isObject(result) && result.ok === false) { - logger.throw(`TikTok API error: ${JSON.stringify(result)}`); - } }; const DEFAULT_IP: WalkerOSMapping.Value = ['ingest.ip', 'event.user.ip']; diff --git a/packages/server/destinations/twitter/src/__tests__/index.test.ts b/packages/server/destinations/twitter/src/__tests__/index.test.ts index ea4a662d0..c51adc499 100644 --- a/packages/server/destinations/twitter/src/__tests__/index.test.ts +++ b/packages/server/destinations/twitter/src/__tests__/index.test.ts @@ -7,6 +7,7 @@ import { createMockContext, createMockLogger, getEvent, + StepError, } from '@walkeros/core'; import { startFlow } from '@walkeros/collector'; import { examples } from '../dev'; @@ -41,7 +42,6 @@ describe('Server Destination Twitter', () => { jest.clearAllMocks(); mockSendServer.mockResolvedValue({ - ok: true, data: { request_id: 'mock-request-id' }, }); @@ -169,7 +169,7 @@ describe('Server Destination Twitter', () => { test('environment customization', async () => { const customSendServer = jest.fn(); - customSendServer.mockResolvedValue({ ok: true, data: {} }); + customSendServer.mockResolvedValue({ data: {} }); const customEnv = { sendServer: customSendServer }; const event = createEvent({ @@ -192,11 +192,9 @@ describe('Server Destination Twitter', () => { }); test('error handling - non-ok response triggers throw', async () => { - mockSendServer.mockResolvedValue({ - ok: false, - data: { message: 'Invalid auth' }, - error: '401 Unauthorized', - }); + mockSendServer.mockRejectedValue( + new StepError('HTTP 401', { code: 'transport', status: 401 }), + ); const event = createEvent({ name: 'form submit', diff --git a/packages/server/destinations/twitter/src/__tests__/stepExamples.test.ts b/packages/server/destinations/twitter/src/__tests__/stepExamples.test.ts index b0140d83f..f780e53d4 100644 --- a/packages/server/destinations/twitter/src/__tests__/stepExamples.test.ts +++ b/packages/server/destinations/twitter/src/__tests__/stepExamples.test.ts @@ -19,7 +19,6 @@ describe('Step Examples', () => { beforeEach(() => { jest.clearAllMocks(); mockSendServer.mockResolvedValue({ - ok: true, data: { request_id: 'mock-request-id' }, }); }); diff --git a/packages/server/destinations/twitter/src/examples/env.ts b/packages/server/destinations/twitter/src/examples/env.ts index fec5a482b..d1fb154e6 100644 --- a/packages/server/destinations/twitter/src/examples/env.ts +++ b/packages/server/destinations/twitter/src/examples/env.ts @@ -20,7 +20,6 @@ type SendServerFn = ( */ const mockSendServer: SendServerFn = async () => { return { - ok: true, data: { request_id: 'mock-request-id' }, }; }; diff --git a/packages/server/destinations/twitter/src/push.ts b/packages/server/destinations/twitter/src/push.ts index a23c70b92..f6bfd6b47 100644 --- a/packages/server/destinations/twitter/src/push.ts +++ b/packages/server/destinations/twitter/src/push.ts @@ -204,20 +204,12 @@ export const push: PushFn = async function ( }); const sendServerFn = (env as Env)?.sendServer || sendServer; - const result = await sendServerFn(endpoint, JSON.stringify(body), { + await sendServerFn(endpoint, JSON.stringify(body), { headers: { Authorization: authHeader.Authorization, 'Content-Type': 'application/json', }, }); - - logger.debug('X Conversions API response', { - ok: isObject(result) ? result.ok : true, - }); - - if (isObject(result) && result.ok === false) { - logger.throw(`X Conversions API error: ${JSON.stringify(result)}`); - } }; const DEFAULT_IP: WalkerOSMapping.Value = ['ingest.ip', 'event.user.ip']; diff --git a/packages/server/stores/fs/package.json b/packages/server/stores/fs/package.json index be3fa21df..45d07c52d 100644 --- a/packages/server/stores/fs/package.json +++ b/packages/server/stores/fs/package.json @@ -36,6 +36,7 @@ "@walkeros/core": "4.8.1" }, "devDependencies": { + "@walkeros/collector": "4.8.1", "@walkeros/core": "4.8.1" }, "repository": { diff --git a/packages/server/stores/gcs/package.json b/packages/server/stores/gcs/package.json index 2a8c76342..10a59a20b 100644 --- a/packages/server/stores/gcs/package.json +++ b/packages/server/stores/gcs/package.json @@ -35,7 +35,9 @@ "dependencies": { "@walkeros/core": "4.8.1" }, - "devDependencies": {}, + "devDependencies": { + "@walkeros/collector": "4.8.1" + }, "repository": { "url": "git+https://github.com/elbwalker/walkerOS.git", "directory": "packages/server/stores/gcs" diff --git a/packages/server/stores/s3/package.json b/packages/server/stores/s3/package.json index 66923047a..ef5402c75 100644 --- a/packages/server/stores/s3/package.json +++ b/packages/server/stores/s3/package.json @@ -36,7 +36,9 @@ "@walkeros/core": "4.8.1", "s3mini": "^0.9.1" }, - "devDependencies": {}, + "devDependencies": { + "@walkeros/collector": "4.8.1" + }, "repository": { "url": "git+https://github.com/elbwalker/walkerOS.git", "directory": "packages/server/stores/s3" diff --git a/packages/server/stores/sheets/package.json b/packages/server/stores/sheets/package.json index 03adc3b0f..aba427ea8 100644 --- a/packages/server/stores/sheets/package.json +++ b/packages/server/stores/sheets/package.json @@ -35,7 +35,9 @@ "dependencies": { "@walkeros/core": "4.8.1" }, - "devDependencies": {}, + "devDependencies": { + "@walkeros/collector": "4.8.1" + }, "repository": { "url": "git+https://github.com/elbwalker/walkerOS.git", "directory": "packages/server/stores/sheets" diff --git a/packages/server/transformers/fingerprint/package.json b/packages/server/transformers/fingerprint/package.json index df09a14f2..212628482 100644 --- a/packages/server/transformers/fingerprint/package.json +++ b/packages/server/transformers/fingerprint/package.json @@ -27,6 +27,7 @@ "update": "npx npm-check-updates -u && npm update" }, "devDependencies": { + "@walkeros/collector": "4.8.1", "@walkeros/core": "4.8.1", "@walkeros/server-core": "4.8.1" }, diff --git a/packages/transformers/ga4/package.json b/packages/transformers/ga4/package.json index 6c2c9df55..591bbc92a 100644 --- a/packages/transformers/ga4/package.json +++ b/packages/transformers/ga4/package.json @@ -35,6 +35,7 @@ "@walkeros/core": "4.8.1" }, "devDependencies": { + "@walkeros/collector": "4.8.1", "@walkeros/core": "4.8.1" }, "repository": { diff --git a/packages/web/core/src/__tests__/loadVendor.test.ts b/packages/web/core/src/__tests__/loadVendor.test.ts new file mode 100644 index 000000000..ddb9d9059 --- /dev/null +++ b/packages/web/core/src/__tests__/loadVendor.test.ts @@ -0,0 +1,132 @@ +import { DEFAULT_DESTINATION_TIMEOUT_MS } from '@walkeros/core'; +import { loadVendor, VENDOR_SCRIPT_MARKER } from '../loadVendor'; + +describe('loadVendor', () => { + let n = 0; + // One src per test: the injected scripts are remembered per page. + const nextSrc = () => `https://vendor.example/sdk-${++n}.js`; + const scriptsOf = (src: string) => + Array.from(document.querySelectorAll('script')).filter( + (script) => script.getAttribute('src') === src, + ); + const fire = (src: string, type: 'load' | 'error') => + scriptsOf(src).forEach((script) => script.dispatchEvent(new Event(type))); + + interface Sdk { + track(): void; + } + let sdk: Sdk | undefined; + const get = () => sdk; + + beforeEach(() => { + sdk = undefined; + }); + + it('injects one marked script per src, however often ready() is called', () => { + const src = nextSrc(); + const vendor = loadVendor(src, get); + vendor.ready().catch(() => undefined); + vendor.ready().catch(() => undefined); + loadVendor(src, get) + .ready() + .catch(() => undefined); + + const scripts = scriptsOf(src); + expect(scripts).toHaveLength(1); + expect(scripts[0].hasAttribute(VENDOR_SCRIPT_MARKER)).toBe(true); + expect(scripts[0].async).toBe(true); + }); + + it('resolves with get() after load', async () => { + const src = nextSrc(); + const ready = loadVendor(src, get).ready(); + sdk = { track: () => undefined }; + fire(src, 'load'); + await expect(ready).resolves.toBe(sdk); + }); + + it('rejects vendor-not-loaded after error, final for the page', async () => { + const src = nextSrc(); + const vendor = loadVendor(src, get); + const ready = vendor.ready(); + fire(src, 'error'); + + await expect(ready).rejects.toMatchObject({ + code: 'vendor-not-loaded', + retryable: false, + }); + sdk = { track: () => undefined }; + await expect(vendor.ready()).rejects.toMatchObject({ + code: 'vendor-not-loaded', + }); + await expect(loadVendor(src, get).ready()).rejects.toMatchObject({ + code: 'vendor-not-loaded', + }); + expect(() => vendor.check()).toThrow( + expect.objectContaining({ code: 'vendor-not-loaded' }), + ); + expect(scriptsOf(src)).toHaveLength(1); + }); + + it('check() passes while the script loads and after it loaded', () => { + const src = nextSrc(); + const vendor = loadVendor(src, get); + expect(() => vendor.check()).not.toThrow(); + fire(src, 'load'); + expect(() => vendor.check()).not.toThrow(); + }); + + it('rejects vendor-not-ready when the script loaded but get() stays empty', async () => { + const src = nextSrc(); + const ready = loadVendor(src, get).ready(); + fire(src, 'load'); + await expect(ready).rejects.toMatchObject({ + code: 'vendor-not-ready', + retryable: true, + }); + }); + + it('rejects vendor-timeout at the resolved deadline, then checks get() on each call', async () => { + const src = nextSrc(); + const vendor = loadVendor(src, get, { timeout: 500 }); + const ready = vendor.ready(); + + jest.advanceTimersByTime(499); + sdk = { track: () => undefined }; + jest.advanceTimersByTime(1); + await expect(ready).rejects.toMatchObject({ + code: 'vendor-timeout', + retryable: true, + }); + + // Past the deadline: synchronous checks, no new script. + await expect(vendor.ready()).resolves.toBe(sdk); + sdk = undefined; + await expect(vendor.ready()).rejects.toMatchObject({ + code: 'vendor-not-ready', + }); + expect(scriptsOf(src)).toHaveLength(1); + }); + + it('uses the default destination timeout without one', async () => { + const src = nextSrc(); + const ready = loadVendor(src, get).ready(); + jest.advanceTimersByTime(DEFAULT_DESTINATION_TIMEOUT_MS - 1); + sdk = { track: () => undefined }; + fire(src, 'load'); + await expect(ready).resolves.toBe(sdk); + }); + + it('without src checks get() on each call, with no deadline and no script', async () => { + const before = document.querySelectorAll('script').length; + const vendor = loadVendor(undefined, get); + + await expect(vendor.ready()).rejects.toMatchObject({ + code: 'vendor-not-ready', + }); + jest.advanceTimersByTime(DEFAULT_DESTINATION_TIMEOUT_MS * 2); + sdk = { track: () => undefined }; + await expect(vendor.ready()).resolves.toBe(sdk); + expect(document.querySelectorAll('script')).toHaveLength(before); + }); +}); diff --git a/packages/web/core/src/__tests__/sendWeb.test.ts b/packages/web/core/src/__tests__/sendWeb.test.ts index 0d69576d0..ad54bf206 100644 --- a/packages/web/core/src/__tests__/sendWeb.test.ts +++ b/packages/web/core/src/__tests__/sendWeb.test.ts @@ -1,18 +1,61 @@ +import { StepError } from '@walkeros/core'; import { sendWeb } from '..'; +type BodyMethod = ReturnType< + Extract unknown> +>; + +/** A Response for jsdom, which has no fetch API of its own. */ +class FakeResponse implements Response { + readonly headers = new Headers(); + readonly redirected = false; + readonly type: ResponseType = 'basic'; + readonly url = ''; + readonly body = null; + readonly bodyUsed = false; + readonly ok: boolean; + + constructor( + private readonly content: string, + readonly status = 200, + readonly statusText = '', + ) { + this.ok = status >= 200 && status < 300; + } + + clone(): Response { + return new FakeResponse(this.content, this.status, this.statusText); + } + text(): Promise { + return Promise.resolve(this.content); + } + json(): Promise { + return Promise.resolve(JSON.parse(this.content)); + } + arrayBuffer(): BodyMethod<'arrayBuffer'> { + return Promise.reject(new Error('unsupported')); + } + blob(): BodyMethod<'blob'> { + return Promise.reject(new Error('unsupported')); + } + bytes(): BodyMethod<'bytes'> { + return Promise.reject(new Error('unsupported')); + } + formData(): BodyMethod<'formData'> { + return Promise.reject(new Error('unsupported')); + } +} + describe('sendWeb', () => { const data = { key: 'value' }; const dataStringified = JSON.stringify({ key: 'value' }); const url = 'https://api.walkeros.io/'; const mockFetch = jest.fn( - () => - Promise.resolve({ - text: () => Promise.resolve(dataStringified), - ok: true, - }) as Promise, + (_input: URL | RequestInfo, _init?: RequestInit): Promise => + Promise.resolve(new FakeResponse(dataStringified)), ); - const mockBeacon = jest.fn(() => true); + const mockBeacon = jest.fn((_url: string, _body?: BodyInit | null) => true); const mockXHROpen = jest.fn(); const mockXHRSend = jest.fn(); const mockXHRHeader = jest.fn(); @@ -29,6 +72,7 @@ describe('sendWeb', () => { const oldBeacon = navigator.sendBeacon; beforeEach(() => { + mockXHR.status = 200; window.fetch = mockFetch; navigator.sendBeacon = mockBeacon; Object.defineProperty(window, 'XMLHttpRequest', { @@ -43,7 +87,7 @@ describe('sendWeb', () => { window.XMLHttpRequest = oldXMLHttpRequest; }); - test('fetch', async () => { + test('fetch resolves with the data', async () => { const response = await sendWeb(url, data, { transport: 'fetch' }); expect(mockFetch).toHaveBeenCalledWith( @@ -54,14 +98,11 @@ describe('sendWeb', () => { mode: 'cors', }), ); - expect(response).toStrictEqual({ - ok: true, - data: dataStringified, - error: undefined, - }); + expect(response).toStrictEqual({ data: dataStringified }); }); - test('fetch cors', async () => { + test('a noCors fetch resolves once the request completes (opaque counts as sent)', async () => { + mockFetch.mockResolvedValueOnce(new FakeResponse('', 0)); const response = await sendWeb(url, data, { transport: 'fetch', noCors: true, @@ -69,44 +110,115 @@ describe('sendWeb', () => { expect(mockFetch).toHaveBeenCalledWith( url, - expect.objectContaining({ - body: dataStringified, - keepalive: true, - mode: 'no-cors', - }), + expect.objectContaining({ mode: 'no-cors' }), ); - expect(response).toStrictEqual({ - ok: true, - data: '', - error: undefined, - }); + expect(response).toStrictEqual({ data: '' }); }); test('fetch credentials', async () => { - const response = await sendWeb(url, data, { - transport: 'fetch', - credentials: 'include', - }); + await sendWeb(url, data, { transport: 'fetch', credentials: 'include' }); expect(mockFetch).toHaveBeenCalledWith( url, - expect.objectContaining({ - body: dataStringified, - keepalive: true, - credentials: 'include', - }), + expect.objectContaining({ credentials: 'include' }), + ); + }); + + test('a 503 throws a retryable StepError with status and body, the body never in the message', async () => { + mockFetch.mockResolvedValueOnce( + new FakeResponse('{"error":"user@example.com"}', 503, 'Busy'), + ); + const sent = sendWeb(url, data, { transport: 'fetch' }); + + await expect(sent).rejects.toBeInstanceOf(StepError); + await sent.catch((error: unknown) => { + expect(error).toMatchObject({ + code: 'transport', + status: 503, + retryable: true, + body: '{"error":"user@example.com"}', + message: 'HTTP 503', + }); + }); + }); + + test('a 400 throws a StepError that is not retryable', async () => { + mockFetch.mockResolvedValueOnce(new FakeResponse('bad', 400)); + await expect( + sendWeb(url, data, { transport: 'fetch' }), + ).rejects.toMatchObject({ + code: 'transport', + status: 400, + retryable: false, + }); + }); + + test('a network error keeps its cause code in the message, never the address', async () => { + const cause = Object.assign( + new Error('getaddrinfo ENOTFOUND vendor.example?key=secret'), + { code: 'ENOTFOUND' }, ); - expect(response).toStrictEqual({ - ok: true, - data: dataStringified, - error: undefined, + mockFetch.mockRejectedValueOnce(new TypeError('fetch failed', { cause })); + await expect( + sendWeb(url, data, { transport: 'fetch' }), + ).rejects.toMatchObject({ + code: 'transport', + message: 'fetch failed (ENOTFOUND)', }); }); - test('beacon', () => { + test('a network error throws code transport', async () => { + mockFetch.mockRejectedValueOnce(new TypeError('Failed to fetch')); + await expect( + sendWeb(url, data, { transport: 'fetch' }), + ).rejects.toMatchObject({ code: 'transport', retryable: true }); + }); + + test('keepalive stops once the bodies in flight would exceed 64 KiB', async () => { + const releases: Array<() => void> = []; + mockFetch.mockImplementation( + () => + new Promise((resolve) => { + releases.push(() => resolve(new FakeResponse(''))); + }), + ); + const body = 'x'.repeat(30 * 1024); + + const sends = [1, 2, 3].map(() => + sendWeb(url, body, { transport: 'fetch' }), + ); + expect(mockFetch.mock.calls.map(([, init]) => init?.keepalive)).toEqual([ + true, + true, + false, + ]); + + releases.forEach((release) => release()); + await Promise.all(sends); + + // The quota is free again once the requests settled. + mockFetch.mockImplementation(() => Promise.resolve(new FakeResponse(''))); + await sendWeb(url, body, { transport: 'fetch' }); + expect(mockFetch.mock.calls[3][1]?.keepalive).toBe(true); + }); + + test('beacon returns when the browser accepts it', () => { const response = sendWeb(url, data, { transport: 'beacon' }); expect(mockBeacon).toHaveBeenCalledWith(url, dataStringified); - expect(response).toStrictEqual({ ok: true, error: undefined }); + expect(response).toStrictEqual({}); + }); + + test('a refused beacon throws beacon-rejected, retryable up to 64 KiB', () => { + mockBeacon.mockReturnValue(false); + expect(() => sendWeb(url, data, { transport: 'beacon' })).toThrow( + expect.objectContaining({ code: 'beacon-rejected', retryable: true }), + ); + expect(() => + sendWeb(url, 'x'.repeat(65 * 1024), { transport: 'beacon' }), + ).toThrow( + expect.objectContaining({ code: 'beacon-rejected', retryable: false }), + ); + mockBeacon.mockReturnValue(true); }); test('xhr', () => { @@ -117,45 +229,31 @@ describe('sendWeb', () => { 'application/json; charset=utf-8', ); expect(mockXHR.send).toHaveBeenCalledWith(dataStringified); - expect(response).toStrictEqual({ - ok: true, - data, - error: undefined, - }); + expect(response).toStrictEqual({ data }); }); test('xhr with custom headers', () => { const headers = { 'Custom-Header': 'customValue' }; - const response = sendWeb(url, data, { - transport: 'xhr', - headers, - }); + sendWeb(url, data, { transport: 'xhr', headers }); - expect(mockXHRHeader).toHaveBeenCalledWith( - 'Content-Type', - 'application/json; charset=utf-8', - ); expect(mockXHRHeader).toHaveBeenCalledWith('Custom-Header', 'customValue'); - expect(mockXHR.send).toHaveBeenCalledWith(dataStringified); - expect(response).toStrictEqual({ - ok: true, - data, - error: undefined, - }); }); test('xhr with method option', () => { - const response = sendWeb(url, data, { - transport: 'xhr', - method: 'PUT', - }); + sendWeb(url, data, { transport: 'xhr', method: 'PUT' }); expect(mockXHROpen).toHaveBeenCalledWith('PUT', url, false); - expect(mockXHR.send).toHaveBeenCalledWith(dataStringified); - expect(response).toStrictEqual({ - ok: true, - data, - error: undefined, - }); + }); + + test('a non-2xx xhr throws a StepError with status and body', () => { + mockXHR.status = 500; + expect(() => sendWeb(url, data, { transport: 'xhr' })).toThrow( + expect.objectContaining({ + code: 'transport', + status: 500, + retryable: true, + body: data, + }), + ); }); describe('timeout validation', () => { @@ -171,105 +269,38 @@ describe('sendWeb', () => { }), ); - test('falls back to DEFAULT_FETCH_TIMEOUT when timeout is 0', async () => { - // Guard must replace 0 with DEFAULT_FETCH_TIMEOUT (10000). - // Advancing 5000ms should NOT have aborted yet (abort is at 10000). - const abortSpy = jest.spyOn(AbortController.prototype, 'abort'); - window.fetch = makeHangingFetch(); - sendWeb(url, data, { transport: 'fetch', timeout: 0 }); - jest.advanceTimersByTime(5000); - expect(abortSpy).not.toHaveBeenCalled(); - }); - - test('falls back to DEFAULT_FETCH_TIMEOUT when timeout is negative', async () => { - const abortSpy = jest.spyOn(AbortController.prototype, 'abort'); - window.fetch = makeHangingFetch(); - sendWeb(url, data, { transport: 'fetch', timeout: -1 }); - jest.advanceTimersByTime(5000); - expect(abortSpy).not.toHaveBeenCalled(); - }); - - test('falls back to DEFAULT_FETCH_TIMEOUT when timeout is NaN', async () => { - const abortSpy = jest.spyOn(AbortController.prototype, 'abort'); - window.fetch = makeHangingFetch(); - sendWeb(url, data, { transport: 'fetch', timeout: NaN }); - jest.advanceTimersByTime(5000); - expect(abortSpy).not.toHaveBeenCalled(); - }); + test.each([0, -1, NaN])( + 'falls back to the default timeout for %p', + (timeout) => { + // The default is 10000 ms: nothing is aborted after 5000 ms. + const abortSpy = jest.spyOn(AbortController.prototype, 'abort'); + window.fetch = makeHangingFetch(); + sendWeb(url, data, { transport: 'fetch', timeout }).catch( + () => undefined, + ); + jest.advanceTimersByTime(5000); + expect(abortSpy).not.toHaveBeenCalled(); + jest.advanceTimersByTime(5000); + }, + ); - test('honors an explicit positive finite timeout', async () => { + test('an explicit timeout rejects with code timeout', async () => { window.fetch = makeHangingFetch(); const promise = sendWeb(url, data, { transport: 'fetch', timeout: 50 }); jest.advanceTimersByTime(50); - const response = await promise; - expect(response.ok).toBe(false); - expect(response.error).toBeDefined(); - }); - }); - - describe('fetch timeout', () => { - test('fetch rejects with timeout error when request exceeds timeout', async () => { - // Never resolves on its own; rejects only when the signal aborts. - const abortingFetch = jest.fn( - (_input: URL | RequestInfo, init?: RequestInit) => - new Promise((_resolve, reject) => { - const signal = init?.signal; - signal?.addEventListener('abort', () => { - reject(new Error('The operation was aborted.')); - }); - }), - ); - window.fetch = abortingFetch; - - const promise = sendWeb(url, data, { - transport: 'fetch', - timeout: 50, + await expect(promise).rejects.toMatchObject({ + code: 'timeout', + retryable: true, }); - - // Advance fake timers so the AbortController's setTimeout fires. - jest.advanceTimersByTime(50); - - const response = await promise; - - expect(response.ok).toBe(false); - expect(response.error).toBeDefined(); }); - test('fetch uses default timeout when none specified', async () => { - const okFetch = jest.fn( - () => - Promise.resolve({ - text: () => Promise.resolve(''), - ok: true, - }) as Promise, - ); - window.fetch = okFetch; - + test('fetch passes an abort signal', async () => { await sendWeb(url, data, { transport: 'fetch' }); - expect(okFetch).toHaveBeenCalledWith( + expect(mockFetch).toHaveBeenCalledWith( url, expect.objectContaining({ signal: expect.any(AbortSignal) }), ); }); - - test('fetch succeeds within timeout', async () => { - const okFetch = jest.fn( - () => - Promise.resolve({ - text: () => Promise.resolve(dataStringified), - ok: true, - }) as Promise, - ); - window.fetch = okFetch; - - const response = await sendWeb(url, data, { - transport: 'fetch', - timeout: 5000, - }); - - expect(response.ok).toBe(true); - expect(response.data).toBe(dataStringified); - }); }); }); diff --git a/packages/web/core/src/index.ts b/packages/web/core/src/index.ts index fcc8e24f9..c7e83defc 100644 --- a/packages/web/core/src/index.ts +++ b/packages/web/core/src/index.ts @@ -4,6 +4,7 @@ export * from './elb'; export * from './environment'; export * from './getHashWeb'; export * from './isVisible'; +export * from './loadVendor'; export * from './sendWeb'; export * from './storage'; diff --git a/packages/web/core/src/loadVendor.ts b/packages/web/core/src/loadVendor.ts new file mode 100644 index 000000000..2b8b57486 --- /dev/null +++ b/packages/web/core/src/loadVendor.ts @@ -0,0 +1,136 @@ +import { resolveDestinationTimeout, StepError } from '@walkeros/core'; + +/** Attribute on every script `loadVendor` injects. */ +export const VENDOR_SCRIPT_MARKER = 'data-walkeros-vendor'; + +export interface LoadVendorOptions { + /** The destination's timeout; the load deadline is its resolved value. */ + timeout?: number; +} + +export interface Vendor { + /** + * Resolves with the vendor once its script loaded and `get()` finds it. + * Rejects `vendor-not-loaded` after the script's `error` (final for the + * page), `vendor-not-ready` when the vendor is missing, `vendor-timeout` + * when the script did not load by the deadline. Past the deadline, and + * without a script, it checks `get()` on each call. + */ + ready(): Promise; + /** + * For vendors that replay their own stub (called synchronously): throws + * `vendor-not-loaded` once the injected script fired `error`. + */ + check(): void; +} + +type Outcome = 'loaded' | 'failed' | 'expired'; + +interface Injected { + state: 'loading' | 'loaded' | 'failed'; + /** The deadline passed before the script loaded or failed. */ + expired: boolean; + timeout: number; + waiting: Array<(outcome: Outcome) => void>; +} + +/** One script per src on the page: a failure is never re-injected. */ +const injected = new Map(); + +function inject(src: string, timeout: number): Injected { + const known = injected.get(src); + if (known) return known; + + const entry: Injected = { + state: 'loading', + expired: false, + timeout, + waiting: [], + }; + injected.set(src, entry); + + const settle = (outcome: Outcome) => { + const waiting = entry.waiting; + entry.waiting = []; + waiting.forEach((notify) => notify(outcome)); + }; + + const timer = setTimeout(() => { + entry.expired = true; + settle('expired'); + }, timeout); + + const script = document.createElement('script'); + script.src = src; + script.async = true; + script.setAttribute(VENDOR_SCRIPT_MARKER, ''); + script.addEventListener('load', () => { + clearTimeout(timer); + entry.state = 'loaded'; + settle('loaded'); + }); + script.addEventListener('error', () => { + clearTimeout(timer); + entry.state = 'failed'; + settle('failed'); + }); + document.head.appendChild(script); + + return entry; +} + +const notLoaded = () => + new StepError('Vendor script failed to load', { code: 'vendor-not-loaded' }); + +const notReady = () => + new StepError('Vendor not ready: its global is missing', { + code: 'vendor-not-ready', + }); + +const timedOut = (timeout: number) => + new StepError(`Vendor script did not load within ${timeout}ms`, { + code: 'vendor-timeout', + }); + +/** + * Injects a vendor script once per `src` (marked `data-walkeros-vendor`) and + * reports its readiness. The deadline runs from the injection and equals the + * destination's resolved timeout, so an awaited `ready()` settles before the + * push's own timeout. Without `src` (the page loads the vendor) nothing is + * injected and there is no deadline. + */ +export function loadVendor( + src: string | undefined, + get: () => T | undefined, + options: LoadVendorOptions = {}, +): Vendor { + const script = src + ? inject(src, resolveDestinationTimeout(options.timeout)) + : undefined; + + const present = (): Promise => { + const vendor = get(); + return vendor === undefined || vendor === null + ? Promise.reject(notReady()) + : Promise.resolve(vendor); + }; + + return { + ready() { + if (!script) return present(); + if (script.state === 'failed') return Promise.reject(notLoaded()); + if (script.state === 'loaded' || script.expired) return present(); + + return new Promise((resolve, reject) => { + script.waiting.push((outcome) => { + if (outcome === 'failed') reject(notLoaded()); + else if (outcome === 'expired') reject(timedOut(script.timeout)); + else present().then(resolve, reject); + }); + }); + }, + check() { + if (script?.state === 'failed') throw notLoaded(); + }, + }; +} diff --git a/packages/web/core/src/sendWeb.ts b/packages/web/core/src/sendWeb.ts index 23713348a..0c1af83b0 100644 --- a/packages/web/core/src/sendWeb.ts +++ b/packages/web/core/src/sendWeb.ts @@ -1,9 +1,10 @@ import type { SendDataValue, SendHeaders, SendResponse } from '@walkeros/core'; import { getHeaders, + networkErrorMessage, + StepError, transformData, tryCatch, - tryCatchAsync, } from '@walkeros/core'; export type SendWebTransport = 'fetch' | 'beacon' | 'xhr'; @@ -22,6 +23,12 @@ export interface SendWebOptionsFetch extends SendWebOptions { const DEFAULT_FETCH_TIMEOUT = 10000; +/** Keepalive and beacon bodies a browser accepts in flight per page. */ +const KEEPALIVE_LIMIT = 64 * 1024; + +/** Bytes of the keepalive fetches in flight on this page. */ +let keepaliveBytes = 0; + type SendWebOptionsDynamic = T extends 'fetch' ? SendWebOptionsFetch : SendWebOptions; @@ -30,6 +37,14 @@ export type SendWebReturn = T extends 'fetch' ? Promise : SendResponse; +/** + * Send a request with `fetch` (default), `navigator.sendBeacon` or a + * synchronous XHR. Resolves (fetch) or returns (beacon, xhr) `{ data }` when + * the request was sent, and throws a `StepError` otherwise: `transport` with + * `status` and `body` for a non-2xx answer or a network error, `timeout` for + * a fetch past its timeout, `beacon-rejected` when the browser refuses a + * beacon. The body stays on the error, never in its message. + */ export function sendWeb( url: string, data?: SendDataValue, @@ -65,38 +80,52 @@ export async function sendWebAsFetch( const controller = new AbortController(); const timeoutId = setTimeout(() => controller.abort(), timeout); - return tryCatchAsync( - async () => { - try { - const response = await fetch(url, { - method: options.method || 'POST', - headers, - keepalive: true, - credentials: options.credentials || 'same-origin', - mode: options.noCors ? 'no-cors' : 'cors', - body, - signal: controller.signal, + // Browsers refuse a keepalive request once the keepalive bodies in flight + // exceed 64 KiB; past that a request goes out without keepalive instead. + const size = body ? byteLength(body) : 0; + const keepalive = keepaliveBytes + size <= KEEPALIVE_LIMIT; + if (keepalive) keepaliveBytes += size; + + try { + let response: Response; + try { + response = await fetch(url, { + method: options.method || 'POST', + headers, + keepalive, + credentials: options.credentials || 'same-origin', + mode: options.noCors ? 'no-cors' : 'cors', + body, + signal: controller.signal, + }); + } catch (error) { + if (controller.signal.aborted) + throw new StepError(`Request timed out after ${timeout}ms`, { + code: 'timeout', + cause: error, }); - - const responseData = options.noCors ? '' : await response.text(); - - return { - ok: response.ok, - data: responseData, - error: response.ok ? undefined : response.statusText, - }; - } finally { - clearTimeout(timeoutId); - } - }, - (error) => { - clearTimeout(timeoutId); - return { - ok: false, - error: (error as Error).message, - }; - }, - )(); + throw new StepError(networkErrorMessage(error), { + code: 'transport', + cause: error, + }); + } + + // A no-cors response is opaque: it counts as sent once it arrived. + if (options.noCors) return { data: '' }; + + const responseData = await response.text(); + if (!response.ok) + throw new StepError(`HTTP ${response.status}`, { + code: 'transport', + status: response.status, + body: responseData, + }); + + return { data: responseData }; + } finally { + clearTimeout(timeoutId); + if (keepalive) keepaliveBytes -= size; + } } export function sendWebAsBeacon( @@ -104,12 +133,17 @@ export function sendWebAsBeacon( data?: SendDataValue, ): SendResponse { const body = transformData(data); - const ok = navigator.sendBeacon(url, body); - return { - ok, - error: ok ? undefined : 'Failed to send beacon', - }; + if (!navigator.sendBeacon(url, body)) { + // Over the browser's 64 KiB quota a retry cannot help; a full queue can. + const size = body ? byteLength(body) : 0; + throw new StepError('Beacon rejected by the browser', { + code: 'beacon-rejected', + retryable: size <= KEEPALIVE_LIMIT, + }); + } + + return {}; } export function sendWebAsXhr( @@ -121,30 +155,47 @@ export function sendWebAsXhr( const method = options.method || 'POST'; const body = transformData(data); - return tryCatch( - () => { - const xhr = new XMLHttpRequest(); - xhr.open(method, url, false); // Synchronous request - for (const header in headers) { - xhr.setRequestHeader(header, headers[header]); - } - xhr.send(body); - - const ok = xhr.status >= 200 && xhr.status < 300; - - const parsedData = tryCatch(JSON.parse, () => xhr.response)(xhr.response); - - return { - ok, - data: parsedData, - error: ok ? undefined : `${xhr.status} ${xhr.statusText}`, - }; - }, - (error) => { - return { - ok: false, - error: (error as Error).message, - }; - }, - )(); + let xhr: XMLHttpRequest; + try { + xhr = new XMLHttpRequest(); + xhr.open(method, url, false); // Synchronous request + for (const header in headers) { + xhr.setRequestHeader(header, headers[header]); + } + xhr.send(body); + } catch (error) { + throw new StepError(networkErrorMessage(error), { + code: 'transport', + cause: error, + }); + } + + const parsedData: unknown = tryCatch( + JSON.parse, + () => xhr.response, + )(xhr.response); + + if (xhr.status < 200 || xhr.status >= 300) + throw new StepError(`HTTP ${xhr.status}`, { + code: 'transport', + status: xhr.status || undefined, + body: parsedData, + }); + + return { data: parsedData }; +} + +/** UTF-8 length of a request body, without TextEncoder. */ +function byteLength(text: string): number { + let bytes = 0; + for (let i = 0; i < text.length; i++) { + const code = text.charCodeAt(i); + if (code < 0x80) bytes += 1; + else if (code < 0x800) bytes += 2; + else if (code >= 0xd800 && code <= 0xdbff) { + bytes += 4; + i++; // The low surrogate belongs to this character. + } else bytes += 3; + } + return bytes; } diff --git a/packages/web/destinations/gtag/package.json b/packages/web/destinations/gtag/package.json index 18dc88222..414cf6402 100644 --- a/packages/web/destinations/gtag/package.json +++ b/packages/web/destinations/gtag/package.json @@ -41,6 +41,9 @@ "@walkeros/core": "4.8.1", "@walkeros/web-core": "4.8.1" }, + "devDependencies": { + "@walkeros/collector": "4.8.1" + }, "repository": { "url": "git+https://github.com/elbwalker/walkerOS.git", "directory": "packages/web/destinations/gtag" From b69a7f0bfb5d83a6b114aabe854eac2fd5475d44 Mon Sep 17 00:00:00 2001 From: alexanderkirtzel Date: Wed, 21 Oct 2026 02:28:13 +0000 Subject: [PATCH 05/41] contract fixes --- .../src/__tests__/delivery-order.test.ts | 124 +++- .../destination-batch-correctness.test.ts | 25 +- .../destination-outcome-state.test.ts | 56 +- .../collector/src/__tests__/shutdown.test.ts | 10 +- packages/collector/src/breaker.ts | 42 +- packages/collector/src/collector.ts | 54 +- .../src/destination-outcome-state.ts | 10 - packages/collector/src/destination.ts | 666 ++++++++++-------- packages/collector/src/handle.ts | 5 +- packages/collector/src/observerEmit.ts | 28 +- packages/collector/src/outcome.ts | 53 +- packages/collector/src/shutdown.ts | 38 +- packages/collector/src/source.ts | 9 +- packages/collector/src/transformer.ts | 10 +- packages/core/src/types/telemetry.ts | 1 - 15 files changed, 722 insertions(+), 409 deletions(-) diff --git a/packages/collector/src/__tests__/delivery-order.test.ts b/packages/collector/src/__tests__/delivery-order.test.ts index 9af653148..70a6a49cd 100644 --- a/packages/collector/src/__tests__/delivery-order.test.ts +++ b/packages/collector/src/__tests__/delivery-order.test.ts @@ -1,6 +1,45 @@ -import type { Destination, WalkerOS } from '@walkeros/core'; +import type { Collector, Destination, Elb, WalkerOS } from '@walkeros/core'; import { startFlow } from '..'; +/** A push's answer, or `'hang'` when it has not answered after many turns. */ +async function within( + push: Promise, +): Promise { + let answer: Elb.PushResult | undefined; + void push.then((result) => { + answer = result; + }); + for (let turn = 0; turn < 1000 && !answer; turn++) await Promise.resolve(); + return answer ?? 'hang'; +} + +/** A consent-gated destination whose init takes several turns. */ +async function gatedFlow(consentDenied: 'skip' | 'queue') { + const names: string[] = []; + const destination: Destination.Instance = { + type: 'gated', + config: { consent: { marketing: true } }, + init: async () => { + for (let turn = 0; turn < 10; turn++) await Promise.resolve(); + }, + push: (event: WalkerOS.Event) => { + names.push(event.name); + }, + }; + const { collector } = await startFlow({ + consentDenied, + logger: { handler: () => undefined }, + destinations: { d: { code: destination } }, + }); + return { collector, names }; +} + +const pushWith = ( + collector: Collector.Instance, + name: string, + marketing: boolean, +) => within(collector.push({ name, consent: { marketing } })); + describe('delivery order across a destination init', () => { it('delivers events pushed without awaiting in push order to a destination without init', async () => { const names: string[] = []; @@ -20,7 +59,7 @@ describe('delivery order across a destination init', () => { expect(names).toEqual(['a one', 'b two', 'c three']); }); - it('keeps push order behind an async init too', async () => { + it('keeps push order behind an async init; a push behind it answers once its event was delivered', async () => { const names: string[] = []; const destination: Destination.Instance = { type: 'recorder', @@ -37,9 +76,26 @@ describe('delivery order across a destination init', () => { destinations: { recorder: { code: destination } }, }); - await Promise.all([elb('a one'), elb('b two'), elb('c three')]); + // What the destination had received when each push answered. + const answered = (name: string) => + elb(name).then((result) => ({ result, seen: [...names] })); + const [a, b, c] = await Promise.all([ + answered('a one'), + answered('b two'), + answered('c three'), + ]); expect(names).toEqual(['a one', 'b two', 'c three']); + for (const [push, name] of [ + [a, 'a one'], + [b, 'b two'], + [c, 'c three'], + ] as const) { + expect(push.seen).toContain(name); + expect(push.result.ok).toBe(true); + expect(push.result.done?.recorder).toBeDefined(); + expect(push.result.queued).toBeUndefined(); + } }); it('fails the events queued behind an init that fails, never leaves them waiting', async () => { @@ -57,12 +113,72 @@ describe('delivery order across a destination init', () => { destinations: { broken: { code: destination } }, }); - await Promise.all([elb('a one'), elb('b two'), elb('c three')]); + const results = await Promise.all([ + elb('a one'), + elb('b two'), + elb('c three'), + ]); + // Every push, the ones queued behind the init too, answers the init's + // failure: none is acknowledged before its event was tried. + for (const result of results) { + expect(result.ok).toBe(false); + expect(result.failed?.broken.reason).toMatchObject({ + code: 'transport', + message: 'init failed', + }); + } expect(collector.status.destinations.broken.failed).toBe(3); expect(collector.destinations.broken.queuePush ?? []).toHaveLength(0); expect( (collector.destinations.broken.dlq ?? []).map(([event]) => event.name), ).toEqual(['a one', 'b two', 'c three']); }); + + it.each<['skip' | 'queue']>([['skip'], ['queue']])( + 'consentDenied %s: pushes behind the init keep their own event consent, all answer and are delivered in order', + async (mode) => { + const { collector, names } = await gatedFlow(mode); + + const results = await Promise.all([ + pushWith(collector, 'a one', true), + pushWith(collector, 'b two', true), + pushWith(collector, 'c three', true), + ]); + + expect(names).toEqual(['a one', 'b two', 'c three']); + for (const result of results) { + expect(result).not.toBe('hang'); + if (result === 'hang') continue; + expect(result.ok).toBe(true); + expect(result.done?.d).toBeDefined(); + } + }, + ); + + it.each<['skip' | 'queue']>([['skip'], ['queue']])( + 'consentDenied %s: a push whose own consent is denied answers at once, the others are delivered', + async (mode) => { + const { collector, names } = await gatedFlow(mode); + + const [a, b, c] = await Promise.all([ + pushWith(collector, 'a one', true), + pushWith(collector, 'b two', false), + pushWith(collector, 'c three', true), + ]); + + expect(names).toEqual(['a one', 'c three']); + for (const result of [a, c]) expect(result).not.toBe('hang'); + expect(b).not.toBe('hang'); + if (b === 'hang') return; + expect(b.ok).toBe(true); + expect(b.done).toBeUndefined(); + if (mode === 'skip') { + expect(b.queued).toBeUndefined(); + expect(collector.status.destinations.d.skipped).toBe(1); + } else { + expect(b.queued?.d.reason?.code).toBe('consent'); + } + }, + ); }); diff --git a/packages/collector/src/__tests__/destination-batch-correctness.test.ts b/packages/collector/src/__tests__/destination-batch-correctness.test.ts index 6e1221ed1..eb5dfefdc 100644 --- a/packages/collector/src/__tests__/destination-batch-correctness.test.ts +++ b/packages/collector/src/__tests__/destination-batch-correctness.test.ts @@ -578,20 +578,23 @@ describe('Destination batch correctness (PROD-004)', () => { await collector.destinations['d'].batches!['* *'].flush(); - const lines = errors.filter( - (e) => e.message === 'Push batch partial failure', + // One cause line: the counts and every cause (the error's code, else its + // name) in the message, the first row's reason in the structured fields. + const lines = errors.filter((e) => + e.message.startsWith('Push batch partial failure'), ); expect(lines).toHaveLength(1); - expect(lines[0].context).toMatchObject({ - failed: 4, - delivered: 1, - entries: 5, + expect(lines[0].message).toContain( + 'Push batch partial failure: 4 of 5 entries failed (A x2, B x1, TypeError x1)', + ); + expect(lines[0].context).toEqual({ + step: 'destination.d', + code: 'transport', + type: 'partial-causes', + count: 4, + retryable: true, + error: 'a first', }); - expect(lines[0].context.causes).toEqual([ - { code: 'A', count: 2, message: 'a first' }, - { code: 'B', count: 1, message: 'b only' }, - { code: 'TypeError', count: 1, message: 'plain' }, - ]); }); it('a BatchOutcome with empty failed counts the whole batch delivered', async () => { diff --git a/packages/collector/src/__tests__/destination-outcome-state.test.ts b/packages/collector/src/__tests__/destination-outcome-state.test.ts index a198a5749..ebda1f159 100644 --- a/packages/collector/src/__tests__/destination-outcome-state.test.ts +++ b/packages/collector/src/__tests__/destination-outcome-state.test.ts @@ -1,4 +1,4 @@ -import { assembleJourneys, Level } from '@walkeros/core'; +import { assembleJourneys, Level, StepError } from '@walkeros/core'; import type { Collector, Destination, @@ -143,11 +143,12 @@ describe('destination states, consent first', () => { message: 'a state delivery to the destination is running or failed', }); expect(held.failed).toBeUndefined(); + // Held is a waiting state: no Observe record until its out or error. expect( states.some( - (state) => state.phase === 'skip' && state.skipReason === 'held', + (state) => state.stepId === 'destination.d' && state.phase === 'skip', ), - ).toBe(true); + ).toBe(false); jest.setSystemTime(1_000_000 + 1001); const blocked = await elb('page view'); @@ -199,6 +200,40 @@ describe('destination states, consent first', () => { expect(codes).toEqual(['transport', 'breaker-open']); }); + it('(d) a deterministic failure never opens the breaker', async () => { + const { logger } = capture(); + const push = jest.fn(async () => { + throw new StepError('Config settings apiKey missing', { + code: 'config', + }); + }); + const { elb, collector } = await startFlow({ + logger, + destinations: { + d: { + code: { + type: 'probe', + config: { breaker: { threshold: 2, cooldown: 60_000 } }, + push, + }, + }, + }, + }); + + const results = []; + for (let i = 0; i < 4; i++) results.push(await elb('page view')); + + expect(push).toHaveBeenCalledTimes(4); + for (const r of results) + expect(r.failed?.d.reason).toMatchObject({ + code: 'config', + retryable: false, + }); + expect(collector.status.breakers['destination.d']?.state ?? 'closed').toBe( + 'closed', + ); + }); + it('(e) held overflow evicts consent waits first, then refuses the newest held event', async () => { const { at, logger } = capture(); const push = jest.fn(); @@ -331,7 +366,7 @@ describe('destination states, consent first', () => { ); }); - it('(g) a batched event waits with reason batch and keeps its batched out; a held event emits skip held', async () => { + it('(g) a batched event waits with reason batch and keeps its batched out; a held event emits nothing yet', async () => { const { logger } = capture(); const { states, observer } = observe(); const pushBatch = jest.fn(); @@ -372,7 +407,7 @@ describe('destination states, consent first', () => { expect(pushBatch).not.toHaveBeenCalled(); // Observe: the batch-stamped out stays the batch signal (no skip for a - // batch); the held destination emits skip with reason held. + // batch); a held event is waiting too: no record until its out or error. const eventId = r.event?.id; const ofEvent = states.filter((state) => state.eventId === eventId); expect( @@ -380,14 +415,9 @@ describe('destination states, consent first', () => { (state) => state.stepId === 'destination.d' && state.phase === 'skip', ), ).toBe(false); - expect( - ofEvent.some( - (state) => - state.stepId === 'destination.h' && - state.phase === 'skip' && - state.skipReason === 'held', - ), - ).toBe(true); + expect(ofEvent.some((state) => state.stepId === 'destination.h')).toBe( + false, + ); const hopOf = () => assembleJourneys(states) .journeys.find((journey) => journey.id === eventId) diff --git a/packages/collector/src/__tests__/shutdown.test.ts b/packages/collector/src/__tests__/shutdown.test.ts index 02223bcfb..e77f0469c 100644 --- a/packages/collector/src/__tests__/shutdown.test.ts +++ b/packages/collector/src/__tests__/shutdown.test.ts @@ -29,8 +29,14 @@ function makeErrorCapture(): { handler: Logger.Handler; } { const messages: string[] = []; - const handler: Logger.Handler = (level, message) => { - if (level === Level.ERROR) messages.push(message); + // A cause line carries the failure's message in its context. + const handler: Logger.Handler = (level, message, context) => { + if (level === Level.ERROR) + messages.push( + typeof context.error === 'string' + ? `${message}: ${context.error}` + : message, + ); }; return { messages, handler }; } diff --git a/packages/collector/src/breaker.ts b/packages/collector/src/breaker.ts index 1370cc82d..2c4679c2d 100644 --- a/packages/collector/src/breaker.ts +++ b/packages/collector/src/breaker.ts @@ -1,4 +1,5 @@ -import type { BreakerState, Destination } from '@walkeros/core'; +import type { BreakerState, Collector, Destination } from '@walkeros/core'; +import { stepId } from '@walkeros/core'; /** * Step-general circuit breaker. @@ -200,3 +201,42 @@ export function isBreakerProbePermitted( if (breaker.state === 'closed' || breaker.state === 'half-open') return true; return breaker.openUntil !== undefined && now >= breaker.openUntil; } + +/** + * Record a destination's push outcome on its breaker, keyed on its canonical + * id (`config.id`, else its id in the flow) like the gate. Presence-gated: + * a no-op without `config.breaker`. The one entry for the delivery pass, the + * batch flush and the outcome sink. + */ +export function recordBreaker( + collector: Collector.Instance, + destination: Destination.Instance, + id: string, + outcome: StepOutcome, +): void { + const config = resolveBreakerConfig(destination.config.breaker); + if (!config) return; + recordStepOutcome( + collector.status.breakers, + stepId('destination', destination.config.id || id), + outcome, + config.threshold, + config.cooldown, + ); +} + +/** + * Release the half-open probe a destination's delivery pass took without + * reaching the transport. Presence-gated like `recordBreaker`. + */ +export function releaseBreakerProbe( + collector: Collector.Instance, + destination: Destination.Instance, + id: string, +): void { + if (!resolveBreakerConfig(destination.config.breaker)) return; + releaseProbe( + collector.status.breakers, + stepId('destination', destination.config.id || id), + ); +} diff --git a/packages/collector/src/collector.ts b/packages/collector/src/collector.ts index 048b38fdc..248f65efb 100644 --- a/packages/collector/src/collector.ts +++ b/packages/collector/src/collector.ts @@ -140,31 +140,35 @@ export async function collector( ); // Registration check of every chain known now: each transformer it names - // must exist (one ERROR per missing id, not one per event). - checkChainTargets(collector, 'collector.next', config.next); - for (const [id, destination] of Object.entries(collector.destinations)) { - checkChainTargets( - collector, - `destination.${id}.before`, - destination.config.before, - ); - checkChainTargets( - collector, - `destination.${id}.next`, - destination.config.next, - ); - } - for (const [id, transformer] of Object.entries(collector.transformers)) { - checkChainTargets( - collector, - `transformer.${id}.before`, - transformer.config.before, - ); - checkChainTargets( - collector, - `transformer.${id}.next`, - transformer.config.next, - ); + // must exist (one ERROR per missing id, not one per event). Build flag: + // CLI bundles check routes at build time and fold this out; the runtime + // still names a missing id once when an event meets it. + if (typeof __WALKEROS_VALIDATE__ === 'undefined' || __WALKEROS_VALIDATE__) { + checkChainTargets(collector, 'collector.next', config.next); + for (const [id, destination] of Object.entries(collector.destinations)) { + checkChainTargets( + collector, + `destination.${id}.before`, + destination.config.before, + ); + checkChainTargets( + collector, + `destination.${id}.next`, + destination.config.next, + ); + } + for (const [id, transformer] of Object.entries(collector.transformers)) { + checkChainTargets( + collector, + `transformer.${id}.before`, + transformer.config.before, + ); + checkChainTargets( + collector, + `transformer.${id}.next`, + transformer.config.next, + ); + } } return collector; diff --git a/packages/collector/src/destination-outcome-state.ts b/packages/collector/src/destination-outcome-state.ts index b9783a52f..7d7b02151 100644 --- a/packages/collector/src/destination-outcome-state.ts +++ b/packages/collector/src/destination-outcome-state.ts @@ -34,16 +34,6 @@ export const HELD_WAIT: QueuedReason = { message: 'a state delivery to the destination is running or failed', }; -/** - * Waiting behind the destination's init, which another push started: the - * event goes out right after it, in push order. - */ -export const INIT_WAIT: QueuedReason = { - code: 'held', - message: - "the destination's init is running; the event goes out right after it", -}; - const READY: DestinationState = { state: 'ready' }; /** diff --git a/packages/collector/src/destination.ts b/packages/collector/src/destination.ts index 465f98c88..6f93f69a5 100644 --- a/packages/collector/src/destination.ts +++ b/packages/collector/src/destination.ts @@ -31,18 +31,28 @@ import { isObject, OBSERVE_ENV_KEY, processEventMapping, + reportCause, StepError, stepId, tryCatchAsync, useHooks, } from '@walkeros/core'; -import { journeyFields, observerStepError, stepState } from './observerEmit'; -import { isTransportFailure, recordFailure, recordSkip } from './outcome'; +import { + emitDestinationState, + journeyFields, + observerStepError, + stepState, +} from './observerEmit'; +import { + initReturnedFalse, + isTransportFailure, + recordFailure, + recordSkip, +} from './outcome'; import { BATCH_WAIT, CONSENT_WAIT, HELD_WAIT, - INIT_WAIT, breakerState, heldState, } from './destination-outcome-state'; @@ -83,11 +93,7 @@ import { } from './report-error'; import { reconcilePending } from './pending'; import { runCollectorNext } from './collector-next'; -import { - recordStepOutcome, - releaseProbe, - resolveBreakerConfig, -} from './breaker'; +import { recordBreaker, releaseBreakerProbe } from './breaker'; /** Default upper-bound on entries per batch. Caps unbounded growth under sustained load. */ const DEFAULT_BATCH_SIZE = 1_000; @@ -415,6 +421,82 @@ export async function pushToDestinations( return mergePushResults(first, failure ? [...rest, failure] : rest); } +/** What became of one event of a delivery pass at one destination. */ +type EntryOutcome = + | { kind: 'delivered'; response?: unknown } + | { kind: 'batched' } + | { kind: 'failed'; reason: FailureReason; error: unknown } + | { kind: 'queued'; reason: QueuedReason } + | { kind: 'skipped' }; + +/** + * Pushes that queued behind a running init wait here for their own events' + * outcome; the pass that runs the init (or the next pass to take the + * events) settles them, so each push answers what became of its own events. + */ +const waiting = new WeakMap< + Destination.Queued, + (outcome: EntryOutcome) => void +>(); + +/** Settle the pushes waiting for these entries, if any. */ +function settle(entries: Destination.Queued[], outcome: EntryOutcome): void { + for (const entry of entries) { + const resolve = waiting.get(entry); + if (!resolve) continue; + waiting.delete(entry); + resolve(outcome); + } +} + +/** + * After a pass: the entries still waiting in a destination's queue (held + * after its init, or waiting for consent) answer as waiting. Not while an + * init of the destination runs: its pass settles them. + */ +function settleWaiting(destination: Destination.Instance): void { + if (initsRunning.has(destination) || initCallers.has(destination)) return; + settleQueued(destination.queuePush || []); +} + +/** + * Passes awaiting `destinationInit`, per destination. `destinationInit` + * drops its running mark one turn before its caller resumes and takes the + * entries queued behind the init; this mark lasts until the caller resumes + * (released in the caller's own `finally`, which runs in that resumption, + * before the synchronous take), so no other pass answers those entries + * early in between. + */ +const initCallers = new WeakMap(); + +function beginInitCall(destination: Destination.Instance): void { + initCallers.set(destination, (initCallers.get(destination) ?? 0) + 1); +} + +function endInitCall(destination: Destination.Instance): void { + const count = (initCallers.get(destination) ?? 1) - 1; + if (count > 0) initCallers.set(destination, count); + else initCallers.delete(destination); +} + +/** The pushes waiting for these queue entries answer that they wait. */ +function settleQueued(entries: Destination.Queued[]): void { + for (const entry of entries) + settle([entry], { + kind: 'queued', + reason: entry.reason === 'consent' ? CONSENT_WAIT : HELD_WAIT, + }); +} + +/** + * Empty a destination's queue (a new run): the pushes still waiting for + * one of its entries answer that it was waiting. + */ +export function clearQueuePush(destination: Destination.Instance): void { + settleQueued(destination.queuePush || []); + destination.queuePush = []; +} + /** What one destination did with one delivery pass. */ interface DestinationOutcome { id: string; @@ -434,6 +516,38 @@ interface DestinationOutcome { batched?: number; } +/** A pass's outcome from its events' outcomes (failed, then delivered, then waiting). */ +function passOutcome( + id: string, + destination: Destination.Instance, + outcomes: EntryOutcome[], + totalDuration = 0, +): DestinationOutcome { + let failed: { reason: FailureReason; error: unknown } | undefined; + let queued: QueuedReason | undefined; + let delivered = 0; + let batched = 0; + let response: unknown; + for (const outcome of outcomes) { + if (outcome.kind === 'failed') failed = failed ?? outcome; + else if (outcome.kind === 'delivered') { + delivered++; + if (outcome.response !== undefined) response = outcome.response; + } else if (outcome.kind === 'batched') batched++; + else if (outcome.kind === 'queued') queued = queued ?? outcome.reason; + } + return { + id, + destination, + totalDuration, + delivered, + batched, + response, + ...(failed ? { failed: failed.reason, error: failed.error } : {}), + ...(queued ? { queued } : {}), + }; +} + /** * Delivers one event (or, without one, the queued events only) to all or a * subset of destinations. Module-private: every caller goes through @@ -493,10 +607,12 @@ async function deliverToDestinations( // unset; routes then resolve against an empty set. const transformers: Transformer.Transformers = collector.transformers || {}; - const results = await Promise.all( - // Process all destinations in parallel - Object.entries(destinations || {}).map( - async ([id, destination]): Promise => { + const passes = Object.entries(destinations || {}); + let results: DestinationOutcome[]; + try { + results = await Promise.all( + // Process all destinations in parallel + passes.map(async ([id, destination]): Promise => { // Disabled destinations are completely skipped — no queuing, no init, no processing if (destination.config.disabled) { return { id, destination, skipped: true }; @@ -509,30 +625,14 @@ async function deliverToDestinations( // never accumulate against the entry the gate inspects and the // breaker never opens. Resolve once here and thread it through. const canonicalId = destination.config.id || id; - const breakerKey = stepId('destination', canonicalId); - const breakerConfig = resolveBreakerConfig(destination.config.breaker); - - // Probe-settle helpers. When the breaker gate below admitted a - // half-open probe, EVERY later path must settle it: either record an - // outcome (the event reached the transport) or release the probe (it - // never did). Otherwise `probing` stays true and the breaker - // deadlocks half-open. Both are presence-gated and internally no-op - // unless half-open+probing. - const recordProbe = (outcome: 'transport-failure' | 'success') => { - if (breakerConfig) { - recordStepOutcome( - collector.status.breakers, - breakerKey, - outcome, - breakerConfig.threshold, - breakerConfig.cooldown, - ); - } - }; - const releaseProbeSlot = () => { - if (breakerConfig) - releaseProbe(collector.status.breakers, breakerKey); - }; + + // Probe settle. When the breaker gate below admitted a half-open + // probe, EVERY later path must settle it: record an outcome (the + // event reached the transport, `recordBreaker`) or release the probe + // (it never did). Otherwise `probing` stays true and the breaker + // deadlocks half-open. A no-op unless half-open and probing. + const releaseProbeSlot = () => + releaseBreakerProbe(collector, destination, id); const queueMax = destination.config.queueMax ?? DEFAULT_QUEUE_MAX; const queueStep = stepId('destination', canonicalId); @@ -564,10 +664,16 @@ async function deliverToDestinations( const oldest = queuePush.findIndex( (queued) => queued.reason === 'consent', ); + // A push waiting for an evicted or dropped consent wait + // answers it waited for consent, as at its push. if (oldest !== -1) { - queuePush.splice(oldest, 1); + settle(queuePush.splice(oldest, 1), { + kind: 'queued', + reason: CONSENT_WAIT, + }); evicted++; } else if (reason === 'consent') { + settle([entry], { kind: 'queued', reason: CONSENT_WAIT }); evicted++; continue; } else { @@ -607,31 +713,49 @@ async function deliverToDestinations( const requeueHeld = (entries: Destination.Queued[]) => enqueue(entries, 'held'); - // The entries concurrent pushes queued behind a running init (held, - // not among `keep`), taken out of the queue in push order. - const takeQueuedBehindInit = (keep: Set) => { + // The entries pushes queued behind a running init (they wait for + // their outcome), taken out of the queue in push order. + const takeWaiting = () => { const queuePush = destination.queuePush || []; - const behind = queuePush.filter( - (queued) => queued.reason === 'held' && !keep.has(queued), - ); destination.queuePush = queuePush.filter( - (queued) => !behind.includes(queued), + (queued) => !waiting.has(queued), ); - return behind; + return queuePush.filter((queued) => waiting.has(queued)); }; - // Fail refused held entries through the one outcome sink. - const refuseHeld = (refused: Destination.Queued[]) => - recordFailure( + // Hold entries in the destination's own queue; those the full queue + // refuses fail with `queue-full` through the one outcome sink. + const holdOrRefuse = ( + entries: Destination.Queued[], + ): DestinationOutcome => { + const refused = requeueHeld(entries); + if (!refused.length) return { id, destination, queued: HELD_WAIT }; + const error = new StepError( + `destination.queuePush full (queueMax ${queueMax}) while the destination is held: newest event refused`, + { code: 'queue-full' }, + ); + const failed = recordFailure( collector, destination, id, refused.map((entry) => entry.event), - new StepError( - `destination.queuePush full (queueMax ${queueMax}) while the destination is held: newest event refused`, - { code: 'queue-full' }, - ), + error, ); + settle(refused, { kind: 'failed', reason: failed, error }); + return { id, destination, failed, error }; + }; + + // A queued entry goes out with the collector's current consent (full + // replace: stale consent must not persist); the entry keeps its + // identity, the event inside is copied. An entry a push is waiting + // for was admitted at that push with its own event consent and keeps + // it: on a server the collector's consent is usually empty, and + // replacing it would deny an event whose consent was granted. + const refresh = (entry: Destination.Queued) => { + if (!waiting.has(entry)) + entry.event = { ...entry.event, consent: collector.consent }; + return entry; + }; // Per-event consent gate: granted events keep only their granted // states and are delivered; denied events emit a skip state so @@ -667,27 +791,24 @@ async function deliverToDestinations( continue; } - const skipState = stepState( + emitDestinationState( collector, - stepId('destination', id), - 'destination', + id, 'skip', - typeof queuedEvent.id === 'string' ? queuedEvent.id : '', - Date.now(), - journeyFields(queuedEvent, destIngest, collector), + queuedEvent, + destIngest, + (state) => { + state.skipReason = 'consent'; + if (consent) state.consent = { ...consent }; + if (destination.config.consent) + state.meta = { required: { ...destination.config.consent } }; + }, ); - skipState.skipReason = 'consent'; - if (consent) skipState.consent = { ...consent }; - if (destination.config.consent) { - skipState.meta = { - required: { ...destination.config.consent }, - }; - } - emitStep(collector, skipState); - if (consentDenied === 'skip') + if (consentDenied === 'skip') { recordSkip(collector, destination, id, queuedEvent, 'consent'); - else denied.push(entry); + settle([entry], { kind: 'skipped' }); + } else denied.push(entry); } return { allowed, denied }; }; @@ -724,9 +845,10 @@ async function deliverToDestinations( // State hold: a state delivery to this destination is running or did // not settle. Consent first: a denied event waits for consent as // anywhere else. A granted one is held in the destination's own queue - // for one retry window (`queued`, reason `held`); once the hold - // outlasts that window the destination is blocked: the event and the - // held ones before it fail (`blocked`) until a retry succeeds. + // for one retry window (`queued`, reason `held`; a waiting state, no + // Observe record until its `out` or `error`); once the hold outlasts + // that window the destination is blocked: the event and the held ones + // before it fail (`blocked`) until a retry succeeds. const held = heldState(destination, timeout); if (held.state !== 'ready') { if (!event) return { id, destination, skipped: true }; @@ -752,29 +874,11 @@ async function deliverToDestinations( [...heldBefore, ...allowed].map((entry) => entry.event), error, ); + settle(heldBefore, { kind: 'failed', reason: failed, error }); return { id, destination, failed, error }; } - const refused = requeueHeld(allowed); - for (const entry of allowed) { - if (refused.includes(entry)) continue; - const skipState = stepState( - collector, - stepId('destination', id), - 'destination', - 'skip', - typeof entry.event.id === 'string' ? entry.event.id : '', - Date.now(), - journeyFields(entry.event, destIngest, collector), - ); - skipState.skipReason = 'held'; - emitStep(collector, skipState); - } - if (refused.length) { - const failed = refuseHeld(refused); - return { id, destination, failed }; - } - return { id, destination, queued: HELD_WAIT }; + return holdOrRefuse(allowed); } // Queued events: refresh consent (full replace — stale consent must @@ -782,10 +886,7 @@ async function deliverToDestinations( // copied. User merge happens for all events below. let currentQueue: Destination.Queued[] = ( destination.queuePush || [] - ).map((entry) => { - entry.event = { ...entry.event, consent: collector.consent }; - return entry; - }); + ).map(refresh); destination.queuePush = []; // Current event: added as-is (consent is already fresh from createEvent). @@ -814,6 +915,7 @@ async function deliverToDestinations( return { id, destination, skipped: true }; } let isInitialized = false; + beginInitCall(destination); try { isInitialized = await destinationInit( collector, @@ -822,16 +924,21 @@ async function deliverToDestinations( true, ); } catch (err) { - // Events queued behind this init by concurrent pushes fail with - // it, rather than waiting for a next push that may never come. - recordFailure( + // Events queued behind this init fail with it, rather than + // waiting for a next push that may never come. + const behind = takeWaiting(); + const reason = recordFailure( collector, destination, id, - takeQueuedBehindInit(new Set()).map((entry) => entry.event), + behind.map((entry) => entry.event), err, { message: 'destination init failed' }, ); + settle(behind, { kind: 'failed', reason, error: err }); + } finally { + // From here to the take below runs without a turn. + endInitCall(destination); } // The flush held the destination: there are no events of this call // to requeue, and any parked by a concurrent push stay queued. @@ -841,10 +948,7 @@ async function deliverToDestinations( // Events parked by a concurrent push while the flush was running go // out with this call, through the consent gate below. if (isInitialized && destination.queuePush?.length) { - currentQueue = destination.queuePush.map((entry) => { - entry.event = { ...entry.event, consent: collector.consent }; - return entry; - }); + currentQueue = destination.queuePush.map(refresh); destination.queuePush = []; } else { return { id, destination, skipped: true }; @@ -879,6 +983,7 @@ async function deliverToDestinations( allowedEvents.map((entry) => entry.event), error, ); + settle(allowedEvents, { kind: 'failed', reason: failed, error }); return { id, destination, failed, error }; } @@ -888,18 +993,24 @@ async function deliverToDestinations( const parked = new Set(destination.queuePush); // An init of this destination (with its queue flush) is running in - // another pass: the admitted events queue behind it, and that pass + // another pass: the admitted events queue behind it and that pass // delivers them right after its own, so the destination receives its - // events in push order. A destination without an `init` runs a no-op - // one through the same path, so it is covered too. + // events in push order. This push answers only once its events were + // tried (delivered, or failed with the reason, a failed init's too), + // never before: a producer must not acknowledge an untried event. if (initsRunning.has(destination)) { releaseProbeSlot(); - const refused = requeueHeld(allowedEvents); - if (refused.length) { - const failed = refuseHeld(refused); - return { id, destination, failed }; - } - return { id, destination, queued: INIT_WAIT }; + // Entries another push queued already keep their own waiter. + const outcomes = allowedEvents + .filter((entry) => !waiting.has(entry)) + .map( + (entry) => + new Promise((resolve) => + waiting.set(entry, resolve), + ), + ); + holdOrRefuse(allowedEvents); + return passOutcome(id, destination, await Promise.all(outcomes)); } // Initialize the destination if needed. An init that throws, or @@ -907,28 +1018,35 @@ async function deliverToDestinations( // events through the outcome sink; init is re-attempted on the next // push. Allowed: at least one event cleared the per-event consent // gate above, so initialization is authorized (true) for the - // chokepoint guard. + // chokepoint guard, and the init answers true or throws. + beginInitCall(destination); try { - if (!(await destinationInit(collector, destination, id, true))) - throw new StepError('destination init was refused', { - code: 'not-ready', - }); + await destinationInit(collector, destination, id, true); } catch (err) { - // The events queued behind this init by concurrent pushes fail with - // it, in push order after this pass's own. + // The events queued behind this init fail with it, in push order + // after this pass's own. + const behind = takeWaiting(); const failed = recordFailure( collector, destination, id, - [...allowedEvents, ...takeQueuedBehindInit(parked)].map( - (entry) => entry.event, - ), + [...allowedEvents, ...behind].map((entry) => entry.event), err, { message: 'destination init failed' }, ); - // A refused init (not-ready) never reached the transport. + // Entries this pass took from the queue may have a waiting push. + settle([...allowedEvents, ...behind], { + kind: 'failed', + reason: failed, + error: err, + }); + // An init that did not reach the transport releases the probe. if (!isTransportFailure(failed)) releaseProbeSlot(); return { id, destination, failed, error: err }; + } finally { + // From here to the take of the arrived entries below runs without + // a turn. + endInitCall(destination); } // The init flush held the destination: a state delivery did not @@ -936,24 +1054,14 @@ async function deliverToDestinations( // already. if (getStateHold(destination)) { releaseProbeSlot(); - const refused = requeueHeld(allowedEvents); - if (refused.length) { - const failed = refuseHeld(refused); - return { id, destination, failed }; - } - return { id, destination, queued: HELD_WAIT }; + return holdOrRefuse(allowedEvents); } // A consent command ran while init was awaited: the events admitted // before it pass the gate again with the consent that holds now. const deniedAfter: Destination.Queued[] = []; if (collector.consent !== admittedWith) { - const regated = admit( - allowedEvents.map((entry) => { - entry.event = { ...entry.event, consent: collector.consent }; - return entry; - }), - ); + const regated = admit(allowedEvents.map(refresh)); allowedEvents.splice(0, allowedEvents.length, ...regated.allowed); deniedAfter.push(...regated.denied); } @@ -967,12 +1075,7 @@ async function deliverToDestinations( destination.queuePush = queuedAfter.filter((queued) => parked.has(queued), ); - const late = admit( - arrived.map((entry) => { - entry.event = { ...entry.event, consent: collector.consent }; - return entry; - }), - ); + const late = admit(arrived.map(refresh)); allowedEvents.push(...late.allowed); deniedAfter.push(...late.denied); } @@ -985,13 +1088,9 @@ async function deliverToDestinations( : { id, destination, skipped: true }; } - // Process the destinations event queue - let error: unknown; - let failure: FailureReason | undefined; - // A push failed at the transport: the outcome sink recorded it for - // the breaker. + // Process the destinations event queue. A push failed at the + // transport: the outcome sink recorded it for the breaker. let pushRecorded = false; - let response: unknown; // The before and next routes; each resolves per event, hop by hop, // with `{ ingest, event }` (the runner resolves nothing ahead). @@ -1031,9 +1130,25 @@ async function deliverToDestinations( // Events the destination actually received. A before-chain fan-out // makes this exceed allowedEvents.length, which counts what entered. let pushedCount = 0; - await Promise.all( - allowedEvents.map(async (entry) => { + const outcomes = await Promise.all( + allowedEvents.map(async (entry): Promise => { const event = entry.event; + // What became of this event (its before chain may fan it out). + const tally: { + delivered: number; + batched: number; + response?: unknown; + failure?: { reason: FailureReason; error: unknown }; + } = { delivered: 0, batched: 0 }; + const settled = (): EntryOutcome => + tally.failure + ? { kind: 'failed', ...tally.failure } + : tally.delivered + ? { kind: 'delivered', response: tally.response } + : tally.batched + ? { kind: 'batched' } + : { kind: 'skipped' }; + // Merge collector user into event (collector as base, event overrides) event.user = assign(user, event.user); @@ -1050,7 +1165,7 @@ async function deliverToDestinations( collector, ); if (cacheResult?.status === 'HIT') { - return event; // Skip before chain + push + return settled(); // Skip before chain + push } if (cacheResult?.status === 'MISS') { cacheMiss = { @@ -1082,31 +1197,32 @@ async function deliverToDestinations( // destination did not get the event. The transformer's failure // is counted and logged already; the destination reports it. if (chainResult.failure) { - failure = chainResult.failure.reason; - error = `Transformer "${chainResult.failure.by}" failed: ${chainResult.failure.reason.message}`; + tally.failure = { + reason: chainResult.failure.reason, + error: `Transformer "${chainResult.failure.by}" failed: ${chainResult.failure.reason.message}`, + }; ensureDestStatus(collector, id).failed++; } if (chainResult.copies.length === 0) { - if (chainResult.failure) return event; + if (chainResult.failure) return settled(); // Dropped or stopped: THIS destination skips the event; other // destinations are unaffected. - const skipState = stepState( + emitDestinationState( collector, - stepId('destination', id), - 'destination', + id, 'skip', - typeof event.id === 'string' ? event.id : '', - Date.now(), - journeyFields(event, destIngest, collector), + event, + destIngest, + (state) => { + state.skipReason = 'dropped'; + state.meta = { + by: chainResult.droppedBy ?? 'route', + at: chainPath, + }; + }, ); - skipState.skipReason = 'dropped'; - skipState.meta = { - by: chainResult.droppedBy ?? 'route', - at: chainPath, - }; - emitStep(collector, skipState); - return event; + return settled(); } // Update respond if the chain produced a wrapped one @@ -1176,8 +1292,7 @@ async function deliverToDestinations( const result = await tryCatchAsync(destinationPush, (err) => { // The one outcome sink: counters, DLQ, breaker, one cause // line. destinationPush emitted the Observe error already. - error = err; - failure = recordFailure( + const reason = recordFailure( collector, destination, id, @@ -1185,7 +1300,8 @@ async function deliverToDestinations( err, { observed: true }, ); - if (isTransportFailure(failure)) pushRecorded = true; + tally.failure = { reason, error: err }; + if (isTransportFailure(reason)) pushRecorded = true; pushFailed = true; return undefined; })( @@ -1245,9 +1361,10 @@ async function deliverToDestinations( // Capture the last response (for single event pushes). // Batched-enqueue sentinel is NOT a real response; don't surface it. if (result !== undefined && !isBatchedResult(result)) { - response = result; + tally.response = result; } if (isBatchedResult(result)) { + tally.batched++; batchedCount++; return false; } @@ -1281,8 +1398,10 @@ async function deliverToDestinations( // full-check miss recorded before the chain is shared, since it // was computed from the one event that entered. Only a child // that actually reached the destination counts as delivered. - if (await deliverOne(child.event, child.ingest, cacheMiss)) + if (await deliverOne(child.event, child.ingest, cacheMiss)) { + tally.delivered++; pushedCount++; + } } // One write per request, after every child settled. @@ -1295,10 +1414,18 @@ async function deliverToDestinations( storeCache(dCacheStore, cacheMiss.key, cacheValue, cacheMiss.ttl); } - return event; + return settled(); }), ); + // Events queued behind this pass's init answer their own pushes; + // this pass reports its own events. + const own: EntryOutcome[] = []; + allowedEvents.forEach((entry, index) => { + if (waiting.has(entry)) settle([entry], outcomes[index]); + else own.push(outcomes[index]); + }); + // Settle a probe this pass may hold: a transport failure was // recorded by the outcome sink; otherwise a synchronous delivery // closes the breaker, and a pass that reached no transport (every @@ -1306,22 +1433,16 @@ async function deliverToDestinations( // by its mapping) releases it. A batched pass settles in its flush, // which records its own outcome. if (!pushRecorded) { - if (pushedCount) recordProbe('success'); + if (pushedCount) recordBreaker(collector, destination, id, 'success'); else if (!batchedCount) releaseProbeSlot(); } - return { - id, - destination, - ...(failure ? { failed: failure, error } : {}), - response, - totalDuration, - delivered: pushedCount, - batched: batchedCount, - }; - }, - ), - ); + return passOutcome(id, destination, own, totalDuration); + }), + ); + } finally { + for (const [, destination] of passes) settleWaiting(destination); + } // Build result objects const done: Record = {}; @@ -1438,17 +1559,7 @@ async function runDestinationInit( destLogger.debug('init'); const initStarted = Date.now(); - emitStep( - collector, - stepState( - collector, - stepId('destination', destId), - 'destination', - 'init', - '', - initStarted, - ), - ); + emitDestinationState(collector, destId, 'init', undefined, undefined); let configResult; try { @@ -1459,29 +1570,24 @@ async function runDestinationInit( collector.logger, )(context); } catch (err) { - const initErrFinished = Date.now(); - const errState = stepState( + emitDestinationState( collector, - stepId('destination', destId), - 'destination', + destId, 'error', - '', - initErrFinished, + undefined, + undefined, + (state) => { + state.durationMs = Date.now() - initStarted; + state.error = observerStepError(err); + }, ); - errState.durationMs = initErrFinished - initStarted; - errState.error = observerStepError(err); - emitStep(collector, errState); throw err; } // Runtime guard: bundled code is not typechecked, so an init may still // return false. Without this, `{ ...(false || config), init: true }` // would install a destination with its settings missing. - if (configResult === false) - throw new StepError( - 'init returned false: throw a StepError that names the missing setting', - { code: 'not-ready' }, - ); + if (configResult === false) throw initReturnedFalse(); // The config to install, marked initialized; the caller installs it. return { @@ -1646,35 +1752,20 @@ export async function destinationPush( try { processed = await processEventMapping(event, config, collector); } catch (err) { - const errState = stepState( - collector, - stepId('destination', destId), - 'destination', - 'error', - typeof event.id === 'string' ? event.id : '', - Date.now(), - journeyFields(event, ingest, collector), - ); - errState.error = observerStepError(err); - emitStep(collector, errState); + emitDestinationState(collector, destId, 'error', event, ingest, (state) => { + state.error = observerStepError(err); + }); throw err; } // An ignored event is a deliberate skip: counted in `skipped`, never `done`. if (processed.ignore) { recordSkip(collector, destination, destId, event, 'ignored'); - const skipState = stepState( - collector, - stepId('destination', destId), - 'destination', - 'skip', - typeof event.id === 'string' ? event.id : '', - Date.now(), - journeyFields(event, ingest, collector), - ); - skipState.skipReason = 'ignored'; - if (processed.mappingKey) skipState.mappingKey = processed.mappingKey; - emitStep(collector, skipState); + const { mappingKey } = processed; + emitDestinationState(collector, destId, 'skip', event, ingest, (state) => { + state.skipReason = 'ignored'; + if (mappingKey) state.mappingKey = mappingKey; + }); return IGNORED_RESULT; } @@ -1796,28 +1887,6 @@ export async function destinationPush( const destIdResolved = destination.config.id || destId; const destStatus = ensureDestStatus(collector, destIdResolved); - // Circuit-breaker accounting for the batch path. Keys on the canonical - // stepId so it matches the gate. Presence-gated. A whole-batch throw is - // a transport failure; any delivered rows (full or partial success) are - // a success. Partial-failure rows themselves are breaker-neutral. - const flushBreakerConfig = resolveBreakerConfig( - destination.config.breaker, - ); - const flushBreakerKey = stepId('destination', destIdResolved); - const recordFlushBreaker = ( - outcome: 'transport-failure' | 'success', - ) => { - if (flushBreakerConfig) { - recordStepOutcome( - collector.status.breakers, - flushBreakerKey, - outcome, - flushBreakerConfig.threshold, - flushBreakerConfig.cooldown, - ); - } - }; - const flushStarted = Date.now(); // Rows the destination named as failed, index -> that row's error. A @@ -1888,10 +1957,7 @@ export async function destinationPush( ); failedRows.set(row.index, rowError); } - const causes = new Map< - string, - { code: string; count: number; message: string } - >(); + const causes = new Map(); for (const [index, rowError] of failedRows) { const reason = recordFailure( collector, @@ -1912,19 +1978,36 @@ export async function destinationPush( : 'Error'; const cause = causes.get(code); if (cause) cause.count++; - else causes.set(code, { code, count: 1, message: reason.message }); + else causes.set(code, { code, count: 1 }); } succeededCount = Math.max( 0, snapshot.entries.length - failedRows.size, ); - if (failedRows.size > 0) - destLogger.error('Push batch partial failure', { - failed: failedRows.size, - delivered: succeededCount, - entries: snapshot.entries.length, - causes: [...causes.values()], - }); + // One cause line, keyed on the first row's reason; the message + // counts the rows and names each cause (the error's `code`, else + // its `name`) with its count. + if (failure) + reportCause( + collector, + collector.logger, + { + step: stepId('destination', destIdResolved), + code: failure.code, + status: failure.status, + }, + `Push batch partial failure: ${failedRows.size} of ${snapshot.entries.length} entries failed (${[ + ...causes.values(), + ] + .map((cause) => `${cause.code} x${cause.count}`) + .join(', ')})`, + { + type: destination.type, + count: failedRows.size, + retryable: failure.retryable, + error: failure.message, + }, + ); } // Batch outcomes fan out one frame per entry, emitted once the flush @@ -1976,9 +2059,10 @@ export async function destinationPush( destStatus.lastAt = Date.now(); collector.status.out += succeededCount; // Any delivered rows mean the transport worked: success closes the - // breaker. Partial-failure rows are breaker-neutral (recorded with + // breaker (a whole-batch throw was recorded by the outcome sink). + // Partial-failure rows are breaker-neutral (recorded with // `breaker: false` above). - recordFlushBreaker('success'); + recordBreaker(collector, destination, destId, 'success'); } return { @@ -2173,23 +2257,22 @@ export async function destinationPush( return response; } catch (err) { - const pushFinished = Date.now(); - const errState = stepState( + const { mappingKey } = processed; + emitDestinationState( collector, - stepId('destination', destId), - 'destination', + destId, 'error', - eventIdValue, - pushFinished, - journey, + processed.event, + ingest, + (state) => { + state.durationMs = Date.now() - pushStarted; + state.error = observerStepError(err); + // Surface the vendor calls recorded before the failure, so a + // mid-push error doesn't discard them. + attachCalls(state, recordedCalls); + if (mappingKey) state.mappingKey = mappingKey; + }, ); - errState.durationMs = pushFinished - pushStarted; - errState.error = observerStepError(err); - // Surface the vendor calls recorded before the failure, so a mid-push - // error doesn't discard them. - attachCalls(errState, recordedCalls); - if (processed.mappingKey) errState.mappingKey = processed.mappingKey; - emitStep(collector, errState); throw err; } } @@ -2237,10 +2320,10 @@ export function registerDestination( /** * Registration checks of a destination, logged once at ERROR (a config value * without effect): a `batch` without `pushBatch` (events would go to `push` - * one by one), and mapping keys that do nothing at the destination position - * (behind the `__WALKEROS_VALIDATE__` build flag: CLI bundles check at build - * time). The destination still runs. With `chains`, the transformers its - * `before` and `next` name must exist (checked once transformers exist). + * one by one), and, behind the `__WALKEROS_VALIDATE__` build flag (CLI + * bundles check at build time), mapping keys that do nothing at the + * destination position and, with `chains`, the transformers its `before` and + * `next` name (checked once transformers exist). The destination still runs. */ export function checkDestinationRegistration( collector: Collector.Instance, @@ -2277,11 +2360,10 @@ export function checkDestinationRegistration( logger.error( `Destination ${id}: ${getMappingNoOpMessage('destination', paths)}`, ); - } - - if (chains) { - checkChainTargets(collector, `destination.${id}.before`, config.before); - checkChainTargets(collector, `destination.${id}.next`, config.next); + if (chains) { + checkChainTargets(collector, `destination.${id}.before`, config.before); + checkChainTargets(collector, `destination.${id}.next`, config.next); + } } } diff --git a/packages/collector/src/handle.ts b/packages/collector/src/handle.ts index 8ced52a92..db181b8f4 100644 --- a/packages/collector/src/handle.ts +++ b/packages/collector/src/handle.ts @@ -10,6 +10,7 @@ import type { import { Const } from './constants'; import { addDestination, + clearQueuePush, markHeldForDrain, mergePushResults, pushToDestinations, @@ -500,9 +501,7 @@ export async function runCollectorDeferred( } // Reset destination queues - Object.values(collector.destinations).forEach((destination) => { - destination.queuePush = []; - }); + Object.values(collector.destinations).forEach(clearQueuePush); // Reset collector queue for this run collector.queue = []; diff --git a/packages/collector/src/observerEmit.ts b/packages/collector/src/observerEmit.ts index 956ce1dcf..a06f39937 100644 --- a/packages/collector/src/observerEmit.ts +++ b/packages/collector/src/observerEmit.ts @@ -1,5 +1,5 @@ import type { Collector, FlowState, Ingest, WalkerOS } from '@walkeros/core'; -import { emitStep, StepError, toFailureReason } from '@walkeros/core'; +import { emitStep, StepError, stepId, toFailureReason } from '@walkeros/core'; /** Journey-correlation trio shared by every FlowState-stamping site. */ export interface JourneyFields { @@ -129,6 +129,32 @@ export function emit( emitStep(collector, buildBaseState(collector, args)); } +/** + * Emit one destination step state: the base fields, the event's id and + * journey trio (none for a state of the destination itself, such as its + * init), and whatever `fill` sets before observers see it. + */ +export function emitDestinationState( + collector: Collector.Instance, + destId: string, + phase: FlowState['phase'], + event: WalkerOS.Event | undefined, + ingest: Ingest | undefined, + fill?: (state: FlowState) => void, +): void { + const state = stepState( + collector, + stepId('destination', destId), + 'destination', + phase, + event && typeof event.id === 'string' ? event.id : '', + Date.now(), + event ? journeyFields(event, ingest, collector) : undefined, + ); + fill?.(state); + emitStep(collector, state); +} + /** * Why a collector-owned position dropped an event. * - `dropped`: a chain hop or a route `stop` (`source.before`, `source.next`, diff --git a/packages/collector/src/outcome.ts b/packages/collector/src/outcome.ts index 02900bbe4..5e6b18fa8 100644 --- a/packages/collector/src/outcome.ts +++ b/packages/collector/src/outcome.ts @@ -5,16 +5,15 @@ import type { WalkerOS, } from '@walkeros/core'; import { - emitStep, reportCause, StepError, stepId, toFailureReason, } from '@walkeros/core'; -import { journeyFields, observerStepError, stepState } from './observerEmit'; +import { emitDestinationState, observerStepError } from './observerEmit'; import { pushBounded, resetOverflowFlag } from './buffers'; import { DEFAULT_DLQ_MAX, bumpDropped, ensureDestStatus } from './report-error'; -import { recordStepOutcome, resolveBreakerConfig } from './breaker'; +import { recordBreaker } from './breaker'; /** * Codes the collector assigns at its own gates: the event never reached the @@ -29,9 +28,22 @@ const GATE_CODES = [ 'mapping', ]; -/** Whether a failure reached the transport (and counts for the breaker). */ +/** + * Whether a failure counts for the breaker: a retryable failure at the + * transport. A deterministic one (`config`, `missing-input`, a 4xx) never + * opens it, or a misconfigured destination would turn into `breaker-open` + * (retryable) and be redelivered for as long as the flow runs. + */ export function isTransportFailure(reason: FailureReason): boolean { - return !GATE_CODES.includes(reason.code); + return reason.retryable && !GATE_CODES.includes(reason.code); +} + +/** The error of an `init` that returned `false` (bundled code is not typechecked). */ +export function initReturnedFalse(): StepError { + return new StepError( + 'init returned false: throw a StepError that names the missing setting', + { code: 'not-ready' }, + ); } /** @@ -108,31 +120,22 @@ export function recordFailure( destStatus.dlqSize = dlq.length; } - // The breaker counts transport failures only. - const breakerConfig = resolveBreakerConfig(destination.config.breaker); - if (breakerConfig && options.breaker !== false && isTransportFailure(reason)) - recordStepOutcome( - collector.status.breakers, - stepId('destination', destination.config.id || id), - 'transport-failure', - breakerConfig.threshold, - breakerConfig.cooldown, - ); + // The breaker counts retryable transport failures only. + if (options.breaker !== false && isTransportFailure(reason)) + recordBreaker(collector, destination, id, 'transport-failure'); if (!options.observed) { - for (const event of events) { - const state = stepState( + for (const event of events) + emitDestinationState( collector, - step, - 'destination', + id, 'error', - typeof event.id === 'string' ? event.id : '', - now, - journeyFields(event, undefined, collector), + event, + undefined, + (state) => { + state.error = observerStepError(error); + }, ); - state.error = observerStepError(error); - emitStep(collector, state); - } } if (options.report === false) return reason; diff --git a/packages/collector/src/shutdown.ts b/packages/collector/src/shutdown.ts index c20ce5a25..13c188477 100644 --- a/packages/collector/src/shutdown.ts +++ b/packages/collector/src/shutdown.ts @@ -5,6 +5,12 @@ import type { Lifecycle, Logger, } from '@walkeros/core'; +import { + reportCause, + StepError, + stepId, + toFailureReason, +} from '@walkeros/core'; import { createPushResult } from './destination'; const STEP_TIMEOUT = 5000; @@ -67,20 +73,36 @@ export async function flushDestinationBatches( timer = setTimeout( () => reject( - new Error(`destination '${id}' batch flush timed out`), + new StepError( + `destination '${id}' batch flush timed out`, + { code: 'timeout' }, + ), ), STEP_TIMEOUT, ); }), ]); } catch (err) { - // The flush did not settle in time (it may still land later): - // named in the result, one ERROR for this destination. - const message = err instanceof Error ? err.message : String(err); - ref.reason = { code: 'timeout', message, retryable: true }; - collector.logger - .scope(dest.type || 'destination') - .error(`destination '${id}' batch flush failed: ${message}`); + // The flush did not settle in time (`timeout`; it may still land + // later) or rejected (its own reason): named in the result, one + // cause line per cause. + const reason = toFailureReason(err); + ref.reason = reason; + reportCause( + collector, + collector.logger, + { + step: stepId('destination', id), + code: reason.code, + status: reason.status, + }, + 'batch flush failed', + { + type: dest.type, + retryable: reason.retryable, + error: reason.message, + }, + ); return undefined; } finally { if (timer) clearTimeout(timer); diff --git a/packages/collector/src/source.ts b/packages/collector/src/source.ts index 5db14db6f..6a73856c5 100644 --- a/packages/collector/src/source.ts +++ b/packages/collector/src/source.ts @@ -140,18 +140,17 @@ export async function initSource( } = sourceDefinition; // Registration checks, once at ERROR (the source still runs): mapping - // keys that do nothing at a source (behind the `__WALKEROS_VALIDATE__` - // build flag; CLI bundles check at build time), and the transformers its - // chains name. + // keys that do nothing at a source, and the transformers its chains name. + // Build flag: CLI bundles check both at build time and fold this out. if (typeof __WALKEROS_VALIDATE__ === 'undefined' || __WALKEROS_VALIDATE__) { const noOps = getMappingNoOpPaths('source', config, 'config'); if (noOps.length) collector.logger .scope('source') .error(`Source ${sourceId}: ${getMappingNoOpMessage('source', noOps)}`); + checkChainTargets(collector, `source.${sourceId}.before`, before); + checkChainTargets(collector, `source.${sourceId}.next`, next); } - checkChainTargets(collector, `source.${sourceId}.before`, before); - checkChainTargets(collector, `source.${sourceId}.next`, next); // Compile declarative state entries once. For a source, all state entries // run before handing the event to `collector.push` (get enriches the event diff --git a/packages/collector/src/transformer.ts b/packages/collector/src/transformer.ts index cf478e86a..62db45126 100644 --- a/packages/collector/src/transformer.ts +++ b/packages/collector/src/transformer.ts @@ -61,14 +61,13 @@ import { getTransformerMappingWarnings, getRouteGraph, reportCause, - StepError, toFailureReason, } from '@walkeros/core'; import { journeyFields, observerStepError, stepState } from './observerEmit'; import { getCacheStore } from './cache'; import { buildReportError, errorMeta } from './report-error'; import { compileStepState, runStepState, warnStateOff } from './state'; -import { asStepError } from './outcome'; +import { asStepError, initReturnedFalse } from './outcome'; /** * Extracts chain property from definition and merges into config. @@ -853,12 +852,7 @@ async function runMember( )(collector, transformer, transformerName); if (!isInitialized) { - const err = - initError ?? - new StepError( - 'init returned false: throw a StepError that names the missing setting', - { code: 'not-ready' }, - ); + const err = initError ?? initReturnedFalse(); return halt({ copies: [], respond: currentRespond, diff --git a/packages/core/src/types/telemetry.ts b/packages/core/src/types/telemetry.ts index 580e2691d..d6fcb9cb7 100644 --- a/packages/core/src/types/telemetry.ts +++ b/packages/core/src/types/telemetry.ts @@ -98,7 +98,6 @@ export interface FlowState { | 'sampled_out' | 'disabled' | 'dropped' - | 'held' | 'ignored' | 'unknown'; /** Free-form metadata: store key/value, cached: true, etc. */ From 4924364d5dd3c3a2a2d1463441b36dfdfed2daf3 Mon Sep 17 00:00:00 2001 From: alexanderkirtzel Date: Wed, 21 Oct 2026 03:52:25 +0000 Subject: [PATCH 06/41] design time --- packages/cli/examples/flow-complete.json | 16 +- .../f27-only-warning.json | 25 + .../integration/bundle/bundler.test.ts | 26 ++ .../integration/bundle/cdn-jsdom.test.ts | 4 +- .../integration/bundle/programmatic.test.ts | 5 +- .../integration/bundle/size-budget.test.ts | 9 +- .../unit/bundle/bundler-codegen.test.ts | 36 ++ .../__tests__/unit/bundle/inline-code.test.ts | 58 +++ .../unit/bundle/server-entry-logger.test.ts | 184 ++++++++ .../unit/commands/setup-init.test.ts | 105 +++++ .../unit/config/config-loader.test.ts | 40 ++ .../unit/validate/adversarial.test.ts | 52 ++- .../__tests__/unit/validate/contract.test.ts | 69 ++- .../src/__tests__/unit/validate/flow.test.ts | 429 +++++++++++++++++- .../commands/bundle/__tests__/bundler.test.ts | 9 +- packages/cli/src/commands/bundle/bundler.ts | 84 +++- .../__tests__/simulate-result-shape.test.ts | 8 +- .../setup/__tests__/lifecycle.test.ts | 8 +- packages/cli/src/commands/setup/index.ts | 54 ++- packages/cli/src/commands/validate/types.ts | 8 +- .../commands/validate/validators/contract.ts | 40 +- .../src/commands/validate/validators/flow.ts | 140 ++++-- .../cli/src/config/__tests__/loader.test.ts | 41 +- packages/cli/src/config/loader.ts | 19 + .../src/examples/flow-complete.manifest.ts | 6 + packages/cli/src/telemetry/flow.json | 74 +-- .../src/__tests__/registration-checks.test.ts | 4 +- .../src/__tests__/report-error.test.ts | 12 +- packages/collector/src/destination.ts | 19 +- packages/collector/src/report-error.ts | 5 +- packages/core/src/__tests__/flow.test.ts | 26 ++ .../core/src/__tests__/mapping-no-ops.test.ts | 2 +- packages/core/src/__tests__/node.test.ts | 163 ++++++- .../src/__tests__/schemas/contract.test.ts | 32 +- .../schemas/validate-flow-config.test.ts | 42 +- .../src/__tests__/validate-structure.test.ts | 334 +++++++++++++- packages/core/src/contract.ts | 16 + packages/core/src/dev.ts | 2 +- packages/core/src/flow.ts | 7 + packages/core/src/mapping-no-ops.ts | 24 +- packages/core/src/mappingValue.ts | 19 +- packages/core/src/node.ts | 162 ++++++- .../schemas/__tests__/config-drift.test-d.ts | 28 ++ packages/core/src/schemas/flow.ts | 29 +- packages/core/src/schemas/logger.ts | 8 +- .../core/src/schemas/validate-flow-config.ts | 55 ++- packages/core/src/schemas/validate.ts | 2 + packages/core/src/step-entry.ts | 30 +- packages/core/src/types/flow.ts | 14 +- packages/core/src/types/logger.ts | 15 + packages/core/src/validate-structure.ts | 282 +++++++++++- .../src/__tests__/resources/packages.test.ts | 90 ++++ .../resources/references-catalog.test.ts | 4 +- .../src/__tests__/tools/flow-schema.test.ts | 7 +- .../src/__tests__/tools/validate-real.test.ts | 4 +- packages/mcps/mcp/src/instructions.ts | 2 +- packages/mcps/mcp/src/resources/references.ts | 12 +- packages/runner/Dockerfile | 3 +- .../runner/src/__tests__/heartbeat.test.ts | 24 + .../runner/src/__tests__/pipeline.test.ts | 45 ++ .../runner/src/__tests__/preview-gate.test.ts | 9 + .../src/__tests__/runtime-logging.test.ts | 162 +++++++ packages/runner/src/health-server.ts | 3 +- packages/runner/src/heartbeat.ts | 6 + packages/runner/src/index.ts | 1 + packages/runner/src/pipeline.ts | 62 ++- packages/runner/src/run.ts | 32 +- packages/runner/src/runner.ts | 8 +- packages/runner/src/runtime-logging.ts | 68 +++ .../src/sqs/__mocks__/@aws-sdk/client-sqs.ts | 12 + .../aws/src/sqs/__tests__/index.test.ts | 20 + .../aws/src/sqs/__tests__/sqs-mock.d.ts | 1 + packages/server/sources/aws/src/sqs/index.ts | 4 +- .../src/pubsub/pull/__tests__/index.test.ts | 15 + .../sources/gcp/src/pubsub/pull/index.ts | 4 +- 75 files changed, 3085 insertions(+), 394 deletions(-) create mode 100644 packages/cli/src/__tests__/fixtures/validate-adversarial/f27-only-warning.json create mode 100644 packages/cli/src/__tests__/unit/bundle/server-entry-logger.test.ts create mode 100644 packages/cli/src/__tests__/unit/commands/setup-init.test.ts create mode 100644 packages/mcps/mcp/src/__tests__/resources/packages.test.ts create mode 100644 packages/runner/src/__tests__/runtime-logging.test.ts create mode 100644 packages/runner/src/runtime-logging.ts diff --git a/packages/cli/examples/flow-complete.json b/packages/cli/examples/flow-complete.json index f22464e11..d082aad2a 100644 --- a/packages/cli/examples/flow-complete.json +++ b/packages/cli/examples/flow-complete.json @@ -515,15 +515,13 @@ "dataLayer": { "add_to_cart": { "name": "product add", - "data": { - "map": { - "id": "data.items.0.item_id", - "name": "data.items.0.item_name", - "price": "data.items.0.price", - "quantity": { - "key": "data.items.0.quantity", - "value": 1 - } + "policy": { + "data.id": "data.items.0.item_id", + "data.name": "data.items.0.item_name", + "data.price": "data.items.0.price", + "data.quantity": { + "key": "data.items.0.quantity", + "value": 1 } } } diff --git a/packages/cli/src/__tests__/fixtures/validate-adversarial/f27-only-warning.json b/packages/cli/src/__tests__/fixtures/validate-adversarial/f27-only-warning.json new file mode 100644 index 000000000..271fcc14b --- /dev/null +++ b/packages/cli/src/__tests__/fixtures/validate-adversarial/f27-only-warning.json @@ -0,0 +1,25 @@ +{ + "version": 4, + "flows": { + "a": { + "config": { + "platform": "server" + }, + "destinations": { + "d": { + "code": { + "type": "d", + "push": "$code:(event) => {}" + }, + "config": { + "mapping": { + "page.view": { + "name": "pv" + } + } + } + } + } + } + } +} diff --git a/packages/cli/src/__tests__/integration/bundle/bundler.test.ts b/packages/cli/src/__tests__/integration/bundle/bundler.test.ts index 84f1ae1ba..c3066d77d 100644 --- a/packages/cli/src/__tests__/integration/bundle/bundler.test.ts +++ b/packages/cli/src/__tests__/integration/bundle/bundler.test.ts @@ -145,6 +145,32 @@ describe('Bundler', () => { ).resolves.not.toThrow(); }); + it('bakes consentDenied skip into a server flow without provenance (the push and simulate call shape)', async () => { + const serverFlow: Flow = { + config: { platform: 'server' }, + destinations: { + sink: { + code: { type: 'sink', push: '$code:() => {}' }, + config: {}, + }, + }, + }; + const output = path.join(testOutputDir, 'consent-skip', 'flow.mjs'); + const buildOptions = createBuildOptions({ + packages: {}, + platform: 'node', + format: 'esm', + output, + }); + + await bundle(serverFlow, buildOptions, logger, false); + + expect(serverFlow.collector?.consentDenied).toBe('skip'); + jest.restoreAllMocks(); + const code = await fs.readFile(output, 'utf-8'); + expect(code).toMatch(/"consentDenied":\s*"skip"/); + }); + describe('Stats Collection', () => { it('should collect bundle stats when requested', async () => { const flowSettings: Flow = { diff --git a/packages/cli/src/__tests__/integration/bundle/cdn-jsdom.test.ts b/packages/cli/src/__tests__/integration/bundle/cdn-jsdom.test.ts index a1707c2c9..86557f40f 100644 --- a/packages/cli/src/__tests__/integration/bundle/cdn-jsdom.test.ts +++ b/packages/cli/src/__tests__/integration/bundle/cdn-jsdom.test.ts @@ -146,15 +146,15 @@ describe('CDN bundle — jsdom smoke', () => { const fetchCalls: FetchCall[] = []; const windowAny = dom.window as unknown as { fetch: (url: unknown, init?: { body?: unknown }) => unknown; - Response: new (body?: unknown, init?: unknown) => unknown; setTimeout: typeof setTimeout; }; + // jsdom has no `Response`: answer with Node's, as a browser would. windowAny.fetch = (async (url: unknown, init?: { body?: unknown }) => { fetchCalls.push({ url: String(url), body: init?.body != null ? String(init.body) : undefined, }); - return new windowAny.Response('{}', { status: 200 }); + return new Response('{}', { status: 200 }); }) as unknown as typeof windowAny.fetch; dom.window.eval(script); diff --git a/packages/cli/src/__tests__/integration/bundle/programmatic.test.ts b/packages/cli/src/__tests__/integration/bundle/programmatic.test.ts index c4803cb9e..0dde250f2 100644 --- a/packages/cli/src/__tests__/integration/bundle/programmatic.test.ts +++ b/packages/cli/src/__tests__/integration/bundle/programmatic.test.ts @@ -188,7 +188,10 @@ describe('Programmatic Bundle API', () => { }, }, destinations: { - api: { config: { settings: { url: '$flow.server.url' } } }, + api: { + package: '@walkeros/web-destination-api', + config: { settings: { url: '$flow.server.url' } }, + }, }, }, }, diff --git a/packages/cli/src/__tests__/integration/bundle/size-budget.test.ts b/packages/cli/src/__tests__/integration/bundle/size-budget.test.ts index 73794bb0a..2de423d7a 100644 --- a/packages/cli/src/__tests__/integration/bundle/size-budget.test.ts +++ b/packages/cli/src/__tests__/integration/bundle/size-budget.test.ts @@ -28,10 +28,11 @@ import { withLocalPackages } from '../../helpers/local-packages.js'; // Budget rule, raw and gzip: the measured size × 1.05, rounded up to the next // 1,000 B. Bump only in a PR that names the feature that grew the bundle. -// Measured 2026-10-02 on the 4.7.0 working tree (minimal flow, every build -// flag off): 93,207 B raw, 32,366 B gzip (level 9). -const RAW_BUDGET_BYTES = 98_000; -const GZIP_BUDGET_BYTES = 34_000; +// Measured 2026-10-21 on the 4.8.1 working tree (minimal flow, every build +// flag off): 108,291 B raw, 38,045 B gzip (level 9), grown by the +// silent-failures failure contract and transport reporting. +const RAW_BUDGET_BYTES = 114_000; +const GZIP_BUDGET_BYTES = 40_000; /** Strings only the observe recorder (connect, poster, call capture) carries. */ const RECORDER_MARKERS = ['elbObserve', '/ingest/', 'X-Walkeros-Binding']; diff --git a/packages/cli/src/__tests__/unit/bundle/bundler-codegen.test.ts b/packages/cli/src/__tests__/unit/bundle/bundler-codegen.test.ts index a696a66d5..d62016c67 100644 --- a/packages/cli/src/__tests__/unit/bundle/bundler-codegen.test.ts +++ b/packages/cli/src/__tests__/unit/bundle/bundler-codegen.test.ts @@ -10,6 +10,7 @@ jest.mock('../../../commands/bundle/bundler.js', () => { }); import { + applyCollectorDefaults, applyCollectorProvenance, buildDataPayload, buildSplitConfigObject, @@ -1596,6 +1597,41 @@ describe('data payload byte stability', () => { }); }); +describe('applyCollectorDefaults', () => { + const flowOn = ( + platform: 'web' | 'server', + collector?: Flow['collector'], + ) => { + const flowSettings: Flow = { + config: { platform }, + sources: {}, + destinations: {}, + }; + if (collector) flowSettings.collector = collector; + return flowSettings; + }; + + it('skips consent-denied events in a server flow', () => { + const flowSettings = flowOn('server'); + applyCollectorDefaults(flowSettings); + expect(buildDataPayload(flowSettings)).toMatchObject({ + collector: { consentDenied: 'skip' }, + }); + }); + + it('keeps an authored consentDenied', () => { + const flowSettings = flowOn('server', { consentDenied: 'queue' }); + applyCollectorDefaults(flowSettings); + expect(flowSettings.collector?.consentDenied).toBe('queue'); + }); + + it('leaves a web flow on the collector default (queue)', () => { + const flowSettings = flowOn('web'); + applyCollectorDefaults(flowSettings); + expect(flowSettings.collector).toBeUndefined(); + }); +}); + describe('applyCollectorProvenance', () => { const fallback = () => 'digest'; diff --git a/packages/cli/src/__tests__/unit/bundle/inline-code.test.ts b/packages/cli/src/__tests__/unit/bundle/inline-code.test.ts index 779ead0ea..4736224d5 100644 --- a/packages/cli/src/__tests__/unit/bundle/inline-code.test.ts +++ b/packages/cli/src/__tests__/unit/bundle/inline-code.test.ts @@ -101,6 +101,64 @@ describe('Inline Code Bundling', () => { }); }); + describe('Every step function survives bundling (E2)', () => { + it('emits on, pushBatch and destroy of inline destination code', () => { + const flowSettings: Flow = { + config: { platform: 'server' }, + destinations: { + console: { + code: { + type: 'console', + init: '$code:() => {}', + on: '$code:(type, context) => { console.log(type); }', + pushBatch: + '$code:(batch) => { console.log(batch.entries.length); }', + destroy: '$code:() => { console.log("bye"); }', + push: '$code:(event) => { console.log(event.name); }', + }, + config: { batch: 10 }, + }, + }, + }; + + const { codeConfigObject: result } = buildSplitConfigObject( + flowSettings, + new Map>(), + ); + + expect(result).toContain('init: () => {}'); + expect(result).toContain('on: (type, context) => { console.log(type); }'); + expect(result).toContain( + 'pushBatch: (batch) => { console.log(batch.entries.length); }', + ); + expect(result).toContain('destroy: () => { console.log("bye"); }'); + expect(result).toContain('push: (event) => { console.log(event.name); }'); + expect(result).not.toContain('$code:'); + }); + + it('emits on of inline source code inside its factory', () => { + const flowSettings: Flow = { + config: { platform: 'server' }, + sources: { + custom: { + code: { + push: '$code:(event) => event', + on: '$code:(type) => { console.log(type); }', + }, + }, + }, + }; + + const { codeConfigObject: result } = buildSplitConfigObject( + flowSettings, + new Map>(), + ); + + expect(result).toContain('code: async (context) => ({'); + expect(result).toContain('on: (type) => { console.log(type); }'); + }); + }); + describe('Source with code object', () => { it('should generate inline source from code object', () => { const flowSettings: Flow = { diff --git a/packages/cli/src/__tests__/unit/bundle/server-entry-logger.test.ts b/packages/cli/src/__tests__/unit/bundle/server-entry-logger.test.ts new file mode 100644 index 000000000..329a50d63 --- /dev/null +++ b/packages/cli/src/__tests__/unit/bundle/server-entry-logger.test.ts @@ -0,0 +1,184 @@ +import fs from 'fs'; +import os from 'os'; +import path from 'path'; +import { Level, type Logger } from '@walkeros/core'; +import { createCLILoggerConfig, type LogEntry } from '@walkeros/core/node'; +import { generateServerEntry } from '../../../commands/bundle/bundler.js'; + +/** + * The runner contract's logger merge (E17), run through the generated + * server entry: a stand-in stage 1 builds the collector's logger from + * `config.logger` with core's `createLogger`, as the collector does, and + * logs one line per level. The host is the runner's terminal logger config. + */ +const STAGE1 = `import { createLogger } from '@walkeros/core'; +export const calls = []; +export function wireConfig(data) { + const config = { ...data.collector }; + if (data.withHandler) + config.logger = { + ...config.logger, + handler: (level, message) => calls.push(['flow', message]), + }; + return config; +} +export async function startFlow(config) { + calls.push(['startFlow']); + const logger = createLogger(config.logger).scope('dest'); + logger.debug('debug line'); + logger.info('info line'); + logger.warn('warn line'); + logger.error('Push failed', { code: 'transport', status: 503, retryable: true }); + return { collector: { push: async () => ({}), command: async () => {} } }; +} +`; + +describe('generateServerEntry merges the flow logger with the host logger', () => { + const dirs: string[] = []; + let errorSpy: jest.SpyInstance; + let logSpy: jest.SpyInstance; + + beforeEach(() => { + errorSpy = jest.spyOn(console, 'error').mockImplementation(() => {}); + logSpy = jest.spyOn(console, 'log').mockImplementation(() => {}); + }); + + afterEach(() => { + errorSpy.mockRestore(); + logSpy.mockRestore(); + for (const dir of dirs) fs.rmSync(dir, { recursive: true, force: true }); + dirs.length = 0; + }); + + async function run( + data: Record, + context: ( + format: { current: Logger.Format }, + calls: unknown[], + ) => Record, + ): Promise<{ ring: Array<[Level, string, LogEntry]>; calls: unknown[] }> { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'server-entry-logger-')); + dirs.push(dir); + const stage1 = path.join(dir, 'stage1.mjs'); + fs.writeFileSync(stage1, STAGE1); + const entry = path.join(dir, 'entry.mjs'); + fs.writeFileSync(entry, generateServerEntry(stage1, JSON.stringify(data))); + + const stage: unknown = await import(stage1); + const module: unknown = await import(entry); + if ( + typeof stage !== 'object' || + stage === null || + !('calls' in stage) || + !Array.isArray(stage.calls) || + typeof module !== 'object' || + module === null || + !('default' in module) || + typeof module.default !== 'function' + ) + throw new Error('the entry has no default factory'); + const calls: unknown[] = stage.calls; + + const ring: Array<[Level, string, LogEntry]> = []; + const format: { current: Logger.Format } = { current: 'text' }; + const extra = context(format, calls); + await module.default({ + ...extra, + logger: createCLILoggerConfig({ + verbose: extra.verbose === true, + format: () => format.current, + onLine: (level, message, line) => { + ring.push([level, message, line]); + calls.push(['host', message]); + }, + }), + }); + return { ring, calls }; + } + + const parse = (value: unknown): unknown => JSON.parse(String(value)); + + it('applies the flow level and format: WARN as JSON lines, INFO suppressed, the ring still fed', async () => { + const settings: unknown[] = []; + const { ring } = await run( + { collector: { logger: { level: 'WARN', format: 'json' } } }, + (format) => ({ + onLogger: (value: { format?: Logger.Format }) => { + settings.push(value); + if (value.format) format.current = value.format; + }, + }), + ); + + expect(settings).toEqual([{ level: 'WARN', format: 'json' }]); + expect(logSpy).not.toHaveBeenCalled(); + expect(errorSpy.mock.calls.map((call) => parse(call[0]))).toEqual([ + expect.objectContaining({ + severity: 'WARNING', + message: '[dest] warn line', + }), + expect.objectContaining({ + severity: 'ERROR', + message: '[dest] Push failed', + context: { code: 'transport', status: 503, retryable: true }, + }), + ]); + expect(ring.map(([level]) => level)).toEqual([Level.WARN, Level.ERROR]); + expect(ring[1][1]).toBe( + '[dest] Push failed {"code":"transport","status":503,"retryable":true}', + ); + }); + + it('tells the host the flow settings before the collector starts', async () => { + const { calls } = await run( + { collector: { logger: { format: 'json' } } }, + (_format, calls) => ({ + onLogger: (value: unknown) => calls.push(['onLogger', value]), + }), + ); + expect(calls.slice(0, 2)).toEqual([ + ['onLogger', { level: undefined, format: 'json' }], + ['startFlow'], + ]); + }); + + it('a level the host forces (--verbose) wins over the flow level', async () => { + const { ring } = await run( + { collector: { logger: { level: 'ERROR' } } }, + () => ({ logLevel: 'DEBUG', verbose: true }), + ); + expect(ring.map(([level]) => level)).toEqual([ + Level.DEBUG, + Level.INFO, + Level.WARN, + Level.ERROR, + ]); + }); + + it('keeps the host default when the flow sets no logger', async () => { + const { ring } = await run({}, () => ({})); + // The host passes every level to its handler, which hides DEBUG unless + // verbose: the same lines reach the terminal as before the merge. + expect(ring).toHaveLength(4); + expect(logSpy.mock.calls.map((call) => call[0])).toEqual([ + '[dest] info line', + '[dest] warn line', + ]); + }); + + it("runs the flow's own handler and the host's for every line, in that order", async () => { + const { calls } = await run( + { collector: { logger: { level: 'WARN' } }, withHandler: true }, + () => ({}), + ); + expect(calls.slice(1)).toEqual([ + ['flow', 'warn line'], + ['host', '[dest] warn line'], + ['flow', 'Push failed'], + [ + 'host', + '[dest] Push failed {"code":"transport","status":503,"retryable":true}', + ], + ]); + }); +}); diff --git a/packages/cli/src/__tests__/unit/commands/setup-init.test.ts b/packages/cli/src/__tests__/unit/commands/setup-init.test.ts new file mode 100644 index 000000000..4aa7112e8 --- /dev/null +++ b/packages/cli/src/__tests__/unit/commands/setup-init.test.ts @@ -0,0 +1,105 @@ +import os from 'os'; +import path from 'path'; +import fs from 'fs-extra'; +import { StepError } from '@walkeros/core'; +import { setupCommand } from '../../../commands/setup/index.js'; +import { createMockLogger } from '../../helpers/mock-logger.js'; + +/** + * A step's `init` that fails during `walkeros setup` ends the command with an + * error naming the reason (non-zero exit through the CLI's error handler), + * never a quiet skip. + */ +describe('walkeros setup and the init contract', () => { + let workDir: string; + const setupCalls: unknown[] = []; + + beforeEach(async () => { + workDir = await fs.mkdtemp(path.join(os.tmpdir(), 'setup-init-')); + setupCalls.length = 0; + Reflect.set(globalThis, 'walkerosSetupCalls', setupCalls); + }); + + afterEach(async () => { + Reflect.deleteProperty(globalThis, 'walkerosSetupCalls'); + Reflect.deleteProperty(globalThis, 'walkerosSetupInitError'); + await fs.remove(workDir); + }); + + async function flowWithInit(init: string): Promise { + const pkgDir = path.join(workDir, 'fixture-dest'); + await fs.ensureDir(pkgDir); + await fs.writeJson(path.join(pkgDir, 'package.json'), { + name: '@walkeros/fixture-dest', + version: '1.0.0', + main: './index.mjs', + }); + await fs.writeFile( + path.join(pkgDir, 'index.mjs'), + `export default { + type: 'fixture', + push: () => {}, + init: ${init}, + setup: async () => { globalThis.walkerosSetupCalls.push('setup'); return { provisioned: true }; }, +}; +`, + ); + const file = path.join(workDir, 'flow.json'); + await fs.writeJson(file, { + version: 4, + flows: { + default: { + config: { + platform: 'server', + bundle: { + packages: { '@walkeros/fixture-dest': { path: pkgDir } }, + }, + }, + destinations: { + d: { package: '@walkeros/fixture-dest', config: { setup: true } }, + }, + }, + }, + }); + return file; + } + + const run = (config: string) => + setupCommand({ + target: 'destination.d', + config, + logger: createMockLogger(), + }); + + it("fails with the StepError's code and message", async () => { + Reflect.set( + globalThis, + 'walkerosSetupInitError', + new StepError('apiKey is required', { code: 'config' }), + ); + const config = await flowWithInit( + '() => { throw globalThis.walkerosSetupInitError; }', + ); + + await expect(run(config)).rejects.toThrow( + 'setup: destination.d init failed (config): apiKey is required', + ); + expect(setupCalls).toEqual([]); + }); + + it('fails as not-ready when init returns false', async () => { + const config = await flowWithInit('() => false'); + + await expect(run(config)).rejects.toThrow( + 'setup: destination.d init failed (not-ready): init returned false, so the step is not ready', + ); + expect(setupCalls).toEqual([]); + }); + + it('runs setup after an init that resolves', async () => { + const config = await flowWithInit('async () => undefined'); + + await expect(run(config)).resolves.toBeUndefined(); + expect(setupCalls).toEqual(['setup']); + }); +}); diff --git a/packages/cli/src/__tests__/unit/config/config-loader.test.ts b/packages/cli/src/__tests__/unit/config/config-loader.test.ts index 43365e3fe..7c3f672fc 100644 --- a/packages/cli/src/__tests__/unit/config/config-loader.test.ts +++ b/packages/cli/src/__tests__/unit/config/config-loader.test.ts @@ -376,6 +376,46 @@ describe('Config Loader', () => { ).toThrow(/platform/i); }); + test('the bundle preflight stops a flow validate rejects, naming every path (D4, E26)', () => { + const code = { type: 'd', push: '$code:(event) => {}' }; + const config = { + version: 4, + flows: { + 'tracking-server': { + config: { platform: 'server', url: 'https://collect.example.com' }, + }, + web: { + config: { platform: 'web' }, + sources: { + s: { + code, + config: { mapping: { page: { view: { data: 'data' } } } }, + }, + }, + destinations: { + api: { + code, + config: { settings: { url: '$flow. tracking-server.url' } }, + }, + }, + }, + }, + }; + + expect(() => + loadBundleConfig(config, { + configPath: '/test/config.json', + flowName: 'web', + }), + ).toThrow( + [ + 'Invalid configuration:', + " - flows.web.sources.s: `config.mapping.page.view.data` does nothing at the source position; only `consent`, `policy` and a rule's `condition`, `policy`, `name` and `ignore` apply; edit the event at a source with `policy`.", + ' - flows.web.destinations.api.config.settings.url: "$flow. tracking-server.url" does not match the $flow.NAME grammar', + ].join('\n'), + ); + }); + test('returns empty array for non-Flow.Json config', () => { const oldFormatConfig = { flow: { platform: 'web' }, diff --git a/packages/cli/src/__tests__/unit/validate/adversarial.test.ts b/packages/cli/src/__tests__/unit/validate/adversarial.test.ts index 3a141de14..2a0ad74c4 100644 --- a/packages/cli/src/__tests__/unit/validate/adversarial.test.ts +++ b/packages/cli/src/__tests__/unit/validate/adversarial.test.ts @@ -721,7 +721,7 @@ describe('C1 default scope and C5 skips', () => { }); it.each([ - ['f24-unknown-step-keys', 'warnings'], + ['f27-only-warning', 'warnings'], ['f17-only-skip', 'skips'], ])( 'C7: text output of %s (%s) under --strict exits 2 and never reads as passed', @@ -777,6 +777,37 @@ describe('C1 default scope and C5 skips', () => { expect(result.details.scope).toMatchObject({ flows: ['a'] }); }); + it('an unknown settings key is an error where the package closes its schema, and nothing where it is open', async () => { + const result = await validate('flow', { + version: 4, + flows: { + a: { + config: { platform: 'server' }, + destinations: { + closed: { + package: 'pkg-old', + config: { settings: { projectId: 'p', projectld: 'typo' } }, + }, + open: { + package: 'pkg-x', + config: { settings: { id: 'G-ABC123', extra: true } }, + }, + }, + }, + }, + }); + expect(result.valid).toBe(false); + expect(result.errors).toEqual([ + expect.objectContaining({ + path: at('flows.a.destinations.closed.config.settings'), + code: 'ENTRY_SCHEMA', + keyword: 'additionalProperties', + message: expect.stringContaining('"projectld"'), + }), + ]); + expect(result.warnings).toEqual([]); + }); + it('N1: a named export of a multi-export package without per-export schemas is a skip, not checked against the default schema', async () => { const result = await validate('flow', { version: 4, @@ -1081,7 +1112,7 @@ describe('C1 default scope and C5 skips', () => { describe('C4 and C9: strict fails warnings and skips in every entry point', () => { it.each([ - ['f24-unknown-step-keys', 'warnings'], + ['f27-only-warning', 'warnings'], ['f17-only-skip', 'skips'], ])( 'validate() with strict on %s (%s) is invalid, with no new errors', @@ -1099,7 +1130,7 @@ describe('C4 and C9: strict fails warnings and skips in every entry point', () = ); it.each([ - ['f24-unknown-step-keys', 'warnings'], + ['f27-only-warning', 'warnings'], ['f17-only-skip', 'skips'], ])('GUARD: without strict %s (%s) stays valid', async (fixture) => { const result = await validate('flow', fixturePath(fixture)); @@ -1120,10 +1151,10 @@ describe('C4 and C9: strict fails warnings and skips in every entry point', () = }); describe('N12 unknown keys on destinations and stores', () => { - it('F24: an unknown top-level key on a destination or store is an UNKNOWN_KEY warning', async () => { + it('F24: an unknown top-level key on a destination or store is an UNKNOWN_KEY error', async () => { const result = await validate('flow', fixturePath('f24-unknown-step-keys')); - expect(result.valid).toBe(true); - expect(result.warnings).toEqual( + expect(result.valid).toBe(false); + expect(result.errors).toEqual( expect.arrayContaining([ expect.objectContaining({ path: 'flows.a.destinations.d.settings', @@ -1138,19 +1169,18 @@ describe('N12 unknown keys on destinations and stores', () => { expect(result.details.scope).toMatchObject({ flows: ['a'] }); }); - it('F24: --strict exits 2 on the unknown key warnings (its only findings)', async () => { + it('F24: the unknown keys are its only findings and fail without --strict', async () => { const result = await validate('flow', fixturePath('f24-unknown-step-keys')); - expect(result.warnings.map((w) => w.code)).toEqual([ + expect(result.errors.map((e) => e.code)).toEqual([ 'UNKNOWN_KEY', 'UNKNOWN_KEY', ]); - expect(result.details.skipped).toEqual([]); + expect(result.warnings).toEqual([]); const { exit } = await runCli({ type: 'flow', input: fixturePath('f24-unknown-step-keys'), - strict: true, }); - expect(exit).toBe(2); + expect(exit).toBe(1); }); }); diff --git a/packages/cli/src/__tests__/unit/validate/contract.test.ts b/packages/cli/src/__tests__/unit/validate/contract.test.ts index 606baeb5c..719fe2d73 100644 --- a/packages/cli/src/__tests__/unit/validate/contract.test.ts +++ b/packages/cli/src/__tests__/unit/validate/contract.test.ts @@ -5,8 +5,12 @@ describe('validateContract', () => { it('should validate a valid named contract', () => { const result = validateContract({ default: { - globals: { required: ['country'] }, - consent: { required: ['analytics'] }, + schema: { + properties: { + globals: { required: ['country'] }, + consent: { required: ['analytics'] }, + }, + }, }, web: { extend: 'default', @@ -60,13 +64,26 @@ describe('validateContract', () => { expect(result.errors[0].code).toBe('CIRCULAR_EXTENDS'); }); - it('should report error for invalid section type', () => { - const result = validateContract({ - web: { globals: 'not an object' }, - }); - expect(result.valid).toBe(false); - expect(result.errors[0].code).toBe('INVALID_SECTION'); - }); + it.each(['globals', 'context', 'custom', 'user', 'consent'])( + 'E9: rejects the %s section, which nothing enforces, and points to schema', + (section) => { + const result = validateContract({ + web: { + description: 'shop', + [section]: { type: 'object', required: ['x'] }, + events: { product: { view: {} } }, + }, + }); + expect(result.valid).toBe(false); + expect(result.errors).toEqual([ + { + path: `web.${section}`, + message: `Contract section "${section}" is not enforced: the validate transformer reads only "events" and "schema". Put the rule into "schema" (a JSON Schema for the whole event), for example "schema": { "properties": { "${section}": { ... } } }.`, + code: 'UNKNOWN_CONTRACT_KEY', + }, + ]); + }, + ); it('should validate events entity-action structure', () => { const result = validateContract({ @@ -82,13 +99,16 @@ describe('validateContract', () => { expect(result.errors[0].code).toBe('INVALID_ACTION_KEY'); }); - it('should accept contract with only consent', () => { + it('rejects a contract with only a consent section (not a flat map)', () => { const result = validateContract({ consent_only: { consent: { required: ['analytics'] }, }, }); - expect(result.valid).toBe(true); + expect(result.valid).toBe(false); + expect(result.errors.map((e) => [e.code, e.path])).toEqual([ + ['UNKNOWN_CONTRACT_KEY', 'consent_only.consent'], + ]); }); it('should accept empty contract', () => { @@ -136,9 +156,12 @@ describe('validateContract', () => { it('should validate deep extend chain', () => { const result = validateContract({ - default: { consent: { required: ['analytics'] } }, + default: { schema: { properties: { consent: { required: ['x'] } } } }, web: { extend: 'default', events: { product: { view: {} } } }, - web_loggedin: { extend: 'web', user: { required: ['id'] } }, + web_loggedin: { + extend: 'web', + schema: { properties: { user: { required: ['id'] } } }, + }, }); expect(result.valid).toBe(true); expect(result.details.contractCount).toBe(3); @@ -191,19 +214,19 @@ describe('validateContract canonical shape enforcement', () => { expect(result.errors).toEqual([]); }); - it('warns on unknown keys next to known ones', () => { + it('rejects unknown keys next to known ones', () => { const result = validateContract({ web: { events: {}, unexpected: 'value' }, }); - expect(result.valid).toBe(true); - expect(result.warnings).toEqual( - expect.arrayContaining([ - expect.objectContaining({ - code: 'UNKNOWN_CONTRACT_KEY', - path: 'web.unexpected', - }), - ]), - ); + expect(result.valid).toBe(false); + expect(result.errors).toEqual([ + { + code: 'UNKNOWN_CONTRACT_KEY', + path: 'web.unexpected', + message: + 'Unknown contract key "unexpected" is not supported. Known keys: extend, tagging, description, events, schema.', + }, + ]); }); it('rejects non-number tagging and non-object schema', () => { diff --git a/packages/cli/src/__tests__/unit/validate/flow.test.ts b/packages/cli/src/__tests__/unit/validate/flow.test.ts index 83320efd0..50ddb0a9b 100644 --- a/packages/cli/src/__tests__/unit/validate/flow.test.ts +++ b/packages/cli/src/__tests__/unit/validate/flow.test.ts @@ -262,7 +262,7 @@ describe('validateFlow', () => { ).toBe(true); }); - it('passes structural warnings through as warnings, not errors', () => { + it('rejects a transformer consent that does nothing there (E20, MAPPING_NO_OP)', () => { const result = validateFlow({ version: 4, flows: { @@ -280,14 +280,15 @@ describe('validateFlow', () => { }, }); - expect(result.errors).toEqual([]); - expect(result.valid).toBe(true); - expect(result.warnings).toContainEqual({ - path: 'flows.default.transformers.shaped', - message: - "`mapping.consent` does nothing at the transformer position; only `policy` and a rule's `condition`, `policy`, `name` and `ignore` apply.", - code: 'TRANSFORMER_MAPPING_NO_OP', - }); + expect(result.valid).toBe(false); + expect(result.errors).toEqual([ + { + path: 'flows.default.transformers.shaped', + message: + "`mapping.consent` does nothing at the transformer position; only `policy` and a rule's `condition`, `policy`, `name` and `ignore` apply.", + code: 'MAPPING_NO_OP', + }, + ]); }); it('warns on colon-instead-of-dot typos ($store:NAME)', () => { @@ -1195,3 +1196,413 @@ describe('validateFlow', () => { }); }); }); + +describe('closed schemas (silent failures E1, E7, E10, E11, E14, E24, E25, E26)', () => { + const code = { + type: 'console', + init: '$code:() => {}', + push: '$code:(event) => { console.log(event.name); }', + }; + + it('E1: rejects unknown keys inside a destination config', () => { + const result = validateFlow({ + version: 4, + flows: { + default: { + config: { platform: 'web' }, + destinations: { + console: { + config: { + queueMaxx: 5, + mappingg: { page: { view: { name: 'pv' } } }, + consentt: { marketing: true }, + }, + code, + }, + }, + }, + }, + }); + expect(result.valid).toBe(false); + expect(result.errors.map((e) => [e.code, e.path])).toEqual([ + ['UNKNOWN_KEY', 'flows.default.destinations.console.config.queueMaxx'], + ['UNKNOWN_KEY', 'flows.default.destinations.console.config.mappingg'], + ['UNKNOWN_KEY', 'flows.default.destinations.console.config.consentt'], + ]); + }); + + it('E1: an unknown key on a destination entry is an error', () => { + const result = validateFlow({ + version: 4, + flows: { + default: { + config: { platform: 'web' }, + destinations: { + console: { validate: { data: { type: 'object' } }, code }, + }, + }, + }, + }); + expect(result.errors).toEqual([ + expect.objectContaining({ + path: 'flows.default.destinations.console.validate', + code: 'UNKNOWN_KEY', + }), + ]); + }); + + it('collector.consentDenied is a known key; an unknown collector key is not', () => { + const flow = (collector: Record) => ({ + version: 4, + flows: { default: { config: { platform: 'server' }, collector } }, + }); + expect(validateFlow(flow({ consentDenied: 'skip' })).errors).toEqual([]); + expect(validateFlow(flow({ consentDeny: 'skip' })).errors).toEqual([ + expect.objectContaining({ + path: 'flows.default.collector.consentDeny', + code: 'UNKNOWN_KEY', + }), + ]); + }); + + it('E2: rejects an inline code key the bundler would drop', () => { + const result = validateFlow({ + version: 4, + flows: { + default: { + config: { platform: 'web' }, + destinations: { + console: { code: { ...code, onConsent: '$code:() => {}' } }, + }, + }, + }, + }); + expect(result.valid).toBe(false); + expect(result.errors[0]).toMatchObject({ + path: 'flows.default.destinations.console.code', + code: 'SCHEMA_VALIDATION', + }); + expect(result.errors[0].message).toContain('onConsent'); + }); + + it('E2: accepts on, pushBatch and destroy in inline code', () => { + const result = validateFlow({ + version: 4, + flows: { + default: { + config: { platform: 'web' }, + destinations: { + console: { + config: { batch: 10 }, + code: { + ...code, + on: '$code:(type) => {}', + pushBatch: '$code:(batch) => {}', + destroy: '$code:() => {}', + }, + }, + }, + }, + }, + }); + expect(result.errors).toEqual([]); + }); + + it('E3: rejects a batch on inline code without pushBatch', () => { + const result = validateFlow({ + version: 4, + flows: { + default: { + config: { platform: 'web' }, + destinations: { console: { config: { batch: 10 }, code } }, + }, + }, + }); + expect(result.errors).toEqual([ + expect.objectContaining({ + path: 'flows.default.destinations.console', + code: 'BATCH_WITHOUT_PUSH_BATCH', + }), + ]); + }); + + it('E7: rejects a mapping value with key and fn', () => { + const result = validateFlow({ + version: 4, + flows: { + default: { + config: { platform: 'web' }, + destinations: { + console: { + config: { + mapping: { + page: { + view: { + data: { + map: { + title: { + key: 'data.title', + fn: '$code:(v) => String(v).toUpperCase()', + }, + }, + }, + }, + }, + }, + }, + code, + }, + }, + }, + }, + }); + expect(result.errors).toEqual([ + expect.objectContaining({ + path: 'flows.default.destinations.console.config.mapping.page.view.data.map.title', + code: 'MAPPING_VALUE_PRODUCERS', + message: expect.stringContaining('`fn` never runs'), + }), + ]); + }); + + it('E14: rejects a mapping object without an operator', () => { + const result = validateFlow({ + version: 4, + flows: { + default: { + config: { platform: 'server' }, + destinations: { + rec: { + code: { type: 'rec', push: '$code:(event, ctx) => {}' }, + config: { + data: { total: 'data.total', currency: 'data.currency' }, + }, + }, + }, + }, + }, + }); + expect(result.errors).toEqual([ + expect.objectContaining({ + path: 'flows.default.destinations.rec.config.data', + code: 'MAPPING_VALUE_NO_OPERATOR', + }), + ]); + }); + + it('E24, E25: rejects mapping fields with no effect at a source and a destination', () => { + const result = validateFlow({ + version: 4, + flows: { + default: { + config: { platform: 'web' }, + sources: { + s: { + code, + config: { mapping: { page: { view: { data: 'data' } } } }, + }, + }, + destinations: { + d: { + code, + config: { + mapping: { page: { view: { extend: { name: 'x' } } } }, + }, + }, + }, + }, + }, + }); + expect(result.errors.map((e) => [e.code, e.path])).toEqual([ + ['MAPPING_NO_OP', 'flows.default.sources.s'], + ['MAPPING_NO_OP', 'flows.default.destinations.d'], + ]); + expect(result.errors[0].message).toContain( + 'edit the event at a source with `policy`', + ); + expect(result.errors[1].message).toContain( + "applies only inside a package's own `settings.mapping` (today `@walkeros/transformer-ga4`)", + ); + }); + + it.each([ + ['$flow. tracking-server.url', 'tracking-server'], + ['$flow.1st.url', '1st'], + ['https://$flow.web.url/e', 'tracking-server'], + ['$store.cache.key', 'tracking-server'], + ['$secret.api_key', 'tracking-server'], + ['see $contract.web', 'tracking-server'], + ])('E26: rejects %s', (url, serverName) => { + const result = validateFlow({ + version: 4, + flows: { + [serverName]: { + config: { platform: 'server', url: 'https://collect.example.com' }, + }, + web: { + config: { platform: 'web' }, + destinations: { api: { code, config: { settings: { url } } } }, + }, + }, + }); + expect(result.valid).toBe(false); + expect(result.errors).toContainEqual( + expect.objectContaining({ + path: 'flows.web.destinations.api.config.settings.url', + code: 'MALFORMED_REFERENCE', + }), + ); + }); + + it('E26: a valid hyphenated $flow reference passes', () => { + const result = validateFlow({ + version: 4, + flows: { + 'tracking-server': { + config: { platform: 'server', url: 'https://collect.example.com' }, + }, + web: { + config: { platform: 'web' }, + destinations: { + api: { + code, + config: { settings: { url: '$flow.tracking-server.url' } }, + }, + }, + }, + }, + }); + expect(result.errors).toEqual([]); + expect(result.valid).toBe(true); + }); + + it('E11: an empty $flow value is unresolved, not inlined', () => { + const result = validateFlow({ + version: 4, + flows: { + server: { + config: { + platform: 'server', + url: 'https://collect.example.com', + settings: { region: '' }, + }, + }, + web: { + config: { platform: 'web' }, + destinations: { + api: { + code, + config: { settings: { region: '$flow.server.settings.region' } }, + }, + }, + }, + }, + }); + expect(result.errors).toEqual([ + { + path: 'flows.web', + message: + '$flow.server.settings.region is empty. Set flows.server.config.settings.region.', + code: 'UNRESOLVED_REFERENCE', + }, + ]); + }); + + it('E10: a validate step contract with $env is recorded in details.skipped', () => { + const result = validateFlow( + { + version: 4, + contract: { + web: { + events: { + product: { + view: { + type: 'object', + required: ['data'], + properties: { + data: { + type: 'object', + properties: { + price: { type: 'number', minimum: '$env.MIN_PRICE' }, + }, + }, + }, + }, + }, + }, + }, + }, + flows: { + default: { + config: { platform: 'web' }, + transformers: { + validate: { + package: '@walkeros/transformer-validate', + config: { + settings: { contract: ['$contract.web'], mode: 'strict' }, + }, + examples: { + cheap: { + in: { + name: 'product view', + entity: 'product', + action: 'view', + data: { price: 1 }, + }, + out: [ + [ + 'return', + { + event: { + name: 'product view', + entity: 'product', + action: 'view', + data: { price: 1 }, + }, + }, + ], + ], + }, + }, + }, + }, + }, + }, + }, + { strict: true }, + ); + expect(result.errors).toEqual([]); + expect(result.details.skipped).toContainEqual( + expect.objectContaining({ + path: 'flows.default.transformers.validate', + check: 'flow:contract-examples', + code: 'CONTRACT_NOT_STATIC', + }), + ); + }); + + it('rejects logger.format on a web flow, accepts it on a server flow', () => { + const flow = (platform: string) => ({ + version: 4, + flows: { + default: { + config: { platform }, + collector: { logger: { level: 'WARN', format: 'json' } }, + destinations: { + d: { code, config: { logger: { format: 'text' } } }, + }, + }, + }, + }); + expect(validateFlow(flow('server')).errors).toEqual([]); + expect( + validateFlow(flow('web')).errors.map((e) => [e.code, e.path]), + ).toEqual([ + ['WEB_LOGGER_FORMAT', 'flows.default.collector.logger.format'], + [ + 'WEB_LOGGER_FORMAT', + 'flows.default.destinations.d.config.logger.format', + ], + ]); + }); +}); diff --git a/packages/cli/src/commands/bundle/__tests__/bundler.test.ts b/packages/cli/src/commands/bundle/__tests__/bundler.test.ts index 437834f85..4a54f0208 100644 --- a/packages/cli/src/commands/bundle/__tests__/bundler.test.ts +++ b/packages/cli/src/commands/bundle/__tests__/bundler.test.ts @@ -13,6 +13,7 @@ import { generateWrapEntryServer, bundleCore, generateCacheKeyContent, + applyCollectorDefaults, } from '../bundler'; import { cacheBuild, getBuildCachePath } from '../../../core/build-cache.js'; import { tmpRunDir } from '../../../core/tmp-names.js'; @@ -730,6 +731,8 @@ describe('bundleCore server nft wiring (Phase 2)', () => { ...(opts.include ? { include: opts.include } : {}), }; const versionsHash = opts.versionsHash ?? (await getHashServer('', 12)); + // The key bundleCore takes: after the collector defaults it bakes in. + applyCollectorDefaults(opts.flowSettings); await cacheBuild( generateCacheKeyContent(opts.flowSettings, buildOptions, versionsHash), opts.bundleText ?? '// cached server bundle\n', @@ -1413,10 +1416,12 @@ describe('bundleCore provenance', () => { expect(expectedDigest).toMatch(/^[0-9a-f]{12}$/); expect(flowSettings.collector).toEqual({ + consentDenied: 'skip', name: 'default', release: expectedDigest, }); expect(JSON.parse(mockCacheBuild.mock.calls[0][0]).flow.collector).toEqual({ + consentDenied: 'skip', name: 'default', release: expectedDigest, }); @@ -1435,10 +1440,10 @@ describe('bundleCore provenance', () => { expect(flowSettings.collector?.release).toBe(expected); }); - it('leaves the collector untouched without provenance (push and simulate)', async () => { + it('bakes only the collector defaults without provenance (push and simulate)', async () => { const { flowSettings, buildOptions } = build(); await bundleCore(flowSettings, buildOptions, logger, false); - expect(flowSettings.collector).toBeUndefined(); + expect(flowSettings.collector).toEqual({ consentDenied: 'skip' }); }); }); diff --git a/packages/cli/src/commands/bundle/bundler.ts b/packages/cli/src/commands/bundle/bundler.ts index 32616ae19..dcf675403 100644 --- a/packages/cli/src/commands/bundle/bundler.ts +++ b/packages/cli/src/commands/bundle/bundler.ts @@ -5,10 +5,12 @@ import { fileURLToPath } from 'url'; import fs from 'fs-extra'; import type { Flow, MoinTarget, ObserveWeb, PreviewKey } from '@walkeros/core'; import { + getPlatform, packageNameToVariable, getStepRuntimeProps, ENV_MARKER_PREFIX, SECRET_MARKER_PREFIX, + STEP_FUNCTION_KEYS, isPathStepEntry, isObject, } from '@walkeros/core'; @@ -45,8 +47,8 @@ export { }; /** - * Type guard to check if a code value is an InlineCode object. - * InlineCode has { push: string, type?: string, init?: string } + * Type guard to check if a code value is an InlineCode object: `push`, an + * optional `type`, and the other step functions (`STEP_FUNCTION_KEYS`). */ function isInlineCode(code: unknown): code is Flow.Code { return ( @@ -67,9 +69,11 @@ function hasCodeReference(code: unknown): boolean { /** * Generates inline code for any component type (source, destination, - * transformer, store). Handles $code: prefix for push/init functions. + * transformer, store). Emits every step function the code defines + * (`STEP_FUNCTION_KEYS`: push, pushBatch, init, on, destroy), each with its + * $code: prefix removed, so none is dropped on the way into the bundle. * - * @param inline - InlineCode object with push, optional init, optional type + * @param inline - InlineCode object with push, optional type and functions * @param runtimeProps - The step's runtime fields (`getStepRuntimeProps`), * emitted beside the code. `config` and `env` default to `{}`. * @param isDestination - Whether this is a destination (uses different code structure) @@ -79,9 +83,11 @@ function generateInlineCode( runtimeProps: Record, isDestination?: boolean, ): string { - const pushFn = inline.push.replace('$code:', ''); - const initFn = inline.init ? inline.init.replace('$code:', '') : undefined; const typeLine = inline.type ? `type: '${inline.type}',` : ''; + const functionLines = STEP_FUNCTION_KEYS.flatMap((key) => { + const fn = inline[key]; + return fn ? [`${key}: ${fn.replace('$code:', '')}`] : []; + }).join(',\n '); const props: Record = { config: {}, @@ -98,8 +104,7 @@ function generateInlineCode( code: { ${typeLine} config: ${processConfigValue(props.config)}, - ${initFn ? `init: ${initFn},` : ''} - push: ${pushFn} + ${functionLines} }, ${propLines} }`; @@ -110,8 +115,7 @@ function generateInlineCode( code: async (context) => ({ ${typeLine} config: context.config, - ${initFn ? `init: ${initFn},` : ''} - push: ${pushFn} + ${functionLines} }), ${propLines} }`; @@ -431,8 +435,10 @@ export async function bundleCore( ); const versionsHash = await getHashServer(sortedVersions.join('\n'), 12); - // Bake provenance before the L1 check and codegen, so the release sits - // in both the cache key and the artifact. + // Collector defaults, for every call shape (`walkeros push` and simulate + // pass no provenance), then provenance: both before the L1 check and + // codegen, so they sit in both the cache key and the artifact. + applyCollectorDefaults(flowSettings); if (provenance) { applyCollectorProvenance( flowSettings, @@ -1597,6 +1603,21 @@ export function buildDataPayload(flowSettings: Flow): Record { .dataPayloadObj; } +/** + * Bake the collector defaults a flow's platform sets: a server collector + * skips consent-denied events (`consentDenied: 'skip'`, counted in + * `status.destinations[id].skipped`, DEBUG) instead of queueing them, since + * no consent change ever reaches a server flow after the event. An authored + * `collector.consentDenied` wins; a web flow keeps the collector default + * (`'queue'`). + */ +export function applyCollectorDefaults(flowSettings: Flow): void { + if (getPlatform(flowSettings) !== 'server') return; + const collector = flowSettings.collector ?? {}; + collector.consentDenied ??= 'skip'; + flowSettings.collector = collector; +} + /** * Bake flow-config provenance onto the collector before codegen. * @@ -1837,6 +1858,41 @@ export function wireConfig(__data) { export { startFlow };`; } +/** + * The runner contract's logger merge, emitted into every server factory + * entry: the host (`runneros`) passes `context.logger` (masking, the error + * ring tap), optionally `context.logLevel` (a level it forces, `--verbose`) + * and `context.onLogger` (receives the flow's own `{ level, format }` before + * the collector starts). The flow's level and format win over the host's; + * a flow handler runs first, then the host's. + */ +const HOST_LOGGER_MERGE = ` // The host's logger (masking, its error ring) joins the flow's own: the + // flow's level and format win, unless the host forces a level (--verbose); + // the flow's handler runs first, then the host's, so neither replaces the + // other. The host learns the flow's settings before the collector starts, + // so every line, the first included, has the flow's format. + if (context.logger) { + const own = config.logger || {}; + const host = context.logger; + if (context.onLogger) context.onLogger({ level: own.level, format: own.format }); + const level = context.logLevel ?? own.level ?? host.level; + const handler = own.handler && host.handler + ? (lvl, message, ctx, scope, original) => { + try { + own.handler(lvl, message, ctx, scope, original); + } finally { + host.handler(lvl, message, ctx, scope, original); + } + } + : host.handler || own.handler; + config.logger = { + ...host, + ...(level !== undefined ? { level } : {}), + ...(own.format ? { format: own.format } : {}), + ...(handler ? { handler } : {}), + }; + }`; + /** * Generate a stage 2 entry file for server bundles. * Imports startFlow and wireConfig from the stage 1 .mjs file, @@ -1854,7 +1910,7 @@ const __configData = ${dataPayload}; export default async function(context = {}) { const config = wireConfig(__configData); - if (context.logger) config.logger = context.logger; +${HOST_LOGGER_MERGE} if (context.sourceSettings && config.sources) { for (const src of Object.values(config.sources)) { @@ -2300,7 +2356,7 @@ export function generateWrapEntryServer(stage1Path: string): string { export default async function(context = {}) { const config = wireConfig(__configData); - if (context.logger) config.logger = context.logger; +${HOST_LOGGER_MERGE} if (context.sourceSettings && config.sources) { for (const src of Object.values(config.sources)) { diff --git a/packages/cli/src/commands/push/__tests__/simulate-result-shape.test.ts b/packages/cli/src/commands/push/__tests__/simulate-result-shape.test.ts index 5259833b3..7202e5182 100644 --- a/packages/cli/src/commands/push/__tests__/simulate-result-shape.test.ts +++ b/packages/cli/src/commands/push/__tests__/simulate-result-shape.test.ts @@ -23,14 +23,16 @@ const validEvent: WalkerOS.DeepPartialEvent = { action: 'complete', }; -// Config object missing the source's `package` → simulateSource throws inside -// the try block and maps the error into a `Simulation.Result`. +// A structurally valid source without a `package` (inline code) → +// simulateSource throws inside the try block and maps the error into a +// `Simulation.Result`. (A source with neither package nor code is rejected +// earlier, by the bundle preflight.) const sourceConfigNoPackage: Flow.Json = { version: 4, flows: { default: { config: { platform: 'web' }, - sources: { browser: { config: {} } }, + sources: { browser: { code: { push: '$code:() => {}' }, config: {} } }, }, }, }; diff --git a/packages/cli/src/commands/setup/__tests__/lifecycle.test.ts b/packages/cli/src/commands/setup/__tests__/lifecycle.test.ts index 7fb2a0a97..61c421b17 100644 --- a/packages/cli/src/commands/setup/__tests__/lifecycle.test.ts +++ b/packages/cli/src/commands/setup/__tests__/lifecycle.test.ts @@ -225,12 +225,14 @@ describe('setupCommand: init → setup → destroy lifecycle (Bug G)', () => { expect(destroyCall?.config).toEqual(setupCall?.config); }); - test('init returning false aborts: setup does NOT run, destroy does NOT run', async () => { + test('init returning false fails as not-ready: setup does NOT run, destroy does NOT run', async () => { mockLoad(buildFlow('@walkeros/__test-init-aborts-destination')); - await setupCommand({ target: 'destination.d', logger }); + await expect( + setupCommand({ target: 'destination.d', logger }), + ).rejects.toThrow('setup: destination.d init failed (not-ready)'); - // Only init was called; setup and destroy skipped because init aborted. + // Only init was called; setup and destroy skipped because init failed. expect(calls.map((c) => c.fn)).toEqual(['init']); }); diff --git a/packages/cli/src/commands/setup/index.ts b/packages/cli/src/commands/setup/index.ts index cad24965f..982beb275 100644 --- a/packages/cli/src/commands/setup/index.ts +++ b/packages/cli/src/commands/setup/index.ts @@ -1,6 +1,11 @@ import fs from 'fs-extra'; import type { Logger } from '@walkeros/core'; -import { ENV_MARKER_PREFIX, SECRET_MARKER_PREFIX } from '@walkeros/core'; +import { + ENV_MARKER_PREFIX, + SECRET_MARKER_PREFIX, + StepError, + toFailureReason, +} from '@walkeros/core'; import { loadFlowConfig } from '../../config/loader.js'; import { createCLILogger } from '../../core/cli-logger.js'; import { createSuccessOutput, writeResult } from '../../core/output.js'; @@ -159,15 +164,12 @@ type InitOutcome = /** * Classify the value returned by a destination/source/store `init` * implementation: - * - `false` → explicit abort, do NOT call setup + * - `false` → the step is not ready (the collector reports it as + * `not-ready`); setup cannot run * - object → resolved Config, use it as the source of truth for setup * and destroy contexts * - void/undefined → init mutated the input config in place (legal but * rare); fall through to the original config - * - * Mirrors the handling in collector/src/destination.ts:534-540, where - * `configResult === false` short-circuits and an object replaces the - * config while undefined keeps the existing one. */ function classifyInitResult(value: unknown): InitOutcome { if (value === false) return { kind: 'aborted' }; @@ -177,6 +179,18 @@ function classifyInitResult(value: unknown): InitOutcome { return { kind: 'void' }; } +/** + * The error a failed `init` ends setup with: the step, the reason's code and + * message (a `StepError`'s own, capped, never its body), so the command + * exits non-zero and says why instead of aborting quietly. + */ +function initFailed(target: string, error: unknown): Error { + const reason = toFailureReason(error); + return new Error( + `setup: ${target} init failed (${reason.code}): ${reason.message}`, + ); +} + export async function setupCommand(opts: SetupCommandOptions): Promise { const startTime = Date.now(); @@ -310,19 +324,27 @@ export async function setupCommand(opts: SetupCommandOptions): Promise { // - false → init explicitly aborted; do NOT run setup let resolvedConfig: unknown = resolvedInputConfig; if (isLifecycleFn(initFn)) { - const initResult = await initFn({ - id: component.id, - config: resolvedInputConfig, - env: resolvedEnv, - logger: scoped, - }); + const target = `${component.kind}.${component.id}`; + let initResult: unknown; + try { + initResult = await initFn({ + id: component.id, + config: resolvedInputConfig, + env: resolvedEnv, + logger: scoped, + }); + } catch (error) { + throw initFailed(target, error); + } const outcome = classifyInitResult(initResult); if (outcome.kind === 'aborted') { - scoped.info( - `setup: skipped ${component.kind}.${component.id} (init returned false)`, + throw initFailed( + target, + new StepError( + 'init returned false, so the step is not ready; its init should throw a StepError that names the missing setting', + { code: 'not-ready' }, + ), ); - await emitSkipEnvelope('init returned false'); - return; } if (outcome.kind === 'config') { resolvedConfig = outcome.config; diff --git a/packages/cli/src/commands/validate/types.ts b/packages/cli/src/commands/validate/types.ts index 046d22d5f..8e412ac84 100644 --- a/packages/cli/src/commands/validate/types.ts +++ b/packages/cli/src/commands/validate/types.ts @@ -50,6 +50,7 @@ export const VALIDATE_CODES = [ 'CONTRACT_VIOLATION', 'MAPPING_DOT_KEY', 'FLOW_REF_WARNING', + 'WEB_LOGGER_FORMAT', 'FLOW_CYCLE', 'VAR_CYCLE', // Structure (the bundler's own preflight, validateFlowStructure) @@ -58,7 +59,11 @@ export const VALIDATE_CODES = [ 'STORE_REFERENCE_NOT_FOUND', 'STATE_STORE_UNKNOWN', 'STATE_STORE_FILE', - 'TRANSFORMER_MAPPING_NO_OP', + 'MAPPING_NO_OP', + 'MAPPING_VALUE_NO_OPERATOR', + 'MAPPING_VALUE_PRODUCERS', + 'BATCH_WITHOUT_PUSH_BATCH', + 'MALFORMED_REFERENCE', // Resolution (getFlowSettings, as the bundler resolves the flow) 'UNRESOLVED_REFERENCE', 'MISSING_ENV', @@ -75,7 +80,6 @@ export const VALIDATE_CODES = [ 'INVALID_TAGGING', 'INVALID_SCHEMA', 'INVALID_EXTENDS', - 'INVALID_SECTION', 'INVALID_EVENTS', 'CIRCULAR_EXTENDS', 'INVALID_ENTITY_KEY', diff --git a/packages/cli/src/commands/validate/validators/contract.ts b/packages/cli/src/commands/validate/validators/contract.ts index 736300f6d..c7cde8f1b 100644 --- a/packages/cli/src/commands/validate/validators/contract.ts +++ b/packages/cli/src/commands/validate/validators/contract.ts @@ -1,19 +1,14 @@ -import { isObject } from '@walkeros/core'; +import { CONTRACT_RULE_KEYS, isObject } from '@walkeros/core'; import type { ValidateResult, ValidationError, ValidationWarning, } from '../types.js'; +const KNOWN_KEYS = new Set(CONTRACT_RULE_KEYS); + +/** Event parts once offered as contract sections; nothing enforces them. */ const SECTION_KEYS = ['globals', 'context', 'custom', 'user', 'consent']; -const KNOWN_KEYS = new Set([ - 'extend', - 'tagging', - 'description', - 'events', - 'schema', - ...SECTION_KEYS, -]); export function validateContract(input: unknown): ValidateResult { const errors: ValidationError[] = []; @@ -67,7 +62,9 @@ export function validateContract(input: unknown): ValidateResult { const keys = Object.keys(obj); const unknownKeys = keys.filter((k) => !KNOWN_KEYS.has(k)); const hasKnownKey = keys.some((k) => KNOWN_KEYS.has(k)); - const objectValuedUnknown = unknownKeys.filter((k) => isObject(obj[k])); + const objectValuedUnknown = unknownKeys.filter( + (k) => isObject(obj[k]) && !SECTION_KEYS.includes(k), + ); // Check for flat entity-action maps (object-valued keys without known keys) if (!hasKnownKey && objectValuedUnknown.length > 0) { @@ -81,11 +78,14 @@ export function validateContract(input: unknown): ValidateResult { continue; } - // Warn on unknown keys + // Unknown keys are errors: the contract schema is closed, and the + // validate transformer would never apply them. for (const key of unknownKeys) { - warnings.push({ + errors.push({ path: `${name}.${key}`, - message: `Unknown contract key "${key}" is ignored`, + message: SECTION_KEYS.includes(key) + ? `Contract section "${key}" is not enforced: the validate transformer reads only "events" and "schema". Put the rule into "schema" (a JSON Schema for the whole event), for example "schema": { "properties": { "${key}": { ... } } }.` + : `Unknown contract key "${key}" is not supported. Known keys: ${[...KNOWN_KEYS].join(', ')}.`, code: 'UNKNOWN_CONTRACT_KEY', }); } @@ -126,20 +126,6 @@ export function validateContract(input: unknown): ValidateResult { } } - // Validate sections - for (const key of SECTION_KEYS) { - if (key in obj) { - if (typeof obj[key] !== 'object' || obj[key] === null) { - errors.push({ - path: `${name}.${key}`, - message: `Section "${key}" must be a JSON Schema object`, - value: obj[key], - code: 'INVALID_SECTION', - }); - } - } - } - // Validate events if (obj.events !== undefined) { if (typeof obj.events !== 'object' || obj.events === null) { diff --git a/packages/cli/src/commands/validate/validators/flow.ts b/packages/cli/src/commands/validate/validators/flow.ts index d8aeb8fe7..c1399397c 100644 --- a/packages/cli/src/commands/validate/validators/flow.ts +++ b/packages/cli/src/commands/validate/validators/flow.ts @@ -9,9 +9,14 @@ import { isObject, REF_CONTRACT, resolveContracts, + SECRET_MARKER_PREFIX, validateStepEntry, } from '@walkeros/core'; -import { schemas, validateFlowStructure } from '@walkeros/core/dev'; +import { + checkStepKeys, + schemas, + validateFlowStructure, +} from '@walkeros/core/dev'; import { validateEventAgainstContract, type ContractSource, @@ -63,8 +68,10 @@ const DEPRECATED_PACKAGES: Record = { 'has been removed. Use the built-in cache by omitting cache.store, or remove the store declaration if it was only used as a cache target.', }; -/** Step sections without their own closed-schema check (transformers have one). */ -const OPEN_STEP_KINDS = [ +/** Step sections and their kind, for the closed key check of every step. */ +const STEP_KINDS = [ + ['sources', 'Source'], + ['transformers', 'Transformer'], ['destinations', 'Destination'], ['stores', 'Store'], ] as const; @@ -96,8 +103,10 @@ export function validateFlow( /** * The whole-file run of `validate()`: {@link validateFlow} plus every step * with a `package` in scope checked against its package settings schema at - * the pinned version. Findings are warnings for one minor (D3); a schema - * that cannot be loaded is a skip; `offline` removes the check and says so. + * the pinned version. An unknown settings key is an error where the package + * closes its schema (`additionalProperties: false`), as every closed schema + * is; other findings are warnings for one minor (D3). A schema that cannot + * be loaded is a skip; `offline` removes the check and says so. */ export async function validateFlowWithPackages( input: unknown, @@ -150,7 +159,11 @@ export async function validateFlowWithPackages( } } for (const outcome of await Promise.all(pending)) { - result.warnings.push(...outcome.findings); + for (const finding of outcome.findings) { + if (finding.keyword === 'additionalProperties') + result.errors.push(finding); + else result.warnings.push(finding); + } run.skipped.push(...outcome.skipped); deferred.push(...outcome.deferred); if (outcome.checked) packages.push(outcome.checked); @@ -159,6 +172,7 @@ export async function validateFlowWithPackages( result.details.scope = describeScope(scope, run.checks); result.details.deferred = [...(result.details.deferred ?? []), ...deferred]; if (packages.length > 0) result.details.packages = packages; + result.valid = result.errors.length === 0; return result; } @@ -236,8 +250,16 @@ function runFlowChecks(input: unknown, options: FlowValidateOptions): FlowRun { for (const issue of coreResult.errors) { const path = issue.path || 'root'; if (!inScope(path)) continue; - if (path === 'contract' && contractErrors > 0) continue; - errors.push({ path, message: issue.message, code: 'SCHEMA_VALIDATION' }); + if ( + (path === 'contract' || path.startsWith('contract.')) && + contractErrors > 0 + ) + continue; + errors.push({ + path, + message: issue.message, + code: issue.code ?? 'SCHEMA_VALIDATION', + }); } for (const issue of coreResult.warnings) { const path = issue.path || 'root'; @@ -254,10 +276,11 @@ function runFlowChecks(input: unknown, options: FlowValidateOptions): FlowRun { }); } - // 6. Step entries. Transformers are closed (errors, single source of - // truth in @walkeros/core); an unknown key on a destination or store is - // dropped by the resolver, so it warns (UNKNOWN_KEY) for one minor. - // Removed packages warn (DEPRECATED_PACKAGE) for one minor (D3). + // 6. Step entries, closed (single source of truth in @walkeros/core): + // a transformer entry's shape, and an unknown key on a destination or + // store entry or in any step's `config`, which the runtime would + // ignore (UNKNOWN_KEY errors). Removed packages warn + // (DEPRECATED_PACKAGE) for one minor (D3). checks.push('flow:steps'); for (const flowName of scope.flows) { const flowValue = flows?.[flowName]; @@ -278,20 +301,13 @@ function runFlowChecks(input: unknown, options: FlowValidateOptions): FlowRun { } } } - for (const [section, kind] of OPEN_STEP_KINDS) { + for (const [section, kind] of STEP_KINDS) { const steps = flowValue[section]; if (!isObject(steps)) continue; for (const [name, step] of Object.entries(steps)) { if (!isObject(step)) continue; const at = `flows.${flowName}.${section}.${name}`; - for (const key of unknownStepKeys(step, kind)) { - warnings.push({ - path: `${at}.${key}`, - message: `Unknown key "${key}" on ${kind} is ignored by the runtime`, - suggestion: 'Remove the key, or move it to where it belongs.', - code: 'UNKNOWN_KEY', - }); - } + errors.push(...checkStepKeys(step, kind, at)); const pkg = step.package; if (typeof pkg === 'string' && pkg in DEPRECATED_PACKAGES) { warnings.push({ @@ -302,6 +318,7 @@ function runFlowChecks(input: unknown, options: FlowValidateOptions): FlowRun { } } } + checkWebLoggerFormat(flowName, flowValue, errors); } // 7. Extract flow details @@ -383,9 +400,10 @@ function runFlowChecks(input: unknown, options: FlowValidateOptions): FlowRun { }; // 10a. The bundler's own structural preflight (identifier names, package - // xor code, per-flow $store targets), so a valid result implies the - // bundle preflight passes. Its warnings (a transformer mapping that - // does nothing) stay warnings: the runtime keeps the step. + // xor code, per-flow $store targets, closed keys, mapping fields that + // do nothing at their position, mapping values, whole-value + // references), so a valid result implies the bundle preflight passes. + // A flow with earlier errors skips it, so nothing is reported twice. perFlow(['flow:structure'], (name, flow, file) => { const structure = validateFlowStructure({ ...file, @@ -542,22 +560,40 @@ function checkResolvedWindowCollector( } /** - * Top-level keys of a destination or store that the runtime does not know, - * read through core's closed step-entry check (one source of truth). + * `logger.format` chooses text or JSON lines for a server flow's output; a + * web flow logs to the browser console, where it has no effect. Checked on + * the collector's logger and on every step's `config.logger`. */ -function unknownStepKeys( - step: Record, - kind: 'Destination' | 'Store', -): string[] { - const unknown: string[] = []; - const rest: Record = { ...step }; - for (;;) { - const verdict = validateStepEntry(rest, kind); - if (verdict.code !== 'UNKNOWN_KEY' || verdict.key === undefined) break; - unknown.push(verdict.key); - Reflect.deleteProperty(rest, verdict.key); +function checkWebLoggerFormat( + flowName: string, + flow: Record, + errors: ValidationError[], +): void { + if (!isObject(flow.config) || flow.config.platform !== 'web') return; + const at = `flows.${flowName}`; + const loggers: Array<[string, unknown]> = []; + if (isObject(flow.collector)) + loggers.push([`${at}.collector.logger`, flow.collector.logger]); + for (const [section] of STEP_KINDS) { + const steps = flow[section]; + if (!isObject(steps)) continue; + for (const [name, step] of Object.entries(steps)) { + if (isObject(step) && isObject(step.config)) + loggers.push([ + `${at}.${section}.${name}.config.logger`, + step.config.logger, + ]); + } + } + for (const [path, logger] of loggers) { + if (!isObject(logger) || logger.format === undefined) continue; + errors.push({ + path: `${path}.format`, + message: + 'logger.format applies to server flows only: a web flow logs to the browser console, which has no output format. Remove it.', + code: 'WEB_LOGGER_FORMAT', + }); } - return unknown; } // --- Deep validation helpers --- @@ -930,6 +966,21 @@ function lintRoute( } } +/** + * Whether a value holds a `$env` or `$secret` value: resolved to the + * resolver's deferred marker, or still written as the reference. + */ +function hasRuntimeValue(value: unknown): boolean { + if (typeof value === 'string') + return ( + value.includes(ENV_MARKER_PREFIX) || + value.startsWith(SECRET_MARKER_PREFIX) || + /\$(?:env|secret)\./.test(value) + ); + if (Array.isArray(value)) return value.some(hasRuntimeValue); + return isObject(value) && Object.values(value).some(hasRuntimeValue); +} + const VALIDATE_PACKAGE = '@walkeros/transformer-validate'; /** A transformer step that runs `@walkeros/transformer-validate`. */ @@ -1055,6 +1106,19 @@ export function checkValidateStepExamples( } if (!step.examples) return; const contracts = resolution.sources; + // A `$env` or `$secret` value in the contract is known only when the + // bundle builds or runs; judged on its literal text, JSON Schema would + // ignore it. The check is deferred, and the skip says so. + if (contracts.some(hasRuntimeValue)) { + skipped?.push({ + path: stepPath, + check: 'flow:contract-examples', + reason: + 'the contract uses a $env or $secret value, known only when the bundle builds or runs, so the examples cannot be judged statically', + code: 'CONTRACT_NOT_STATIC', + }); + return; + } const mode = settings.mode === 'strict' ? 'strict' : 'pass'; const format = settings.format === true; diff --git a/packages/cli/src/config/__tests__/loader.test.ts b/packages/cli/src/config/__tests__/loader.test.ts index 888991eb7..893237d37 100644 --- a/packages/cli/src/config/__tests__/loader.test.ts +++ b/packages/cli/src/config/__tests__/loader.test.ts @@ -111,9 +111,12 @@ describe('loadBundleConfig deferred env by platform', () => { flows: { serverflow: { config: { platform: 'server' }, - collector: { url: '$env.COLLECTOR_URL' }, + collector: { globals: { url: '$env.COLLECTOR_URL' } }, destinations: { - d: { config: { apiKey: '$env.API_KEY' } }, + d: { + package: '@walkeros/server-destination-api', + config: { settings: { apiKey: '$env.API_KEY' } }, + }, }, }, }, @@ -121,8 +124,9 @@ describe('loadBundleConfig deferred env by platform', () => { const result = loadBundleConfig(config, { configPath: './test.json', }); - const collector = result.flowSettings.collector as Record; - expect(collector.url).toBe('__WALKEROS_ENV:COLLECTOR_URL'); + expect(result.flowSettings.collector?.globals?.url).toBe( + '__WALKEROS_ENV:COLLECTOR_URL', + ); }); it('resolves env normally for web flows', () => { @@ -132,7 +136,7 @@ describe('loadBundleConfig deferred env by platform', () => { flows: { webflow: { config: { platform: 'web' }, - collector: { url: '$env.WEB_KEY' }, + collector: { globals: { url: '$env.WEB_KEY' } }, destinations: {}, }, }, @@ -140,8 +144,7 @@ describe('loadBundleConfig deferred env by platform', () => { const result = loadBundleConfig(config, { configPath: './test.json', }); - const collector = result.flowSettings.collector as Record; - expect(collector.url).toBe('baked-value'); + expect(result.flowSettings.collector?.globals?.url).toBe('baked-value'); delete process.env.WEB_KEY; }); }); @@ -170,9 +173,11 @@ describe('loadBundleConfig web $env build environment', () => { d: { package: '@walkeros/web-destination-gtag', config: { - declared: '$env.GA4_ID:unset', - shell: '$env.SHELL_ID:unset', - kek: '$env.KEK:unset', + settings: { + declared: '$env.GA4_ID:unset', + shell: '$env.SHELL_ID:unset', + kek: '$env.KEK:unset', + }, }, }, }, @@ -187,9 +192,7 @@ describe('loadBundleConfig web $env build environment', () => { { configPath: './test.json' }, ); expect(result.flowSettings.destinations?.d.config).toEqual({ - declared: 'G-DECLARED', - shell: 'from-config', - kek: 'secret', + settings: { declared: 'G-DECLARED', shell: 'from-config', kek: 'secret' }, }); }); @@ -199,9 +202,7 @@ describe('loadBundleConfig web $env build environment', () => { buildEnv: {}, }); expect(result.flowSettings.destinations?.d.config).toEqual({ - declared: 'G-DECLARED', - shell: 'unset', - kek: 'unset', + settings: { declared: 'G-DECLARED', shell: 'unset', kek: 'unset' }, }); }); @@ -211,9 +212,7 @@ describe('loadBundleConfig web $env build environment', () => { buildEnv: { GA4_ID: 'G-BASE', SHELL_ID: 'from-base' }, }); expect(result.flowSettings.destinations?.d.config).toEqual({ - declared: 'G-DECLARED', - shell: 'from-base', - kek: 'unset', + settings: { declared: 'G-DECLARED', shell: 'from-base', kek: 'unset' }, }); }); @@ -227,7 +226,7 @@ describe('loadBundleConfig web $env build environment', () => { destinations: { d: { package: '@walkeros/server-destination-api', - config: { key: '$env.KEY' }, + config: { settings: { key: '$env.KEY' } }, }, }, }, @@ -236,7 +235,7 @@ describe('loadBundleConfig web $env build environment', () => { { configPath: './test.json', buildEnv: {} }, ); expect(result.flowSettings.destinations?.d.config).toEqual({ - key: '__WALKEROS_ENV:KEY', + settings: { key: '__WALKEROS_ENV:KEY' }, }); }); }); diff --git a/packages/cli/src/config/loader.ts b/packages/cli/src/config/loader.ts index 414b85aee..f412fb0ac 100644 --- a/packages/cli/src/config/loader.ts +++ b/packages/cli/src/config/loader.ts @@ -8,6 +8,7 @@ import path from 'path'; import fs from 'fs-extra'; import { getFlowSettings, getPlatform, type Flow } from '@walkeros/core'; +import { validateFlowStructure } from '@walkeros/core/dev'; import type { BuildOptions } from '../types/bundle.js'; import { validateFlowConfig, @@ -71,6 +72,23 @@ export interface LoadConfigOptions { }; } +/** + * The bundle preflight: the structural checks `walkeros validate` runs + * (`validateFlowStructure`), on the flow being built. A flow validate + * rejects (an unknown key, a mapping field with no effect at its position, + * a malformed reference) stops here, before anything is resolved or built, + * and the error names every path. + */ +function assertFlowStructure(config: Flow.Json, flowName: string): void { + const { errors } = validateFlowStructure({ + ...config, + flows: { [flowName]: config.flows[flowName] }, + }); + if (errors.length === 0) return; + const lines = errors.map((error) => ` - ${error.path}: ${error.message}`); + throw new Error(`Invalid configuration:\n${lines.join('\n')}`); +} + /** * Load and parse bundle configuration. * @@ -110,6 +128,7 @@ export function loadBundleConfig( // Determine which flow to use const flowName = resolveFlow(config, options.flowName, availableFlows); + assertFlowStructure(config, flowName); assertLiteralBuildEnv(config, flowName); // Resolve with deferred mode first (markers don't affect platform detection) diff --git a/packages/cli/src/examples/flow-complete.manifest.ts b/packages/cli/src/examples/flow-complete.manifest.ts index 9ee3fd6b1..ad4486992 100644 --- a/packages/cli/src/examples/flow-complete.manifest.ts +++ b/packages/cli/src/examples/flow-complete.manifest.ts @@ -1885,6 +1885,12 @@ export const flowCompleteCoverage = { push: 'code-push', type: 'code-type', init: { excluded: 'the inline pageGroup step needs no setup' }, + pushBatch: { + excluded: + 'the inline pageGroup step is a transformer; batching is a destination concern', + }, + on: { excluded: 'the inline pageGroup step reacts to no collector event' }, + destroy: { excluded: 'the inline pageGroup step holds nothing to release' }, } satisfies Coverage, source: { package: 'step-package', diff --git a/packages/cli/src/telemetry/flow.json b/packages/cli/src/telemetry/flow.json index 60749008b..5b1d992af 100644 --- a/packages/cli/src/telemetry/flow.json +++ b/packages/cli/src/telemetry/flow.json @@ -3,43 +3,49 @@ "contract": { "telemetry": { "description": "walkerOS internal usage telemetry from @walkeros/cli and @walkeros/mcp. Anonymous, opt-out, no PII.", - "user": { + "schema": { "type": "object", "properties": { - "device": { - "type": "string", - "description": "Installation UUID generated once per machine and persisted in ~/.config/walkeros/config.json. Not linked to any account." - }, - "session": { - "type": "string", - "description": "Ephemeral per-process UUID. Only present on MCP events; CLI omits because its session is a single event." - }, - "os": { - "type": "string" - }, - "osVersion": { - "type": "string" - }, - "node": { - "type": "string" - }, - "language": { - "type": "string" + "user": { + "type": "object", + "properties": { + "device": { + "type": "string", + "description": "Installation UUID generated once per machine and persisted in ~/.config/walkeros/config.json. Not linked to any account." + }, + "session": { + "type": "string", + "description": "Ephemeral per-process UUID. Only present on MCP events; CLI omits because its session is a single event." + }, + "os": { + "type": "string" + }, + "osVersion": { + "type": "string" + }, + "node": { + "type": "string" + }, + "language": { + "type": "string" + }, + "timezone": { + "type": "string" + } + } }, - "timezone": { - "type": "string" - } - } - }, - "consent": { - "type": "object", - "properties": { - "telemetry": { - "type": "boolean", - "const": true + "consent": { + "type": "object", + "properties": { + "telemetry": { + "type": "boolean", + "const": true + } + }, + "required": ["telemetry"] } }, - "required": ["telemetry"] + "required": ["consent"] }, "events": { "mcp": { @@ -101,7 +107,9 @@ "api": { "package": "@walkeros/server-destination-api", "config": { - "url": "$env.WALKEROS_APP_URL/api/telemetry" + "settings": { + "url": "$env.WALKEROS_APP_URL/api/telemetry" + } } } } diff --git a/packages/collector/src/__tests__/registration-checks.test.ts b/packages/collector/src/__tests__/registration-checks.test.ts index 5a77914ae..98e533354 100644 --- a/packages/collector/src/__tests__/registration-checks.test.ts +++ b/packages/collector/src/__tests__/registration-checks.test.ts @@ -110,8 +110,8 @@ describe('mapping keys with no effect at their position (D4)', () => { }); expect(lines).toEqual([ - "Destination d: `config.mapping.page.view.extend` does nothing at the destination position; a rule's `extend` is merged only by a package onto its own shipped rules; write the fields on the rule itself.", - "Source probe: `config.mapping.page.view.data` does nothing at the source position; only `consent`, `policy` and a rule's `condition`, `policy`, `name` and `ignore` apply.", + "Destination d: `config.mapping.page.view.extend` does nothing at the destination position; a rule's `extend` applies only inside a package's own `settings.mapping` (today `@walkeros/transformer-ga4`), merged onto its shipped rules; write the fields on the rule itself.", + "Source probe: `config.mapping.page.view.data` does nothing at the source position; only `consent`, `policy` and a rule's `condition`, `policy`, `name` and `ignore` apply; edit the event at a source with `policy`.", ]); await elb({ name: 'page view', data: { id: 1 } }); diff --git a/packages/collector/src/__tests__/report-error.test.ts b/packages/collector/src/__tests__/report-error.test.ts index 71ab78b80..79cb6fb9c 100644 --- a/packages/collector/src/__tests__/report-error.test.ts +++ b/packages/collector/src/__tests__/report-error.test.ts @@ -49,7 +49,7 @@ describe('reportError', () => { const key = stepId('destination', 'bigquery'); expect(c.status.connectionErrors[key]).toBe(1); expect(c.status.failed).toBe(0); - expect(logger.error).toHaveBeenCalled(); + expect(logger.warn).toHaveBeenCalled(); }); test('accumulates connectionErrors across repeated orphan reports', async () => { @@ -72,10 +72,10 @@ describe('reportError', () => { expect(c.status.failed).toBe(0); }); - test('does not throw when the scoped logger.error throws', async () => { + test('does not throw when the scoped logger.warn throws', async () => { const c = await collector({}); const logger = createMockLogger(); - logger.error.mockImplementation(() => { + logger.warn.mockImplementation(() => { throw new Error('logger exploded'); }); const reportError = buildReportError( @@ -109,11 +109,13 @@ describe('reportError', () => { reportError(googleError); - expect(logger.error).toHaveBeenCalledWith('connection error', { + // No event was lost: WARN (Q8), never ERROR. + expect(logger.warn).toHaveBeenCalledWith('connection error', { error: 'Total timeout of API exceeded 10000 milliseconds', name: 'GoogleError', code: 4, }); + expect(logger.error).not.toHaveBeenCalled(); }); test('orphan log meta stays lean for a plain Error (no name/code noise)', async () => { @@ -128,7 +130,7 @@ describe('reportError', () => { reportError(new Error('stream broken')); - expect(logger.error).toHaveBeenCalledWith('connection error', { + expect(logger.warn).toHaveBeenCalledWith('connection error', { error: 'stream broken', }); }); diff --git a/packages/collector/src/destination.ts b/packages/collector/src/destination.ts index 6f93f69a5..fd4e58eda 100644 --- a/packages/collector/src/destination.ts +++ b/packages/collector/src/destination.ts @@ -23,9 +23,9 @@ import { emitStep, getId, getGrantedConsent, + getMappingKeyPaths, getMappingNoOpMessage, getMappingNoOpPaths, - isArray, isDefined, isFunction, isObject, @@ -2335,19 +2335,10 @@ export function checkDestinationRegistration( const logger = collector.logger.scope(destination.type || 'unknown'); if (!destination.pushBatch) { - const batchPaths: string[] = []; - if (config.batch !== undefined) batchPaths.push('config.batch'); - for (const [entity, actions] of Object.entries(config.mapping || {})) { - for (const [action, value] of Object.entries(actions || {})) { - const list = isArray(value) ? value : [value]; - list.forEach((rule, index) => { - if (rule?.batch === undefined) return; - batchPaths.push( - `config.mapping.${entity}.${action}${isArray(value) ? `[${index}]` : ''}.batch`, - ); - }); - } - } + const batchPaths = getMappingKeyPaths(config, 'config', { + config: ['batch'], + rule: ['batch'], + }); if (batchPaths.length) logger.error( `Destination ${id}: ${batchPaths.map((path) => `\`${path}\``).join(', ')} has no effect: the destination has no pushBatch, so events go to push one by one`, diff --git a/packages/collector/src/report-error.ts b/packages/collector/src/report-error.ts index ee53f4375..5f7935330 100644 --- a/packages/collector/src/report-error.ts +++ b/packages/collector/src/report-error.ts @@ -138,10 +138,11 @@ export function buildReportError( } // Orphan / connection-level error: count it under connectionErrors, - // not failed. + // not failed. No event was lost, so the user acts but intent is still + // met: WARN (an event a step fails reports above, at ERROR). collector.status.connectionErrors[key] = (collector.status.connectionErrors[key] ?? 0) + 1; - logger.error('connection error', errorMeta(err)); + logger.warn('connection error', errorMeta(err)); } catch { // Contained: reportError runs on a detached tick and must never throw. } diff --git a/packages/core/src/__tests__/flow.test.ts b/packages/core/src/__tests__/flow.test.ts index 5d659c57d..3e0c9cbdd 100644 --- a/packages/core/src/__tests__/flow.test.ts +++ b/packages/core/src/__tests__/flow.test.ts @@ -2616,6 +2616,32 @@ describe('$flow references', () => { expect(() => getFlowSettings(config, 'web')).toThrow(/url/); }); + test('treats an empty value as unresolved, as soft mode does', () => { + const config: Flow.Json = { + version: 4, + flows: { + server: { + config: { + platform: 'server', + url: 'https://collect.example.com', + settings: { region: '' }, + }, + }, + web: { + config: { platform: 'web' }, + destinations: { + api: { + config: { settings: { region: '$flow.server.settings.region' } }, + }, + }, + }, + }, + }; + expect(() => getFlowSettings(config, 'web')).toThrow( + '$flow.server.settings.region is empty. Set flows.server.config.settings.region.', + ); + }); + test('detects cycles', () => { const config: Flow.Json = { version: 4, diff --git a/packages/core/src/__tests__/mapping-no-ops.test.ts b/packages/core/src/__tests__/mapping-no-ops.test.ts index 33a11fd8f..58114e111 100644 --- a/packages/core/src/__tests__/mapping-no-ops.test.ts +++ b/packages/core/src/__tests__/mapping-no-ops.test.ts @@ -48,7 +48,7 @@ describe('getMappingNoOpPaths', () => { expect( getMappingNoOpMessage('source', ['config.mapping.page.view.data']), ).toBe( - "`config.mapping.page.view.data` does nothing at the source position; only `consent`, `policy` and a rule's `condition`, `policy`, `name` and `ignore` apply.", + "`config.mapping.page.view.data` does nothing at the source position; only `consent`, `policy` and a rule's `condition`, `policy`, `name` and `ignore` apply; edit the event at a source with `policy`.", ); }); }); diff --git a/packages/core/src/__tests__/node.test.ts b/packages/core/src/__tests__/node.test.ts index 3d4843875..10caeb904 100644 --- a/packages/core/src/__tests__/node.test.ts +++ b/packages/core/src/__tests__/node.test.ts @@ -1,6 +1,13 @@ import os from 'os'; import { Level } from '../types/logger'; -import { createCLILogger, getTmpPath } from '../node'; +import type { LogEntry } from '../node'; +import { + createCLILogger, + createCLILoggerConfig, + getTmpPath, + MAX_JSON_LINE_BYTES, +} from '../node'; +import { createLogger } from '../logger'; describe('createCLILogger', () => { let errorSpy: jest.SpyInstance; @@ -158,6 +165,160 @@ describe('createCLILogger', () => { }); }); +describe('createCLILogger format json', () => { + let errorSpy: jest.SpyInstance; + let logSpy: jest.SpyInstance; + + beforeEach(() => { + errorSpy = jest.spyOn(console, 'error').mockImplementation(() => {}); + logSpy = jest.spyOn(console, 'log').mockImplementation(() => {}); + }); + + afterEach(() => { + errorSpy.mockRestore(); + logSpy.mockRestore(); + }); + + const parse = (call: unknown[]): Record => { + const parsed: unknown = JSON.parse(String(call[0])); + if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) + throw new Error('not a JSON object'); + return Object.fromEntries(Object.entries(parsed)); + }; + + it('prints one JSON object with exactly severity, message, scope, context and time', () => { + createCLILogger({ format: 'json' }) + .scope('gcp-bigquery') + .error('connection error', { code: 'transport', status: 503 }); + + const line = parse(errorSpy.mock.calls[0]); + expect(Object.keys(line).sort()).toEqual([ + 'context', + 'message', + 'scope', + 'severity', + 'time', + ]); + expect(line).toMatchObject({ + severity: 'ERROR', + message: '[gcp-bigquery] connection error', + scope: 'gcp-bigquery', + context: { code: 'transport', status: 503 }, + }); + expect(new Date(String(line.time)).toISOString()).toBe(line.time); + }); + + it('masks known and pattern secrets in the message and in nested context', () => { + createCLILogger({ + format: 'json', + knownSecrets: ['value-123456', '98765432'], + }).error('auth failed with value-123456 "quoted"\nnext', { + nested: { + token: 'value-123456', + key: 'sk-abcdefghijklmnopqrstuvwxyz123456', + }, + port: 98765432, + }); + + const raw = String(errorSpy.mock.calls[0][0]); + expect(raw).not.toContain('value-123456'); + expect(raw).not.toContain('sk-abcdefghijklmnopqrstuvwxyz123456'); + const line = parse(errorSpy.mock.calls[0]); + expect(line.message).toBe('auth failed with *** "quoted"\nnext'); + expect(line.context).toMatchObject({ port: '***' }); + }); + + it('caps a line above 32 KiB with contextTruncated, and it still parses', () => { + createCLILogger({ format: 'json' }).error('big', { + rows: 'x"'.repeat(MAX_JSON_LINE_BYTES), + }); + + const raw = String(errorSpy.mock.calls[0][0]); + expect(Buffer.byteLength(raw)).toBeLessThanOrEqual(MAX_JSON_LINE_BYTES); + const line = parse(errorSpy.mock.calls[0]); + expect(line.contextTruncated).toBe(true); + expect(typeof line.context).toBe('string'); + expect(String(line.context).startsWith('{"rows":"x\\"')).toBe(true); + }); + + it('sends ERROR and WARN to stderr, INFO and DEBUG to stdout', () => { + const logger = createCLILogger({ format: 'json', verbose: true }); + logger.error('e'); + logger.warn('w'); + logger.info('i'); + logger.debug('d'); + + expect(errorSpy.mock.calls.map((call) => parse(call).severity)).toEqual([ + 'ERROR', + 'WARNING', + ]); + expect(logSpy.mock.calls.map((call) => parse(call).severity)).toEqual([ + 'INFO', + 'DEBUG', + ]); + }); + + it('reads a function-form format per line', () => { + let format: 'text' | 'json' = 'text'; + const logger = createCLILogger({ format: () => format }); + logger.error('first'); + format = 'json'; + logger.error('second'); + + expect(errorSpy.mock.calls[0][0]).toBe('first'); + expect(parse(errorSpy.mock.calls[1]).message).toBe('second'); + }); + + it('keeps text mode and the json option as they were', () => { + const logger = createCLILogger(); + logger.warn('plain', { a: 1 }); + expect(logSpy).toHaveBeenCalledWith('plain {"a":1}'); + + const machine = createCLILogger({ json: true, format: 'json' }); + machine.error('hidden'); + machine.info('hidden'); + machine.json({ ok: true }); + expect(errorSpy).not.toHaveBeenCalled(); + expect(logSpy).toHaveBeenLastCalledWith( + JSON.stringify({ ok: true }, null, 2), + ); + }); + + it('hands onLine the text line and the structured entry, in both formats', () => { + const taps: Array<[Level, string, LogEntry]> = []; + const logger = createLogger( + createCLILoggerConfig({ + onLine: (level, message, entry) => taps.push([level, message, entry]), + }), + ).scope('collector'); + logger.error('Push failed', { + code: 'transport', + status: 503, + retryable: true, + suppressed: 2, + }); + + expect(taps).toEqual([ + [ + Level.ERROR, + '[collector] Push failed {"code":"transport","status":503,"retryable":true,"suppressed":2}', + { + severity: 'ERROR', + message: '[collector] Push failed', + scope: 'collector', + context: { + code: 'transport', + status: 503, + retryable: true, + suppressed: 2, + }, + time: expect.any(String), + }, + ], + ]); + }); +}); + describe('getTmpPath', () => { it('joins segments under an absolute custom root', () => { expect(getTmpPath('/custom', 'cache', 'builds')).toBe( diff --git a/packages/core/src/__tests__/schemas/contract.test.ts b/packages/core/src/__tests__/schemas/contract.test.ts index 4f86a7fb1..b272ce856 100644 --- a/packages/core/src/__tests__/schemas/contract.test.ts +++ b/packages/core/src/__tests__/schemas/contract.test.ts @@ -6,7 +6,7 @@ describe('Contract schema validation', () => { version: 4, contract: { default: { - globals: { required: ['country'] }, + schema: { properties: { globals: { required: ['country'] } } }, events: { product: { '*': { @@ -23,7 +23,7 @@ describe('Contract schema validation', () => { }, web: { extend: 'default', - consent: { required: ['analytics'] }, + schema: { properties: { consent: { required: ['analytics'] } } }, }, }, flows: { @@ -43,18 +43,24 @@ describe('Contract schema validation', () => { expect(result.success).toBe(false); }); - it('should accept contract with only sections, no events', () => { - const result = JsonSchema.safeParse({ - version: 4, - contract: { - consent_only: { - consent: { required: ['analytics'] }, + it.each(['globals', 'context', 'custom', 'user', 'consent'])( + 'rejects the %s section: nothing enforces it (E9)', + (section) => { + const result = JsonSchema.safeParse({ + version: 4, + contract: { + web: { [section]: { required: ['analytics'] } }, }, - }, - flows: { default: { config: { platform: 'web' } } }, - }); - expect(result.success).toBe(true); - }); + flows: { default: { config: { platform: 'web' } } }, + }); + expect(result.success).toBe(false); + expect(result.error?.issues[0]).toMatchObject({ + code: 'unrecognized_keys', + keys: [section], + path: ['contract', 'web'], + }); + }, + ); it('should accept v4 setup without contract', () => { const result = JsonSchema.safeParse({ diff --git a/packages/core/src/__tests__/schemas/validate-flow-config.test.ts b/packages/core/src/__tests__/schemas/validate-flow-config.test.ts index 4072e22a3..ac233ac4a 100644 --- a/packages/core/src/__tests__/schemas/validate-flow-config.test.ts +++ b/packages/core/src/__tests__/schemas/validate-flow-config.test.ts @@ -1613,7 +1613,7 @@ describe('validateFlowConfig', () => { ['$secret.api_key', 'grammar'], ['$store.cache.key', 'grammar'], ['$flow.server.settings.my-key', 'grammar'], - ])('warns on %s (%s)', (value, kind) => { + ])('rejects %s (%s)', (value, kind) => { const result = validateFlowConfig( JSON.stringify(flowWithSetting({ url: value }), null, 2), ); @@ -1621,12 +1621,14 @@ describe('validateFlowConfig', () => { kind === 'inline' ? 'resolve only as the whole value' : 'does not match the'; - const warning = result.warnings.find((w) => w.message.includes(text)); - expect(warning).toBeDefined(); - expect(warning?.path).toBe( + const error = result.errors.find((e) => e.message.includes(text)); + expect(error).toBeDefined(); + expect(error?.code).toBe('MALFORMED_REFERENCE'); + expect(error?.path).toBe( 'flows.web.destinations.api.config.settings.url', ); - expect(warning?.line).toBeGreaterThan(1); + expect(error?.line).toBeGreaterThan(1); + expect(result.valid).toBe(false); }); it('points a longer $flow string at $var or $env composition', () => { @@ -1638,15 +1640,15 @@ describe('validateFlowConfig', () => { ), ); expect( - result.warnings.some((w) => - w.message.includes( + result.errors.some((e) => + e.message.includes( 'To build a longer string, compose with $var or $env.', ), ), ).toBe(true); }); - it('does not warn on prose that mentions a reference', () => { + it('does not reject prose that mentions a reference', () => { const result = validateFlowConfig( JSON.stringify( { @@ -1680,8 +1682,8 @@ describe('validateFlowConfig', () => { ); expect(result.errors).toEqual([]); expect( - result.warnings.some((w) => - /whole value|does not match the/.test(w.message), + result.errors.some((e) => + /whole value|does not match the/.test(e.message), ), ).toBe(false); }); @@ -1692,13 +1694,13 @@ describe('validateFlowConfig', () => { ['https://$env.HOST:x.example/collect'], ['$var.base/collect'], ['$code:(e) => "$flow.x"'], - ])('does not warn on %s', (value) => { + ])('does not reject %s', (value) => { const result = validateFlowConfig( JSON.stringify(flowWithSetting({ url: value }), null, 2), ); expect( - result.warnings.some((w) => - /whole value|does not match the/.test(w.message), + result.errors.some((e) => + /whole value|does not match the/.test(e.message), ), ).toBe(false); }); @@ -1789,9 +1791,9 @@ describe('validateFlowConfig', () => { 2, ); } - const warned = (json: string) => - validateFlowConfig(json).warnings.some((w) => - /whole value|does not match the/.test(w.message), + const rejected = (json: string) => + validateFlowConfig(json).errors.some((e) => + /whole value|does not match the/.test(e.message), ); it.each([ @@ -1814,7 +1816,7 @@ describe('validateFlowConfig', () => { ['a settings description', { settings: { description: bad } }], ['a settings title', { settings: { title: bad } }], ])('checks %s', (_label, overrides) => { - expect(warned(flowWith(overrides))).toBe(true); + expect(rejected(flowWith(overrides))).toBe(true); }); it.each([ @@ -1831,7 +1833,7 @@ describe('validateFlowConfig', () => { ['a root title', { root: { title: bad } }], ['nothing but the contract', {}], ])('skips %s', (_label, overrides) => { - expect(warned(flowWith(overrides))).toBe(false); + expect(rejected(flowWith(overrides))).toBe(false); }); it('skips a step settings.contract schema but checks settings.title', () => { @@ -1840,8 +1842,8 @@ describe('validateFlowConfig', () => { description: 'Read from $store.x.y', properties: { id: { type: 'string', description: bad } }, }; - expect(warned(flowWith({ settings: { contract } }))).toBe(false); - expect(warned(flowWith({ settings: { contract, title: bad } }))).toBe( + expect(rejected(flowWith({ settings: { contract } }))).toBe(false); + expect(rejected(flowWith({ settings: { contract, title: bad } }))).toBe( true, ); }); diff --git a/packages/core/src/__tests__/validate-structure.test.ts b/packages/core/src/__tests__/validate-structure.test.ts index 1693e83c7..df3b287d2 100644 --- a/packages/core/src/__tests__/validate-structure.test.ts +++ b/packages/core/src/__tests__/validate-structure.test.ts @@ -132,7 +132,7 @@ describe('validateFlowStructure', () => { destinations: { api: { package: '@walkeros/server-destination-api', - config: { token: '$store.secrets' }, + config: { settings: { token: '$store.secrets' } }, }, }, }, @@ -166,7 +166,7 @@ describe('validateFlowStructure', () => { expect(error?.path).toBe('flows.default.transformers.enrich'); }); - it('warns, without rejecting, on a transformer mapping that never runs or does nothing', () => { + it('rejects a transformer mapping that never runs or does nothing (MAPPING_NO_OP)', () => { const result = validateFlowStructure( flow({ default: { @@ -191,25 +191,25 @@ describe('validateFlowStructure', () => { }), ); - expect(result.valid).toBe(true); - expect(result.errors).toEqual([]); - expect(result.warnings).toEqual([ + expect(result.valid).toBe(false); + expect(result.warnings).toEqual([]); + expect(result.errors).toEqual([ { path: 'flows.default.transformers.packaged', message: '`mapping` is ignored: a transformer with `package` never runs it; a mapping applies only to a transformer without code.', - code: 'TRANSFORMER_MAPPING_NO_OP', + code: 'MAPPING_NO_OP', }, { path: 'flows.default.transformers.shaped', message: "`config.mapping.mapping.order.complete[0].consent` does nothing at the transformer position; only `policy` and a rule's `condition`, `policy`, `name` and `ignore` apply.", - code: 'TRANSFORMER_MAPPING_NO_OP', + code: 'MAPPING_NO_OP', }, ]); }); - it('warns that config.mapping overrides a step-level mapping', () => { + it('rejects a step-level mapping that config.mapping overrides', () => { const result = validateFlowStructure( flow({ default: { @@ -226,12 +226,12 @@ describe('validateFlowStructure', () => { }), ); - expect(result.valid).toBe(true); - expect(result.warnings).toEqual([ + expect(result.valid).toBe(false); + expect(result.errors).toEqual([ { path: 'flows.default.transformers.both', message: '`mapping` is overridden: `config.mapping` wins; remove one.', - code: 'TRANSFORMER_MAPPING_NO_OP', + code: 'MAPPING_NO_OP', }, ]); }); @@ -254,12 +254,12 @@ describe('validateFlowStructure', () => { }), ); - expect(result.warnings).toEqual([ + expect(result.errors).toEqual([ { path: 'flows.default.transformers.packaged', message: '`config.mapping` and `mapping` are ignored: a transformer with `package` never runs them; a mapping applies only to a transformer without code.', - code: 'TRANSFORMER_MAPPING_NO_OP', + code: 'MAPPING_NO_OP', }, ]); }); @@ -280,6 +280,314 @@ describe('validateFlowStructure', () => { }); }); +describe('closed schemas (validate and the bundle preflight)', () => { + const code = { + type: 'd', + init: '$code:() => {}', + push: '$code:(event) => {}', + }; + const codes = (config: Flow.Json) => + validateFlowStructure(config).errors.map((error) => [ + error.code, + error.path, + ]); + + it('rejects unknown keys in a step config and on a destination (E1)', () => { + const result = validateFlowStructure( + flow({ + default: { + config: { platform: 'web' }, + destinations: { + console: { + code, + config: { + queueMaxx: 5, + consentt: { marketing: true }, + mappingg: { page: { view: { name: 'pv' } } }, + }, + }, + }, + }, + }), + ); + expect(result.valid).toBe(false); + expect(result.errors.map((error) => error.path)).toEqual([ + 'flows.default.destinations.console.config.queueMaxx', + 'flows.default.destinations.console.config.consentt', + 'flows.default.destinations.console.config.mappingg', + ]); + expect(result.errors[1]).toMatchObject({ + code: 'UNKNOWN_KEY', + message: expect.stringContaining( + 'Unknown key "consentt" in Destination config is ignored by the runtime. Known keys:', + ), + }); + expect(result.errors[1].message).toContain(' consent,'); + }); + + it.each([ + ['sources', { code, config: { primaryy: true } }, 'config.primaryy'], + ['transformers', { code, config: { nextt: 'x' } }, 'config.nextt'], + ['stores', { code, config: { fil: true } }, 'config.fil'], + ['stores', { code, settings: {} }, 'settings'], + ])('rejects an unknown key on %s at %s', (section, step, key) => { + expect( + codes( + flow({ + default: { config: { platform: 'web' }, [section]: { s: step } }, + }), + ), + ).toContainEqual(['UNKNOWN_KEY', `flows.default.${section}.s.${key}`]); + }); + + it('knows collector.consentDenied and rejects an unknown collector key', () => { + // As read from a file: the typo is not a typed key. + const file: Flow.Json = JSON.parse( + JSON.stringify({ + version: 4, + flows: { + default: { + config: { platform: 'server' }, + collector: { consentDenied: 'skip', queueMaxx: 1 }, + }, + }, + }), + ); + expect(codes(file)).toEqual([ + ['UNKNOWN_KEY', 'flows.default.collector.queueMaxx'], + ]); + }); + + it('accepts every known config key', () => { + expect( + validateFlowStructure( + flow({ + default: { + config: { platform: 'web' }, + destinations: { + d: { + code, + config: { + consent: { marketing: true }, + settings: { any: 'thing' }, + queueMax: 5, + batch: { size: 10 }, + }, + }, + }, + }, + }), + ).errors, + ).toEqual( + [['flows.default.destinations.d', 'BATCH_WITHOUT_PUSH_BATCH']].map( + ([path, code]) => expect.objectContaining({ path, code }), + ), + ); + }); + + it('rejects a batch on inline code without pushBatch (E3)', () => { + const result = validateFlowStructure( + flow({ + default: { + config: { platform: 'web' }, + destinations: { + d: { + code, + config: { + batch: 10, + mapping: { page: { view: [{ batch: 5 }] } }, + }, + }, + batched: { + code: { ...code, pushBatch: '$code:(batch) => {}' }, + config: { batch: 10 }, + }, + }, + }, + }), + ); + expect(result.errors).toEqual([ + { + path: 'flows.default.destinations.d', + message: + '`config.batch`, `config.mapping.page.view[0].batch` have no effect: the inline code has no `pushBatch`, so events go to `push` one by one. Add a `pushBatch` function to `code`, or remove the batch.', + code: 'BATCH_WITHOUT_PUSH_BATCH', + }, + ]); + }); + + it('rejects a mapping value with key and fn (E7)', () => { + const result = validateFlowStructure( + flow({ + default: { + config: { platform: 'web' }, + destinations: { + d: { + code, + config: { + mapping: { + page: { + view: { + data: { + map: { + title: { + key: 'data.title', + fn: '$code:(v) => String(v).toUpperCase()', + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }), + ); + expect(result.errors).toEqual([ + { + path: 'flows.default.destinations.d.config.mapping.page.view.data.map.title', + message: + "`fn` never runs: `key` is this value's producer (one producer per value, tried in the order loop, map, set, key, fn). Keep one producer; to transform a path, read it inside `fn`.", + code: 'MAPPING_VALUE_PRODUCERS', + }, + ]); + }); + + it('rejects a mapping object without an operator (E14), also in a source ingest', () => { + const result = validateFlowStructure( + flow({ + default: { + config: { platform: 'server' }, + sources: { + s: { code, config: { ingest: { url: 'url', ip: 'ip' } } }, + }, + destinations: { + rec: { + code, + config: { + data: { total: 'data.total', currency: 'data.currency' }, + policy: { 'data.id': { map: { id: 'data.id' } } }, + }, + }, + }, + }, + }), + ); + expect(result.errors).toEqual([ + expect.objectContaining({ + path: 'flows.default.sources.s.config.ingest', + code: 'MAPPING_VALUE_NO_OPERATOR', + }), + { + path: 'flows.default.destinations.rec.config.data', + message: + 'Mapping value has no operator (total, currency), so it is dropped at runtime. Wrap the fields in map ({ "map": { ... } }), or use key, value, fn, loop or set.', + code: 'MAPPING_VALUE_NO_OPERATOR', + }, + ]); + }); + + it('rejects mapping keys that do nothing at the source and destination position (E24, E25)', () => { + const result = validateFlowStructure( + flow({ + default: { + config: { platform: 'web' }, + sources: { + s: { + code, + config: { + mapping: { page: { view: { data: { map: { a: 'data.a' } } } } }, + }, + }, + }, + destinations: { + d: { + code, + config: { + mapping: { page: { view: { extend: { name: 'x' } } } }, + }, + }, + }, + transformers: { + t: { mapping: { mapping: { page: { view: { silent: true } } } } }, + }, + }, + }), + ); + expect(result.warnings).toEqual([]); + expect(result.errors).toEqual([ + { + path: 'flows.default.transformers.t', + message: + "`mapping.mapping.page.view.silent` does nothing at the transformer position; only `policy` and a rule's `condition`, `policy`, `name` and `ignore` apply.", + code: 'MAPPING_NO_OP', + }, + { + path: 'flows.default.sources.s', + message: + "`config.mapping.page.view.data` does nothing at the source position; only `consent`, `policy` and a rule's `condition`, `policy`, `name` and `ignore` apply; edit the event at a source with `policy`.", + code: 'MAPPING_NO_OP', + }, + { + path: 'flows.default.destinations.d', + message: + "`config.mapping.page.view.extend` does nothing at the destination position; a rule's `extend` applies only inside a package's own `settings.mapping` (today `@walkeros/transformer-ga4`), merged onto its shipped rules; write the fields on the rule itself.", + code: 'MAPPING_NO_OP', + }, + ]); + }); + + it.each([ + ['$flow. tracking-server.url', 'does not match the $flow.NAME grammar'], + ['$flow.1st.url', 'does not match the $flow.NAME grammar'], + ['https://$flow.web.url/e', 'uses $flow. inline'], + ['$store.cache.key', 'does not match the $store.NAME grammar'], + ['Bearer $secret.API_KEY', 'uses $secret. inline'], + ['see $contract.web', 'uses $contract. inline'], + ])('rejects the malformed reference %s (E26)', (url, text) => { + const result = validateFlowStructure( + flow({ + 'tracking-server': { + config: { platform: 'server', url: 'https://collect.example.com' }, + }, + web: { + config: { platform: 'web' }, + destinations: { api: { code, config: { settings: { url } } } }, + }, + }), + ); + expect(result.valid).toBe(false); + expect(result.errors).toContainEqual({ + path: 'flows.web.destinations.api.config.settings.url', + message: expect.stringContaining(text), + code: 'MALFORMED_REFERENCE', + }); + }); + + it('accepts a valid hyphenated $flow reference', () => { + expect( + validateFlowStructure( + flow({ + 'tracking-server': { + config: { platform: 'server', url: 'https://collect.example.com' }, + }, + web: { + config: { platform: 'web' }, + destinations: { + api: { + code, + config: { settings: { url: '$flow.tracking-server.url' } }, + }, + }, + }, + }), + ).errors, + ).toEqual([]); + }); +}); + describe('state store references', () => { const entry = (store?: string) => ({ mode: 'get' as const, diff --git a/packages/core/src/contract.ts b/packages/core/src/contract.ts index f2e393b58..6611132c9 100644 --- a/packages/core/src/contract.ts +++ b/packages/core/src/contract.ts @@ -78,6 +78,22 @@ const NAME_MAP_KEYWORDS = new Set([ 'definitions', ]); +/** + * The keys a named contract rule may carry, one list for the flow schema and + * validate; `config-drift.test-d.ts` fails the build when it and + * `Flow.ContractRule` drift apart. Sections such as `globals` or `consent` + * are not among them: nothing enforces them, so a rule on those event parts + * goes into `schema`. A plain literal, so a bundle that does not use it drops + * it. + */ +export const CONTRACT_RULE_KEYS = [ + 'extend', + 'tagging', + 'description', + 'events', + 'schema', +] as const satisfies ReadonlyArray; + /** Options for {@link resolveContracts}. */ export interface ResolveContractsOptions { /** diff --git a/packages/core/src/dev.ts b/packages/core/src/dev.ts index 9aea9f475..c8fde19c1 100644 --- a/packages/core/src/dev.ts +++ b/packages/core/src/dev.ts @@ -8,7 +8,7 @@ export { expectSimulationResolves } from './expectSimulationResolves'; export * as routeCases from './examples/route-cases'; export type { Flow } from './types/flow'; -export { validateFlowStructure } from './validate-structure'; +export { checkStepKeys, validateFlowStructure } from './validate-structure'; export type { ValidateResult, ValidateResultType, diff --git a/packages/core/src/flow.ts b/packages/core/src/flow.ts index 4cf766c33..07425cb9b 100644 --- a/packages/core/src/flow.ts +++ b/packages/core/src/flow.ts @@ -323,6 +323,13 @@ function resolvePatterns( } } else { resolved = walkPath(resolved, path, `$flow.${flowName}`); + // Empty is unresolved, as in soft mode: inlining "" would ship a + // value the target flow never set. + if (resolved === undefined || resolved === null || resolved === '') { + throwError( + formatUnresolvedFlowMessage(flowName, path, 'missing-key'), + ); + } } } diff --git a/packages/core/src/mapping-no-ops.ts b/packages/core/src/mapping-no-ops.ts index 130411ae0..a544ecc5a 100644 --- a/packages/core/src/mapping-no-ops.ts +++ b/packages/core/src/mapping-no-ops.ts @@ -52,14 +52,14 @@ export const MAPPING_NO_OPS: Record< }, }; -/** What applies at a position, for the message that names a no-op. */ +/** What applies at a position, and the fix, for the message that names a no-op. */ const APPLIES: Record = { transformer: "only `policy` and a rule's `condition`, `policy`, `name` and `ignore` apply", source: - "only `consent`, `policy` and a rule's `condition`, `policy`, `name` and `ignore` apply", + "only `consent`, `policy` and a rule's `condition`, `policy`, `name` and `ignore` apply; edit the event at a source with `policy`", destination: - "a rule's `extend` is merged only by a package onto its own shipped rules; write the fields on the rule itself", + "a rule's `extend` applies only inside a package's own `settings.mapping` (today `@walkeros/transformer-ga4`), merged onto its shipped rules; write the fields on the rule itself", }; /** @@ -70,9 +70,25 @@ export function getMappingNoOpPaths( position: MappingPosition, mapping: unknown, field: string, +): string[] { + return getMappingKeyPaths(mapping, field, MAPPING_NO_OPS[position]); +} + +/** + * Dotted paths of the `keys` set in a step config, rooted at `field`: + * config-level keys (any config key, `batch` included), then each mapping + * rule's keys. Rules in array form are indexed. + */ +export function getMappingKeyPaths( + mapping: unknown, + field: string, + keys: { + config: ReadonlyArray; + rule: ReadonlyArray; + }, ): string[] { if (!isObject(mapping)) return []; - const { config, rule: ruleKeys } = MAPPING_NO_OPS[position]; + const { config, rule: ruleKeys } = keys; const paths: string[] = []; for (const key of config) { if (mapping[key] !== undefined) paths.push(`${field}.${key}`); diff --git a/packages/core/src/mappingValue.ts b/packages/core/src/mappingValue.ts index 35d873026..3acbeebd8 100644 --- a/packages/core/src/mappingValue.ts +++ b/packages/core/src/mappingValue.ts @@ -18,7 +18,7 @@ export interface ValueTrace { } /** The keys a value config is made of; an object with none of them has no operator. */ -const VALUE_KEYS = [ +export const VALUE_KEYS = [ 'condition', 'consent', 'fn', @@ -30,6 +30,18 @@ const VALUE_KEYS = [ 'value', ] as const satisfies ReadonlyArray; +/** + * The producers of a value, in the order `processMappingValue` tries them: + * the first one set runs, the rest never do. + */ +export const VALUE_PRODUCERS = [ + 'loop', + 'map', + 'set', + 'key', + 'fn', +] as const satisfies ReadonlyArray; + /** * Report a value dropped by throwing user code or a malformed config: one * line per cause through the collector's cause log, with the event's name @@ -172,8 +184,9 @@ async function processMappingValue( let mappingValue: unknown = isDefined(staticValue) ? staticValue : value; - // One producer per value, in this order; the rest never run. A producer - // that yields nothing leaves the result to the `value` fallback. + // One producer per value, in the order of VALUE_PRODUCERS; the rest + // never run. A producer that yields nothing leaves the result to the + // `value` fallback. if (loop) { const [scope, itemMapping] = loop; const data = diff --git a/packages/core/src/node.ts b/packages/core/src/node.ts index c86f86610..dd9131d9d 100644 --- a/packages/core/src/node.ts +++ b/packages/core/src/node.ts @@ -15,12 +15,13 @@ import os from 'os'; import path from 'path'; import { createLogger } from './logger'; import { maskKnownValues, scrubSecrets } from './redactLine'; +import { maskKnownNumbers } from './scrubJson'; export { scrubSecrets, redactLine } from './redactLine'; export type { ScrubOptions } from './redactLine'; export { scrubJson } from './scrubJson'; export { toPrintable } from './toPrintable'; -import type { Config, Instance } from './types/logger'; +import type { Config, Format, Instance, LogContext } from './types/logger'; import { Level } from './types/logger'; const DEFAULT_TMP_ROOT = os.tmpdir(); @@ -75,18 +76,52 @@ export function getDefaultTmpRoot(): string { return DEFAULT_TMP_ROOT; } +/** + * One log line as a structured entry: the shape a JSON line prints and the + * `onLine` tap receives, masked like the text line. `message` is + * `[scope] message` without the context; `scope` and `context` are omitted + * when empty; `context` is a string prefix of its JSON, with + * `contextTruncated`, when the line would pass {@link MAX_JSON_LINE_BYTES}. + */ +export interface LogEntry { + severity: 'ERROR' | 'WARNING' | 'INFO' | 'DEBUG'; + message: string; + scope?: string; + context?: Record | string; + contextTruncated?: true; + time: string; +} + +/** + * Size cap of one JSON line, far below the 256 KiB a log platform accepts + * per entry: a longer entry may be dropped whole rather than cut. + */ +export const MAX_JSON_LINE_BYTES = 32 * 1024; + export interface CLILoggerOptions { verbose?: boolean; silent?: boolean; json?: boolean; stderr?: boolean; - onLine?: (level: Level, message: string) => void; + /** + * Taps every line before any visibility gate: the masked text line, and + * the same line as a structured entry (scope, and the context with fields + * such as code, status, retryable and suppressed) for forwarding. + */ + onLine?: (level: Level, message: string, entry: LogEntry) => void; /** * Exact secret values masked in every line (see `scrubSecrets`). An array is * copied at creation; a function is read per line, so values learned after * the logger exists (secrets fetched at startup) are masked too. */ knownSecrets?: readonly string[] | (() => readonly string[]); + /** + * Output format (`text` by default): `json` prints one JSON object per + * line, ERROR and WARN to stderr, INFO and DEBUG to stdout, without + * colour. A function is read per line, so a host can switch to the + * format a flow sets once it knows it. `json` (machine output) wins. + */ + format?: Format | (() => Format | undefined); } /** Formats one already-scrubbed line for the terminal. */ @@ -128,6 +163,70 @@ function labelTmpRoot(line: string, root: string): string { return result; } +const SEVERITY: Record = { + [Level.ERROR]: 'ERROR', + [Level.WARN]: 'WARNING', + [Level.INFO]: 'INFO', + [Level.DEBUG]: 'DEBUG', +}; + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +/** + * The structured entry of one line and its JSON text, masked in the text + * line's order: numbers that print a known value become `'***'` before + * serializing (masking inside serialized JSON would break it), then known + * values, the temp root label and the pattern rules, applied to the message + * and to the serialized context. A context that cannot serialize, or whose + * masked form no longer parses, becomes `"[unserializable context]"`. A + * line above {@link MAX_JSON_LINE_BYTES} keeps a prefix of its masked + * context JSON, so the cut never splits a secret. + */ +function buildLogEntry( + level: Level, + message: string, + context: LogContext, + scope: string[], + mask: (text: string) => string, + known: readonly string[] | undefined, +): { entry: LogEntry; line: string } { + const scopeName = scope.join(':'); + const entry: LogEntry = { + severity: SEVERITY[level], + message: mask(`${scopeName ? `[${scopeName}] ` : ''}${message}`), + time: new Date().toISOString(), + }; + if (scopeName) entry.scope = scopeName; + + let contextJson: string | undefined; + if (Object.keys(context).length > 0) { + try { + contextJson = mask( + JSON.stringify(maskKnownNumbers(context, known ?? [])), + ); + const parsed: unknown = JSON.parse(contextJson); + entry.context = isRecord(parsed) ? parsed : '[unserializable context]'; + } catch { + contextJson = undefined; + entry.context = '[unserializable context]'; + } + } + + let line = JSON.stringify(entry); + if (contextJson && Buffer.byteLength(line) > MAX_JSON_LINE_BYTES) { + entry.contextTruncated = true; + let size = MAX_JSON_LINE_BYTES - Buffer.byteLength(entry.message) - 256; + do { + entry.context = contextJson.slice(0, Math.max(size, 0)); + line = JSON.stringify(entry); + size = Math.floor(size / 2); + } while (Buffer.byteLength(line) > MAX_JSON_LINE_BYTES && size > 0); + } + return { entry, line }; +} + /** * Build the `Logger.Config` (level + handler + jsonHandler) that backs a * terminal logger. Returned separately so the runtime can hand the SAME @@ -156,6 +255,7 @@ export function createCLILoggerConfig( json = false, stderr = false, knownSecrets, + format, } = options; const knownCopy = typeof knownSecrets === 'function' @@ -172,6 +272,14 @@ export function createCLILoggerConfig( const infoColor = colors.info ?? identity; const debugColor = colors.debug ?? identity; + const readFormat = (): Format => + (typeof format === 'function' ? format() : format) ?? 'text'; + // The text line's masking order, shared by the JSON entry: known values + // first, so the label can never split one; then the temp root label; then + // the pattern rules. + const mask = (text: string): string => + scrubSecrets(labelTmpRoot(maskKnownValues(text, readKnown()), tmpRoot)); + return { // Let handler control visibility: pass everything through. With the gate // at DEBUG, ERROR always reaches the handler (and the ring) even without @@ -206,41 +314,47 @@ export function createCLILoggerConfig( // edge); then the temp root label, since a long per-user root (macOS // `/var/folders/...`) would otherwise read as a token; then the // pattern rules. - const fullMessage = scrubSecrets( - labelTmpRoot( - maskKnownValues(`${scopePath}${message}${meta}`, readKnown()), - tmpRoot, - ), + const fullMessage = mask(`${scopePath}${message}${meta}`); + const { entry, line } = buildLogEntry( + level, + message, + context, + scope, + mask, + readKnown(), ); // Tap every line before any early return so no level is dropped from capture. try { - options.onLine?.(level, fullMessage); + options.onLine?.(level, fullMessage, entry); } catch { // Swallow: a throwing consumer must never break logging (and we must // not call logger methods here to avoid infinite recursion). } - // ERROR: always shown unless json mode - if (level === Level.ERROR) { - if (!json) console.error(errorColor(fullMessage)); - return; - } - - // Non-errors suppressed in silent or json mode - if (silent || json) return; + // Machine output (`json`) prints only its result; silent keeps errors; + // DEBUG needs verbose. + if (json) return; + if (level !== Level.ERROR && silent) return; + if (level === Level.DEBUG && !verbose) return; - // DEBUG: only with verbose - if (level === Level.DEBUG) { - if (!verbose) return; - out(` ${debugColor(fullMessage)}`); + // JSON lines: one object per line, no colour; ERROR and WARN on stderr + // (where platforms that ship only stderr still see them), INFO and + // DEBUG on stdout. + if (readFormat() === 'json') { + if (level === Level.ERROR || level === Level.WARN) console.error(line); + else out(line); return; } - // WARN / INFO: normal output - out( - level === Level.WARN ? warnColor(fullMessage) : infoColor(fullMessage), - ); + if (level === Level.ERROR) console.error(errorColor(fullMessage)); + else if (level === Level.DEBUG) out(` ${debugColor(fullMessage)}`); + else + out( + level === Level.WARN + ? warnColor(fullMessage) + : infoColor(fullMessage), + ); }, jsonHandler: (data) => { if (!silent) out(JSON.stringify(data, null, 2)); diff --git a/packages/core/src/schemas/__tests__/config-drift.test-d.ts b/packages/core/src/schemas/__tests__/config-drift.test-d.ts index d690d7fb0..c273be8e8 100644 --- a/packages/core/src/schemas/__tests__/config-drift.test-d.ts +++ b/packages/core/src/schemas/__tests__/config-drift.test-d.ts @@ -30,6 +30,7 @@ import type * as TransformerTypes from '../../types/transformer'; import type * as StoreTypes from '../../types/store'; import type * as CollectorTypes from '../../types/collector'; import type * as WalkerOSTypes from '../../types/walkeros'; +import type { Flow } from '../../types/flow'; import type { ConfigSchema as DestConfigSchema } from '../destination'; import type { ConfigSchema as SourceConfigSchema } from '../source'; @@ -40,6 +41,9 @@ import type { InitConfigSchema as CollectorInitConfigSchema, } from '../collector'; import type { SourceFieldsSchema, UserFieldsSchema } from '../walkeros'; +import type { CodeSchema, ContractRuleSchema } from '../flow'; +import type { STEP_FUNCTION_KEYS } from '../../step-entry'; +import type { CONTRACT_RULE_KEYS } from '../../contract'; import type { IsExactly, Expect } from './type-utils'; @@ -100,3 +104,27 @@ type _sourceEventCheck = Expect< type _UserEventZodKeys = keyof z.infer; type _UserEventTsKeys = keyof WalkerOSTypes.UserFields; type _userEventCheck = Expect>; + +// Flow.Code (inline code, closed): the step functions `STEP_FUNCTION_KEYS` +// lists plus `type`. +type _CodeZodKeys = keyof z.infer; +type _CodeTsKeys = keyof Flow.Code; +type _codeCheck = Expect>; + +// Flow.ContractRule (closed): the keys `CONTRACT_RULE_KEYS` lists. +type _ContractRuleZodKeys = keyof z.infer; +type _ContractRuleTsKeys = keyof Flow.ContractRule; +type _contractRuleCheck = Expect< + IsExactly<_ContractRuleZodKeys, _ContractRuleTsKeys> +>; + +// The two key lists name every key of their type, no more. +type _stepFunctionKeysCheck = Expect< + IsExactly< + (typeof STEP_FUNCTION_KEYS)[number], + Exclude + > +>; +type _contractRuleKeysCheck = Expect< + IsExactly<(typeof CONTRACT_RULE_KEYS)[number], keyof Flow.ContractRule> +>; diff --git a/packages/core/src/schemas/flow.ts b/packages/core/src/schemas/flow.ts index 5d013ebee..5d3f64111 100644 --- a/packages/core/src/schemas/flow.ts +++ b/packages/core/src/schemas/flow.ts @@ -150,8 +150,10 @@ export const BundleSchema = z * @remarks * Object-only inline code block. Enables custom sources, transformers, * destinations, and stores without npm packages. The `push` function is - * required; `type` and `init` are optional. To select a named export from - * a package, use the step's `import` field instead. + * required; `type` and the other step functions (`STEP_FUNCTION_KEYS`: + * `init`, `pushBatch`, `on`, `destroy`) are optional. Closed: any other key + * is an error, because the bundler emits only these. To select a named + * export from a package, use the step's `import` field instead. * * @example * ```json @@ -181,7 +183,26 @@ export const CodeSchema = z .describe( 'Optional initialization function. Use $code: prefix for inline JavaScript.', ), + pushBatch: z + .string() + .optional() + .describe( + 'Optional batch push function (destinations with config.batch). Use $code: prefix for inline JavaScript.', + ), + on: z + .string() + .optional() + .describe( + 'Optional event handler, called with the event type and context (consent, ready, run). Use $code: prefix for inline JavaScript.', + ), + destroy: z + .string() + .optional() + .describe( + 'Optional destroy function, called at shutdown. Use $code: prefix for inline JavaScript.', + ), }) + .strict() .meta({ id: 'FlowCode', title: 'Flow.Code', @@ -725,6 +746,10 @@ export const ContractRuleSchema = z events: ValidateEventsSchema.optional(), schema: JsonSchemaSchema.optional(), }) + // Closed (`CONTRACT_RULE_KEYS`): a section such as `globals` or `consent` + // would be accepted and never enforced, so it is an error; a rule on the + // whole event goes into `schema`. + .strict() .meta({ id: 'ContractRule', title: 'Flow.ContractRule', diff --git a/packages/core/src/schemas/logger.ts b/packages/core/src/schemas/logger.ts index bebfb6374..158cb0448 100644 --- a/packages/core/src/schemas/logger.ts +++ b/packages/core/src/schemas/logger.ts @@ -36,6 +36,12 @@ export const LoggerConfigSchema = z .union([z.number(), z.enum(['ERROR', 'WARN', 'INFO', 'DEBUG'])]) .optional() .describe('Minimum log level (default: ERROR)'), + format: z + .enum(['text', 'json']) + .optional() + .describe( + "Output format of a server flow's log lines: text (default) or json, one JSON object per line. Web flows reject it.", + ), handler: LoggerHandlerSchema.optional().describe( 'Custom log handler function', ), @@ -44,5 +50,5 @@ export const LoggerConfigSchema = z id: 'LoggerConfig', title: 'Logger.Config', description: - 'Logger configuration (level, handler) to override the collector defaults', + 'Logger configuration (level, format, handler) to override the collector defaults', }); diff --git a/packages/core/src/schemas/validate-flow-config.ts b/packages/core/src/schemas/validate-flow-config.ts index 5eb36a677..d75986083 100644 --- a/packages/core/src/schemas/validate-flow-config.ts +++ b/packages/core/src/schemas/validate-flow-config.ts @@ -269,7 +269,19 @@ function checkReferences( checkEnvReferences(text, strings, warnings); checkFlowReferences(text, strings, context, warnings); checkSecretReferences(text, parsed, context, errors, warnings); - checkWholeValueReferences(text, parsed, warnings); + for (const issue of findMalformedReferences(parsed)) { + const encoded = JSON.stringify(issue.value); + const idx = text.indexOf(encoded); + errors.push({ + message: issue.message, + severity: 'error', + path: issue.path, + code: MALFORMED_REFERENCE, + ...(idx === -1 + ? { line: 1, column: 1 } + : offsetToPosition(text, idx, encoded.length)), + }); + } } /** A string value that can carry references, with its place in the text. */ @@ -487,17 +499,28 @@ function isProsePath(path: readonly string[]): boolean { return path.length === 7 && path[4] === 'examples'; } +/** The code of a whole-value reference that would not resolve. */ +export const MALFORMED_REFERENCE = 'MALFORMED_REFERENCE'; + +/** A string value of a flow file that a check rejects, by its path. */ +export interface FlowValueIssue { + /** Dotted path from the config root. */ + path: string; + /** The string as written. */ + value: string; + message: string; +} + /** - * Warn on whole-value references that would not resolve: used inside a - * larger string (they ship as literal text) or malformed (the value starts - * with the prefix but fails the anchored grammar). Walks parsed string - * values; `$code:` payloads are code, not references, and are skipped. + * Whole-value references that would not resolve: used inside a larger + * string (they would ship as literal text) or malformed (the value starts + * with the prefix but fails the anchored grammar). One definition for + * `validateFlowConfig` (validate, the MCP `flow_validate`, the editor) and + * `validateFlowStructure` (the bundle preflight). Walks parsed string + * values; `$code:` payloads are code, not references, and prose is skipped. */ -function checkWholeValueReferences( - text: string, - parsed: unknown, - warnings: ValidationIssue[], -): void { +export function findMalformedReferences(parsed: unknown): FlowValueIssue[] { + const issues: FlowValueIssue[] = []; walkStringValues(parsed, [], (value, path) => { if (value.startsWith(REF_CODE_PREFIX)) return; if (isProsePath(path)) return; @@ -511,18 +534,10 @@ function checkWholeValueReferences( } else { message = `"${value}" uses ${prefix} inline. ${prefix} references resolve only as the whole value; it would ship as literal text. Use the reference as the entire value, or compose strings with $var or $env.`; } - const encoded = JSON.stringify(value); - const idx = text.indexOf(encoded); - warnings.push({ - message, - severity: 'warning', - path: path.join('.'), - ...(idx === -1 - ? { line: 1, column: 1 } - : offsetToPosition(text, idx, encoded.length)), - }); + issues.push({ path: path.join('.'), value, message }); } }); + return issues; } function walkStringValues( diff --git a/packages/core/src/schemas/validate.ts b/packages/core/src/schemas/validate.ts index 6a89a92d9..d6ba560da 100644 --- a/packages/core/src/schemas/validate.ts +++ b/packages/core/src/schemas/validate.ts @@ -4,6 +4,8 @@ export interface ValidationIssue { message: string; severity: 'error' | 'warning'; path?: string; + /** A stable code for checks shared with `validateFlowStructure`. */ + code?: string; line: number; column: number; endLine?: number; diff --git a/packages/core/src/step-entry.ts b/packages/core/src/step-entry.ts index 26bc5ebeb..f227e3ce8 100644 --- a/packages/core/src/step-entry.ts +++ b/packages/core/src/step-entry.ts @@ -174,6 +174,22 @@ export const STEP_OPERATIVE_FIELDS: { Store: ['code', 'package', 'import', 'cache'], }; +export type StepFunctionKey = Exclude; + +/** + * The functions inline `code` defines, each a `$code:` string. One list for + * `Flow.Code`, the flow schema's `CodeSchema`, the bundler's emitter and + * validate. `config-drift.test-d.ts` fails the build when it and `Flow.Code` + * drift apart. A plain literal, so a bundle that does not use it drops it. + */ +export const STEP_FUNCTION_KEYS = [ + 'push', + 'pushBatch', + 'init', + 'on', + 'destroy', +] as const satisfies ReadonlyArray; + /** * Keys accepted on a step entry that are not declared Flow step fields. They * are runtime registration keys (`Collector` init shapes) that @@ -251,7 +267,7 @@ export function validateStepEntry( ok: false, code: 'INVALID_CODE_SHAPE', key: 'code', - reason: `code must be an object ({ push, type?, init? }) or a resolved function value.`, + reason: `code must be an object ({ push, type?, init?, pushBatch?, on?, destroy? }) or a resolved function value.`, }; } @@ -296,6 +312,18 @@ export function validateStepEntry( return { ok: true }; } +/** + * Every top-level key of a step entry the runtime does not know, in entry + * order: the keys {@link validateStepEntry} rejects as `UNKNOWN_KEY`. + */ +export function getUnknownStepKeys( + entry: Record, + kind: Flow.StepKind, +): string[] { + const allowed = allowedKeys(kind); + return Object.keys(entry).filter((key) => !allowed.has(key)); +} + export function isPathStepEntry( entry: Record, kind: Flow.StepKind, diff --git a/packages/core/src/types/flow.ts b/packages/core/src/types/flow.ts index 55b23c8b6..50b379835 100644 --- a/packages/core/src/types/flow.ts +++ b/packages/core/src/types/flow.ts @@ -367,6 +367,12 @@ export namespace Flow { type?: string; /** Optional "$code:..." init function. */ init?: string; + /** Optional "$code:..." batch push function (destinations). */ + pushBatch?: string; + /** Optional "$code:..." event handler (`on('consent')` and others). */ + on?: string; + /** Optional "$code:..." destroy function. */ + destroy?: string; } /** @@ -392,7 +398,7 @@ export namespace Flow { /** * Inline code definition (object form only). * - * Object: `{push, type?, init?}` for inline code definition. + * Object: `{push, type?, init?, pushBatch?, on?, destroy?}` for inline code definition. * For named-export selection from a package, use the `import` field with `package`. */ code?: Code; @@ -487,7 +493,7 @@ export namespace Flow { /** * Inline code definition (object form only). * - * Object: `{push, type?, init?}` for inline code definition. + * Object: `{push, type?, init?, pushBatch?, on?, destroy?}` for inline code definition. * For named-export selection from a package, use the `import` field with `package`. */ code?: Code; @@ -562,7 +568,7 @@ export namespace Flow { /** * Inline code definition (object form only). * - * Object: `{push, type?, init?}` for inline code definition. + * Object: `{push, type?, init?, pushBatch?, on?, destroy?}` for inline code definition. * For named-export selection from a package, use the `import` field with `package`. */ code?: Code; @@ -643,7 +649,7 @@ export namespace Flow { /** * Inline code definition (object form only). * - * Object: `{push, type?, init?}` for inline code definition. + * Object: `{push, type?, init?, pushBatch?, on?, destroy?}` for inline code definition. * For named-export selection from a package, use the `import` field with `package`. */ code?: Code; diff --git a/packages/core/src/types/logger.ts b/packages/core/src/types/logger.ts index 6b263bb31..ab3506a8f 100644 --- a/packages/core/src/types/logger.ts +++ b/packages/core/src/types/logger.ts @@ -108,6 +108,13 @@ export interface Instance { scope: (name: string) => Instance; } +/** + * Output format of a server terminal logger: one text line per call, or one + * JSON object per line (`severity`, `message`, `scope`, `context`, `time`), + * which Cloud Run, Loki and Datadog read as structured entries. + */ +export type Format = 'text' | 'json'; + /** * Logger configuration options */ @@ -118,6 +125,14 @@ export interface Config { */ level?: Level | keyof typeof Level; + /** + * Output format of a server flow's log lines (`@walkeros/core/node`'s + * terminal logger, used by `runneros`). Web flows log to the browser + * console and reject it. + * @default 'text' + */ + format?: Format; + /** * Custom log handler function * Receives originalHandler to preserve default behavior diff --git a/packages/core/src/validate-structure.ts b/packages/core/src/validate-structure.ts index a0d683cc2..a2935929e 100644 --- a/packages/core/src/validate-structure.ts +++ b/packages/core/src/validate-structure.ts @@ -1,7 +1,25 @@ import type { Flow } from './types/flow'; -import { isObject } from './is'; -import { validateStepEntry } from './step-entry'; -import { getTransformerMappingWarnings } from './transformer-mapping'; +import { isArray, isObject } from './is'; +import { getUnknownStepKeys, validateStepEntry } from './step-entry'; +import { + getTransformerMapping, + getTransformerMappingWarnings, +} from './transformer-mapping'; +import { + getMappingKeyPaths, + getMappingNoOpMessage, + getMappingNoOpPaths, +} from './mapping-no-ops'; +import { VALUE_KEYS, VALUE_PRODUCERS } from './mappingValue'; +import { + findMalformedReferences, + MALFORMED_REFERENCE, +} from './schemas/validate-flow-config'; +import { ConfigSchema as SourceConfigSchema } from './schemas/source'; +import { ConfigSchema as TransformerConfigSchema } from './schemas/transformer'; +import { ConfigSchema as DestinationConfigSchema } from './schemas/destination'; +import { ConfigSchema as StoreConfigSchema } from './schemas/store'; +import { InitConfigSchema as CollectorInitConfigSchema } from './schemas/collector'; import { validateComponentNames, validateReference, @@ -77,6 +95,188 @@ function messageOf(error: unknown): string { const STATE_SECTIONS = ['sources', 'transformers', 'destinations'] as const; +/** + * The keys a step's `config` may carry, per kind, read from core's config + * schemas (kept equal to the config types by `config-drift.test-d.ts`). + * Collector-owned: the runtime reads them by name and ignores any other. + */ +const CONFIG_KEYS: Record> = { + Source: Object.keys(SourceConfigSchema.shape), + Transformer: Object.keys(TransformerConfigSchema.shape), + Destination: Object.keys(DestinationConfigSchema.shape), + Store: Object.keys(StoreConfigSchema.shape), +}; + +/** The keys a flow's `collector` block may carry (`Collector.InitConfig`). */ +const COLLECTOR_KEYS: ReadonlyArray = Object.keys( + CollectorInitConfigSchema.shape, +); + +function unknownKeyErrors( + value: unknown, + known: ReadonlyArray, + path: string, + owner: string, +): ValidationError[] { + if (!isObject(value)) return []; + return Object.keys(value) + .filter((key) => !known.includes(key)) + .map((key) => ({ + path: `${path}.${key}`, + message: `Unknown key "${key}" in ${owner} is ignored by the runtime. Known keys: ${[...known].sort().join(', ')}.`, + code: 'UNKNOWN_KEY', + })); +} + +/** + * Unknown keys on a step entry and inside its `config`: both are owned by + * the collector, which reads them by name, so a typo (`consentt`) would + * silently have no effect. Step-level keys are checked here for + * destinations and stores; a source is closed by the flow schema and a + * transformer by {@link validateStepEntry}. `at` is the step's path. + */ +export function checkStepKeys( + step: Record, + kind: Flow.StepKind, + at: string, +): ValidationError[] { + const errors: ValidationError[] = []; + if (kind === 'Destination' || kind === 'Store') { + for (const key of getUnknownStepKeys(step, kind)) { + errors.push({ + path: `${at}.${key}`, + message: `Unknown key "${key}" on ${kind} is ignored by the runtime. Remove the key, or move it to where it belongs.`, + code: 'UNKNOWN_KEY', + }); + } + } + errors.push( + ...unknownKeyErrors( + step.config, + CONFIG_KEYS[kind], + `${at}.config`, + `${kind} config`, + ), + ); + return errors; +} + +/** + * Mapping value objects that do not do what they look like: an object + * without an operator (`{ total: 'data.total' }` where `map` was meant) is + * dropped at runtime, and of several producers only the first one runs. + * Walks every value position of a mapping config (`data`, `policy`, each + * rule's `data` and `policy`) and, for a source, `ingest`. + */ +function checkMappingValues( + config: unknown, + at: string, + errors: ValidationError[], +): void { + if (!isObject(config)) return; + checkValue(config.data, `${at}.data`, errors); + checkPolicy(config.policy, `${at}.policy`, errors); + const rules = config.mapping; + if (!isObject(rules)) return; + for (const [entity, actions] of Object.entries(rules)) { + if (!isObject(actions)) continue; + for (const [action, value] of Object.entries(actions)) { + const list: unknown[] = isArray(value) ? value : [value]; + list.forEach((rule, index) => { + if (!isObject(rule)) return; + const rulePath = `${at}.mapping.${entity}.${action}${ + isArray(value) ? `.${index}` : '' + }`; + checkValue(rule.data, `${rulePath}.data`, errors); + checkPolicy(rule.policy, `${rulePath}.policy`, errors); + }); + } + } +} + +function checkPolicy( + policy: unknown, + at: string, + errors: ValidationError[], +): void { + if (!isObject(policy)) return; + for (const [key, value] of Object.entries(policy)) + checkValue(value, `${at}.${key}`, errors); +} + +const PRODUCER_ORDER = VALUE_PRODUCERS.join(', '); + +function checkValue( + value: unknown, + at: string, + errors: ValidationError[], +): void { + if (isArray(value)) { + value.forEach((item, index) => checkValue(item, `${at}.${index}`, errors)); + return; + } + if (!isObject(value)) return; + const keys = Object.keys(value); + if (keys.length === 0) return; + if (!VALUE_KEYS.some((key) => key in value)) { + errors.push({ + path: at, + message: `Mapping value has no operator (${keys.join(', ')}), so it is dropped at runtime. Wrap the fields in map ({ "map": { ... } }), or use key, value, fn, loop or set.`, + code: 'MAPPING_VALUE_NO_OPERATOR', + }); + return; + } + const producers = VALUE_PRODUCERS.filter((key) => value[key] !== undefined); + if (producers.length > 1) { + const [runs, ...never] = producers; + errors.push({ + path: at, + message: `${never.map((key) => `\`${key}\``).join(', ')} never ${ + never.length === 1 ? 'runs' : 'run' + }: \`${runs}\` is this value's producer (one producer per value, tried in the order ${PRODUCER_ORDER}). Keep one producer; to transform a path, read it inside \`fn\`.`, + code: 'MAPPING_VALUE_PRODUCERS', + }); + } + if (isObject(value.map)) + for (const [key, item] of Object.entries(value.map)) + checkValue(item, `${at}.map.${key}`, errors); + if (isArray(value.set)) + value.set.forEach((item, index) => + checkValue(item, `${at}.set.${index}`, errors), + ); + if (isArray(value.loop)) + value.loop.forEach((item, index) => + checkValue(item, `${at}.loop.${index}`, errors), + ); +} + +/** + * A `batch` on a destination whose inline `code` has no `pushBatch` has no + * effect: events go to `push` one by one. Checked where the implementation + * is known statically (inline code); a package step is checked at + * registration. + */ +function checkInlineBatch( + destination: Flow.Destination, + at: string, + errors: ValidationError[], +): void { + const { code } = destination; + if (!isObject(code) || code.pushBatch !== undefined) return; + const paths = getMappingKeyPaths(destination.config, 'config', { + config: ['batch'], + rule: ['batch'], + }); + if (paths.length === 0) return; + errors.push({ + path: at, + message: `${paths.map((path) => `\`${path}\``).join(', ')} ${ + paths.length === 1 ? 'has' : 'have' + } no effect: the inline code has no \`pushBatch\`, so events go to \`push\` one by one. Add a \`pushBatch\` function to \`code\`, or remove the batch.`, + code: 'BATCH_WITHOUT_PUSH_BATCH', + }); +} + /** * Every `state` entry (on the step or in its `config`) must name a store * declared in `flow.stores`, and that store may not be a `file: true` byte @@ -147,7 +347,14 @@ export function validateStateStores( * - each source/destination/store reference specifies exactly one of * package or code, and each transformer entry is a closed, valid shape, * - every `$store.` reference points at a defined store, - * - every `state` store is declared and is not a `file: true` store. + * - every `state` store is declared and is not a `file: true` store, + * - no unknown key on a step entry, in a step's `config` or in the + * `collector` block (`UNKNOWN_KEY`), + * - no mapping field that does nothing at its position (`MAPPING_NO_OP`), + * - no mapping value object without an operator or with two producers, + * - no `batch` on inline code without `pushBatch`, + * - no whole-value reference that would ship as literal text + * (`MALFORMED_REFERENCE`). * * Intended for fast deploy preflight: catch a bad flow config in well under a * second without spinning up a container or compiling anything. @@ -230,18 +437,65 @@ export function validateFlowStructure(flowConfig: Flow.Json): ValidateResult { code: result.code, }); } - // A mapping that never runs, or fields in it that do nothing at - // the transformer position, warn: rejecting would skip the step. + // A mapping that never runs, or fields in it that do nothing at the + // transformer position. The runtime still runs the step and logs it + // once at registration; validate and the bundle reject it. const { config } = transformer; - for (const message of getTransformerMappingWarnings({ + const entry = { ...transformer, config: isObject(config) ? config : undefined, - })) { - warnings.push({ path, message, code: 'TRANSFORMER_MAPPING_NO_OP' }); + }; + for (const message of getTransformerMappingWarnings(entry)) { + errors.push({ path, message, code: 'MAPPING_NO_OP' }); } + const mapping = getTransformerMapping(entry); + const field = + entry.config?.mapping !== undefined ? 'config.mapping' : 'mapping'; + checkMappingValues(mapping, `${path}.${field}`, errors); } } + // 2c. Closed keys on every step and its config, the collector block, + // mapping fields that do nothing at their position (a transformer's + // are checked in 2b), mapping value objects, and a batch the inline + // code cannot honour. + for (const section of STEP_SECTIONS) { + const steps = getSection(flow, section); + if (!steps) continue; + const kind = SECTION_TO_KIND[section]; + for (const [name, step] of Object.entries(steps)) { + const path = `flows.${flowName}.${section}.${name}`; + errors.push(...checkStepKeys({ ...step }, kind, path)); + if (kind !== 'Source' && kind !== 'Destination') continue; + const position = kind === 'Source' ? 'source' : 'destination'; + const noOps = getMappingNoOpPaths(position, step.config, 'config'); + if (noOps.length > 0) + errors.push({ + path, + message: getMappingNoOpMessage(position, noOps), + code: 'MAPPING_NO_OP', + }); + checkMappingValues(step.config, `${path}.config`, errors); + if (isObject(step.config)) + checkValue(step.config.ingest, `${path}.config.ingest`, errors); + } + } + for (const [name, destination] of Object.entries(flow.destinations || {})) { + checkInlineBatch( + destination, + `flows.${flowName}.destinations.${name}`, + errors, + ); + } + errors.push( + ...unknownKeyErrors( + flow.collector, + COLLECTOR_KEYS, + `flows.${flowName}.collector`, + 'collector config', + ), + ); + // 3. Every `$store.` reference must point at a defined store. const storeIds = new Set(Object.keys(flow.stores || {})); try { @@ -258,6 +512,16 @@ export function validateFlowStructure(flowConfig: Flow.Json): ValidateResult { errors.push(...validateStateStores(flow, flowName)); } + // 5. Whole-value references that would ship as literal text, the same + // check `validateFlowConfig` runs. + for (const issue of findMalformedReferences(flowConfig)) { + errors.push({ + path: issue.path, + message: issue.message, + code: MALFORMED_REFERENCE, + }); + } + return { valid: errors.length === 0, type: 'flow', diff --git a/packages/mcps/mcp/src/__tests__/resources/packages.test.ts b/packages/mcps/mcp/src/__tests__/resources/packages.test.ts new file mode 100644 index 000000000..b5993ca02 --- /dev/null +++ b/packages/mcps/mcp/src/__tests__/resources/packages.test.ts @@ -0,0 +1,90 @@ +import '../support/version.js'; + +import { clearCatalogCache } from '../../catalog.js'; +import { KNOWLEDGE_RESOURCES } from '../../specs.js'; +import { registerResourceSpec } from '../../server.js'; +import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; +import { Client } from '@modelcontextprotocol/sdk/client/index.js'; +import { InMemoryTransport } from '@modelcontextprotocol/sdk/inMemory.js'; +import { record, str } from '../support/tool-result.js'; + +/** + * H4: the packages resource answers what package_search's browse mode + * answers, `{ catalog, count, warnings? }`, so a failed npm search reads as + * "npm was unreachable", never as "no packages exist". + */ +async function readPackages(): Promise { + const server = new McpServer({ name: 'test', version: '0.0.0' }); + for (const spec of KNOWLEDGE_RESOURCES) registerResourceSpec(server, spec); + const [clientTransport, serverTransport] = + InMemoryTransport.createLinkedPair(); + const client = new Client({ name: 'test', version: '0' }); + await Promise.all([ + server.connect(serverTransport), + client.connect(clientTransport), + ]); + const result = await client.readResource({ + uri: 'walkeros://reference/packages', + }); + return JSON.parse(str(record(result.contents[0]).text)); +} + +describe('walkeros://reference/packages', () => { + const originalFetch = global.fetch; + + afterEach(() => { + global.fetch = originalFetch; + clearCatalogCache(); + }); + + it('answers an empty catalog with the npm failure as a warning', async () => { + const fetchMock: typeof fetch = async (input) => { + const url = typeof input === 'string' ? input : input.toString(); + if (url.startsWith('https://registry.npmjs.org/')) + throw new TypeError('fetch failed'); + return new Response('{}', { status: 200 }); + }; + global.fetch = fetchMock; + + expect(await readPackages()).toEqual({ + catalog: [], + count: 0, + warnings: [ + 'npm package search failed (fetch failed); the package catalog is empty. Try again, or use package_get with an exact package name.', + ], + }); + }); + + it('answers catalog and count without warnings when the search succeeds', async () => { + const fetchMock: typeof fetch = async (input) => { + const url = typeof input === 'string' ? input : input.toString(); + if (url.startsWith('https://registry.npmjs.org/')) + return new Response( + JSON.stringify({ + objects: [ + { + package: { + name: '@walkeros/web-destination-gtag', + version: '1.0.0', + description: 'GA4', + }, + }, + ], + }), + { status: 200 }, + ); + return new Response( + JSON.stringify({ $meta: { type: 'destination', platform: 'web' } }), + { status: 200 }, + ); + }; + global.fetch = fetchMock; + + const answer = record(await readPackages()); + expect(Object.keys(answer).sort()).toEqual(['catalog', 'count']); + expect(answer.count).toBe(1); + expect(answer.catalog).toEqual([ + expect.objectContaining({ name: '@walkeros/web-destination-gtag' }), + ]); + }); +}); diff --git a/packages/mcps/mcp/src/__tests__/resources/references-catalog.test.ts b/packages/mcps/mcp/src/__tests__/resources/references-catalog.test.ts index 5ad0d0534..dd8fcd30c 100644 --- a/packages/mcps/mcp/src/__tests__/resources/references-catalog.test.ts +++ b/packages/mcps/mcp/src/__tests__/resources/references-catalog.test.ts @@ -78,7 +78,9 @@ describe('packages reference resource', () => { uri: 'walkeros://reference/packages', }); const parsed: unknown = JSON.parse(firstText(result)); - expect(rows(parsed)[0].name).toBe('@walkeros/web-destination-gtag'); + expect(rows(record(parsed).catalog)[0].name).toBe( + '@walkeros/web-destination-gtag', + ); expect( fetchedUrls.every( (u) => diff --git a/packages/mcps/mcp/src/__tests__/tools/flow-schema.test.ts b/packages/mcps/mcp/src/__tests__/tools/flow-schema.test.ts index b1aea0b1d..7991d2fce 100644 --- a/packages/mcps/mcp/src/__tests__/tools/flow-schema.test.ts +++ b/packages/mcps/mcp/src/__tests__/tools/flow-schema.test.ts @@ -175,11 +175,10 @@ describe('reference resources', () => { }); it('packages resource returns catalog entries', async () => { - const parsed = await readReference('packages'); + const parsed = record(await readReference('packages')); - expect(Array.isArray(parsed)).toBe(true); - expect(parsed).toHaveLength(1); - expect(rows(parsed)[0]?.name).toBe('@walkeros/test-pkg'); + expect(parsed.count).toBe(1); + expect(rows(parsed.catalog)[0]?.name).toBe('@walkeros/test-pkg'); }); }); }); diff --git a/packages/mcps/mcp/src/__tests__/tools/validate-real.test.ts b/packages/mcps/mcp/src/__tests__/tools/validate-real.test.ts index bae4ec291..a51fea5c2 100644 --- a/packages/mcps/mcp/src/__tests__/tools/validate-real.test.ts +++ b/packages/mcps/mcp/src/__tests__/tools/validate-real.test.ts @@ -168,7 +168,7 @@ describe('flow_validate on the real validator', () => { }); it.each([ - ['f24-unknown-step-keys', 'warnings'], + ['f27-only-warning', 'warnings'], ['f17-only-skip', 'skips'], ])( 'C9: strict: true on %s (%s) is invalid, the verdict of CLI --strict', @@ -187,7 +187,7 @@ describe('flow_validate on the real validator', () => { ); it.each([ - ['f24-unknown-step-keys', 'warnings'], + ['f27-only-warning', 'warnings'], ['f17-only-skip', 'skips'], ])('C9 GUARD: without strict %s (%s) stays valid', async (fixture) => { const result = await callTool({ diff --git a/packages/mcps/mcp/src/instructions.ts b/packages/mcps/mcp/src/instructions.ts index e3bd9fc2e..c4f35e45b 100644 --- a/packages/mcps/mcp/src/instructions.ts +++ b/packages/mcps/mcp/src/instructions.ts @@ -58,7 +58,7 @@ Every component in a flow is a **step**: sources capture events, transformers pr - **Consent** gates destinations (\`config.consent\`) and individual fields (a value's \`consent\`). A mapping rule's \`consent\` is not enforced yet. Privacy-first by design. - **Routing** uses one grammar in every chain field (\`source.before\`, \`source.next\`, \`transformer.before\`, \`transformer.next\`, \`collector.next\`, \`destination.before\`, \`destination.next\`). A route is a string ID, a sequence (\`["a", "b"]\`), or a RouteConfig: \`{ match?, next }\`, \`{ match?, one: [...] }\` (first match wins), \`{ match?, many: [...] }\` (every match becomes its own copy of the event with its own id and finishes the rest of the path on its own, never merged), or \`{ match?, stop: true }\` (ends that copy). In a sequence, a member's own \`next\` runs right after it, then the sequence continues. \`match\` reads \`{ ingest, event }\` and each route is evaluated when the event reaches it. A \`stop\` in \`collector.next\` drops the event for every destination; per-destination filtering belongs in \`destination.before\`. An array of only route configs is an implicit \`one\`. The optional \`match\` field is omitted to mean always-match (no wildcard literal). - **Pass-through steps** are transformer entries with no \`code\` and no \`package\`. The runtime synthesizes the push for them automatically. Three flavors share this shape: a \`before\` / \`next\`-only step (named hop reusable across destinations), a \`cache\`-only step (dedup, rate-limit), and a \`mapping\`-only step (declarative event-to-event transform). A pass-through must declare at least one of \`before\`, \`next\`, \`cache\`, or \`mapping\`. -- **Mapping at the transformer position** uses the same \`Mapping.Config\` shape as destinations, but the semantic is event-to-event: only config-level \`policy\` and a rule's \`condition\`, \`policy\`, \`name\` (rename) and \`ignore\` (drop from chain) apply. Everything that shapes a vendor payload or gates a destination is a no-op there: config-level \`consent\`, \`include\`, \`data\`, and a rule's \`consent\`, \`include\`, \`remove\`, \`batch\`, \`settings\`, \`extend\`, \`data\`, \`silent\`; validation warns about them (\`TRANSFORMER_MAPPING_NO_OP\`). A \`config.mapping\` wins over a step-level \`mapping\`. +- **Mapping at the transformer position** uses the same \`Mapping.Config\` shape as destinations, but the semantic is event-to-event: only config-level \`policy\` and a rule's \`condition\`, \`policy\`, \`name\` (rename) and \`ignore\` (drop from chain) apply. Everything that shapes a vendor payload or gates a destination is a no-op there: config-level \`consent\`, \`include\`, \`data\`, and a rule's \`consent\`, \`include\`, \`remove\`, \`batch\`, \`settings\`, \`extend\`, \`data\`, \`silent\`. At a source only config-level \`consent\` and \`policy\` and a rule's \`condition\`, \`policy\`, \`name\` and \`ignore\` apply (edit the event there with \`policy\`); at a destination a rule's \`extend\` applies only inside a package's own \`settings.mapping\`. Validation and the bundle reject these keys (\`MAPPING_NO_OP\`). A \`config.mapping\` wins over a step-level \`mapping\`. - **Closed-schema rule on transformer entries:** known keys only. Operative keys are \`code\`, \`package\`, \`before\`, \`next\`, \`cache\`, \`mapping\`. Combined with structural keys (\`config\`, \`env\`, \`validate\`, \`disabled\`, \`id\`, etc.). Unknown top-level keys on a transformer step are errors (catches typos like \`dedup: {}\` instead of nested under \`cache\`). \`code\` + \`package\` together is a conflict. - **Cache** memoizes step output on a configured store. Set \`cache.stop: true\` to short-circuit the chain on HIT (sources default to stop-on-HIT); in a source chain or \`collector.next\` that halts the event for all destinations. Set \`cache.namespace\` to prefix keys; omit it to write keys directly. \`CacheRule.match\` is optional, omit for always-match. diff --git a/packages/mcps/mcp/src/resources/references.ts b/packages/mcps/mcp/src/resources/references.ts index ee71d09cb..d076ec309 100644 --- a/packages/mcps/mcp/src/resources/references.ts +++ b/packages/mcps/mcp/src/resources/references.ts @@ -159,11 +159,17 @@ export const REFERENCE_RESOURCES: readonly StaticResourceSpec[] = [ name: 'packages', uri: 'walkeros://reference/packages', description: - 'Complete walkerOS package catalog: all sources, destinations, transformers, and stores', + 'Complete walkerOS package catalog: all sources, destinations, transformers, and stores, as { catalog, count, warnings? }. A failed npm search answers an empty catalog with a warning that says so.', mimeType: JSON_MIME, read: async () => { - const { entries } = await fetchCatalog(); - return json(entries); + // The same fields package_search's browse mode answers: a failed npm + // search is a warning next to an empty catalog, never a silent []. + const { entries, warnings } = await fetchCatalog(); + return json({ + catalog: entries, + count: entries.length, + ...(warnings.length > 0 ? { warnings } : {}), + }); }, }, ]; diff --git a/packages/runner/Dockerfile b/packages/runner/Dockerfile index b4b97cc3f..36290cc52 100644 --- a/packages/runner/Dockerfile +++ b/packages/runner/Dockerfile @@ -43,7 +43,8 @@ ENV PORT=8080 ENV BUNDLE=/app/flow/flow.mjs # Sized for the 512 MB container: V8 fails with a heap-limit trace instead of # being OOM-killed silently. Revisit to 768 if the container grows to 1 GB. -ENV NODE_OPTIONS=--max-old-space-size=384 +# --no-warnings: the runner logs process warnings itself (WARN, flow format). +ENV NODE_OPTIONS="--max-old-space-size=384 --no-warnings" ENV PATH="/app/node_modules/.bin:${PATH}" EXPOSE 8080 diff --git a/packages/runner/src/__tests__/heartbeat.test.ts b/packages/runner/src/__tests__/heartbeat.test.ts index 437183b8b..f0a05ab94 100644 --- a/packages/runner/src/__tests__/heartbeat.test.ts +++ b/packages/runner/src/__tests__/heartbeat.test.ts @@ -151,6 +151,30 @@ describe('heartbeat', () => { ); }); + it.each([500, 429, 400])( + 'warns, without an auth hint, when a beat is rejected with %s', + async (status) => { + globalThis.fetch = jest.fn().mockResolvedValue({ status, ok: false }); + + const heartbeat = createHeartbeat( + { + appUrl: 'http://localhost:3000', + token: 'sk-walkeros-test', + projectId: 'proj_1', + intervalMs: 60000, + }, + mockLogger, + ); + + await heartbeat.sendOnce(); + + expect(mockLogger.warn).toHaveBeenCalledWith( + `Heartbeat rejected (${status}); its counters carry over to the next beat.`, + ); + expect(mockLogger.error).not.toHaveBeenCalled(); + }, + ); + it('logs debug on network error (no status)', async () => { globalThis.fetch = jest.fn().mockRejectedValue(new Error('ECONNREFUSED')); diff --git a/packages/runner/src/__tests__/pipeline.test.ts b/packages/runner/src/__tests__/pipeline.test.ts index 778f52793..cbe93fa5c 100644 --- a/packages/runner/src/__tests__/pipeline.test.ts +++ b/packages/runner/src/__tests__/pipeline.test.ts @@ -3,6 +3,7 @@ import { EventEmitter } from 'events'; import type { PipelineOptions } from '../pipeline.js'; import { createOutOfBandErrorTracker, + handleProcessWarning, handleUnhandledRejection, handleUncaughtException, registerProcessGuards, @@ -294,6 +295,45 @@ describe('process error guards', () => { }); }); + describe('JSON mode and process warnings', () => { + it('writes one fixed, pre-serialized line when a guard fails in JSON mode', () => { + const write = jest + .spyOn(process.stderr, 'write') + .mockImplementation(() => true); + const consoleError = jest + .spyOn(console, 'error') + .mockImplementation(() => {}); + try { + handleUncaughtException(new Error('stray throw'), { + logger: makeThrowingLogger(), + exit: jest.fn(), + format: () => 'json', + }); + expect(consoleError).not.toHaveBeenCalled(); + expect(write).toHaveBeenCalledWith( + '{"severity":"ERROR","message":"[runner] Process guard handler failed (continuing)"}\n', + ); + } finally { + write.mockRestore(); + consoleError.mockRestore(); + } + }); + + it('logs a Node process warning at WARN through the logger', () => { + const warn = jest.fn(); + const base = createLogger({ level: Level.DEBUG }); + handleProcessWarning( + Object.assign(new Error('Failed to access metadata server'), { + name: 'MetadataLookupWarning', + }), + { logger: { ...base, warn }, exit: jest.fn() }, + ); + expect(warn).toHaveBeenCalledWith( + 'MetadataLookupWarning: Failed to access metadata server', + ); + }); + }); + describe('handleUncaughtException', () => { it('logs the error and keeps the process serving for non-fatal cases', () => { const { logger, errors } = makeLogger(); @@ -342,6 +382,7 @@ describe('process error guards', () => { describe('registerProcessGuards', () => { let preexistingException: NodeJS.UncaughtExceptionListener[] = []; let preexistingRejection: NodeJS.UnhandledRejectionListener[] = []; + let preexistingWarning: NodeJS.WarningListener[] = []; beforeEach(() => { // Snapshot any guards the test harness/runner already attached, isolate @@ -349,6 +390,7 @@ describe('process error guards', () => { // test registers fresh. preexistingException = process.listeners('uncaughtException'); preexistingRejection = process.listeners('unhandledRejection'); + preexistingWarning = process.listeners('warning'); process.removeAllListeners('uncaughtException'); process.removeAllListeners('unhandledRejection'); resetProcessGuardsForTest(); @@ -357,6 +399,9 @@ describe('process error guards', () => { afterEach(() => { process.removeAllListeners('unhandledRejection'); process.removeAllListeners('uncaughtException'); + for (const listener of process.listeners('warning')) + if (!preexistingWarning.includes(listener)) + process.removeListener('warning', listener); for (const listener of preexistingException) process.on('uncaughtException', listener); for (const listener of preexistingRejection) diff --git a/packages/runner/src/__tests__/preview-gate.test.ts b/packages/runner/src/__tests__/preview-gate.test.ts index 79906eedd..6b01c9146 100644 --- a/packages/runner/src/__tests__/preview-gate.test.ts +++ b/packages/runner/src/__tests__/preview-gate.test.ts @@ -257,6 +257,15 @@ describe('preview intake gate', () => { }); expect(res.status).toBe(500); expect(res.body).not.toContain('Unauthorized'); + // A request that answered 500 is a failure: ERROR, not INFO. + expect(mockLogger.error).toHaveBeenCalledWith( + expect.stringContaining( + 'flow handler error on POST /collect: flow handler exploded', + ), + ); + expect(mockLogger.info).not.toHaveBeenCalledWith( + expect.stringContaining('flow handler error'), + ); }); it('REJECTS a validly-signed grant minted for a DIFFERENT session (sb binding)', async () => { diff --git a/packages/runner/src/__tests__/runtime-logging.test.ts b/packages/runner/src/__tests__/runtime-logging.test.ts new file mode 100644 index 000000000..3c418ee56 --- /dev/null +++ b/packages/runner/src/__tests__/runtime-logging.test.ts @@ -0,0 +1,162 @@ +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { Level } from '@walkeros/core'; +import type { LogEntry } from '@walkeros/core/node'; +import { createRuntimeLogging } from '../runtime-logging.js'; +import { loadFlow } from '../runner.js'; + +/** + * E17: the flow's own `logger: { level, format }` reaches the runner. The + * bundle reports it through the flow logger hook before its collector + * starts; the runner's lines and the collector's then share its format. + */ +describe('createRuntimeLogging', () => { + let errorSpy: jest.SpyInstance; + let logSpy: jest.SpyInstance; + + beforeEach(() => { + errorSpy = jest.spyOn(console, 'error').mockImplementation(() => {}); + logSpy = jest.spyOn(console, 'log').mockImplementation(() => {}); + }); + + afterEach(() => { + errorSpy.mockRestore(); + logSpy.mockRestore(); + }); + + const parse = (value: unknown): unknown => JSON.parse(String(value)); + + it("switches the runner's own lines to the flow's format once the bundle reports it", () => { + const logging = createRuntimeLogging({}); + logging.logger.info('Starting flow...'); + logging.flowLogger.onLogger({ level: 'WARN', format: 'json' }); + logging.logger.scope('runner').info('Flow running'); + + expect(logSpy.mock.calls[0][0]).toBe('Starting flow...'); + expect(parse(logSpy.mock.calls[1][0])).toMatchObject({ + severity: 'INFO', + message: '[runner] Flow running', + scope: 'runner', + }); + expect(logging.format()).toBe('json'); + }); + + it('keeps text when the flow names no format', () => { + const logging = createRuntimeLogging({}); + logging.flowLogger.onLogger({ level: 'WARN' }); + logging.logger.warn('plain'); + expect(logSpy).toHaveBeenCalledWith('plain'); + }); + + it('forces DEBUG over the flow level only with --verbose', () => { + expect(createRuntimeLogging({ verbose: true }).flowLogger.logLevel).toBe( + Level.DEBUG, + ); + expect(createRuntimeLogging({}).flowLogger).not.toHaveProperty('logLevel'); + }); + + it('hands the tap the structured fields of a collector line next to the text line', () => { + const taps: Array<[Level, string, LogEntry]> = []; + const { collectorConfig } = createRuntimeLogging({ + silent: true, + onLine: (level, message, entry) => taps.push([level, message, entry]), + }); + collectorConfig.handler?.( + Level.ERROR, + 'Push failed', + { code: 'transport', status: 503, retryable: true, suppressed: 3 }, + ['destination', 'api'], + () => {}, + ); + + expect(taps).toEqual([ + [ + Level.ERROR, + '[destination:api] Push failed {"code":"transport","status":503,"retryable":true,"suppressed":3}', + expect.objectContaining({ + severity: 'ERROR', + scope: 'destination:api', + context: { + code: 'transport', + status: 503, + retryable: true, + suppressed: 3, + }, + }), + ], + ]); + }); +}); + +describe('loadFlow hands the bundle the flow logger contract', () => { + const parse = (value: unknown): unknown => JSON.parse(String(value)); + const dirs: string[] = []; + let originalCwd: string; + let errorSpy: jest.SpyInstance; + let logSpy: jest.SpyInstance; + + beforeEach(() => { + originalCwd = process.cwd(); + errorSpy = jest.spyOn(console, 'error').mockImplementation(() => {}); + logSpy = jest.spyOn(console, 'log').mockImplementation(() => {}); + }); + + afterEach(() => { + errorSpy.mockRestore(); + logSpy.mockRestore(); + process.chdir(originalCwd); + for (const dir of dirs) fs.rmSync(dir, { recursive: true, force: true }); + dirs.length = 0; + }); + + it('reports the flow format before the collector logs, and the runner follows it', async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'runtime-logging-')); + dirs.push(dir); + const file = path.join(dir, 'bundle.mjs'); + // A bundle factory honouring the contract the CLI's server entry emits: + // report the flow's settings, then build the collector's logger from the + // host config with the flow's level. + fs.writeFileSync( + file, + `import { createLogger } from '@walkeros/core'; +export default async function(context = {}) { + const own = { level: 'WARN', format: 'json' }; + context.onLogger?.(own); + const logger = createLogger({ + ...context.logger, + level: context.logLevel ?? own.level, + }); + logger.scope('api').info('hidden by the flow level'); + logger.scope('api').warn('collector warning'); + return { collector: { push: async () => ({}), command: async () => {} } }; +} +`, + ); + + const logging = createRuntimeLogging({}); + await loadFlow( + file, + { port: 0, ...logging.flowLogger }, + logging.logger, + logging.collectorConfig, + ); + logging.logger.info('Flow running'); + + expect(errorSpy.mock.calls.map((call) => parse(call[0]))).toEqual([ + expect.objectContaining({ + severity: 'WARNING', + message: '[api] collector warning', + }), + ]); + expect(parse(logSpy.mock.calls.at(-1)?.[0])).toMatchObject({ + severity: 'INFO', + message: 'Flow running', + }); + expect( + logSpy.mock.calls.some((call) => + String(call[0]).includes('hidden by the flow level'), + ), + ).toBe(false); + }); +}); diff --git a/packages/runner/src/health-server.ts b/packages/runner/src/health-server.ts index 465077429..97ba71070 100644 --- a/packages/runner/src/health-server.ts +++ b/packages/runner/src/health-server.ts @@ -160,7 +160,8 @@ function guardPreviewIntake( try { onPass(); } catch (error) { - logger.info( + // The request failed (500): the user's intent is not met, so ERROR. + logger.error( `${PREVIEW_LOG} flow handler error on ${req.method} ${req.url}: ${ error instanceof Error ? error.message : String(error) }`, diff --git a/packages/runner/src/heartbeat.ts b/packages/runner/src/heartbeat.ts index 1778751d9..11cef36a0 100644 --- a/packages/runner/src/heartbeat.ts +++ b/packages/runner/src/heartbeat.ts @@ -214,6 +214,12 @@ export function createHeartbeat( logger.error( `Heartbeat auth failed (${response.status}). Token may have expired, redeploy to rotate.`, ); + } else if (!response.ok) { + // Nothing is lost (the deltas carry over), but the app did not take + // this beat, so the operator should know. + logger.warn( + `Heartbeat rejected (${response.status}); its counters carry over to the next beat.`, + ); } } catch (error) { // Deltas accumulate on failure, next successful send includes them diff --git a/packages/runner/src/index.ts b/packages/runner/src/index.ts index 00d19d6b1..023725828 100644 --- a/packages/runner/src/index.ts +++ b/packages/runner/src/index.ts @@ -15,6 +15,7 @@ export { type ShutdownDeps, } from './pipeline.js'; export { loadFlow, type RuntimeConfig, type FlowHandle } from './runner.js'; +export type { FlowLogger, FlowLoggerSettings } from './runtime-logging.js'; export { resolveBundle, isPrebuiltArtifact, diff --git a/packages/runner/src/pipeline.ts b/packages/runner/src/pipeline.ts index 8de48c730..efa974477 100644 --- a/packages/runner/src/pipeline.ts +++ b/packages/runner/src/pipeline.ts @@ -41,6 +41,7 @@ import { createTracePoller, type TracePollerHandle } from './trace-poller.js'; import { fetchSecrets } from './secrets-fetcher.js'; import { RunnerAuthError } from './runner-auth-error.js'; import type { ErrorRing, LogRing } from './log-ring.js'; +import type { FlowLogger } from './runtime-logging.js'; import { VERSION } from './version.js'; export interface PipelineOptions { @@ -55,6 +56,14 @@ export interface PipelineOptions { * the handler controls visibility; ERROR is always emitted into the ring. */ loggerConfig?: Logger.Config; + /** + * The flow logger contract handed to the bundle beside `loggerConfig`: a + * forced level (`--verbose`) and the hook that receives the flow's own + * logger settings, so the flow's level and format win. + */ + flowLogger?: FlowLogger; + /** The output format in effect, for lines written around the logger. */ + format?: () => Logger.Format; errorRing?: ErrorRing; logRing?: LogRing; /** @@ -87,7 +96,7 @@ export async function runPipeline(options: PipelineOptions): Promise { // emit('error', ...) on a detached tick) must land in the net rather than // crash the container before the guards are up. The degrade hook is wired in // once the health server exists; the listeners stay registered from here. - const guards = registerProcessGuards(logger); + const guards = registerProcessGuards(logger, undefined, options.format); // The artifact this container was deployed to run. Read before the secrets // are injected into the environment, so a secret cannot replace it. @@ -153,7 +162,7 @@ export async function runPipeline(options: PipelineOptions): Promise { : undefined; // Load flow - const runtimeConfig: RuntimeConfig = { port }; + const runtimeConfig: RuntimeConfig = { port, ...options.flowLogger }; let handle: FlowHandle; try { handle = await loadFlow( @@ -347,6 +356,11 @@ export async function runShutdown( export interface ProcessGuardDeps { logger: Logger.Instance; exit: (code: number) => void; + /** + * The output format in effect. In JSON mode a failed guard writes one + * fixed, pre-serialized line, so the stream stays one JSON object per line. + */ + format?: () => Logger.Format; /** * Fed by BOTH guard handlers on every out-of-band error so a windowed * counter can auto-degrade `/ready` after a sustained loop (and so a later @@ -356,6 +370,39 @@ export interface ProcessGuardDeps { onOutOfBandError?: () => void; } +/** + * The line a failed guard writes in JSON mode: fixed and pre-serialized, + * without the error text, so nothing in it can fail to serialize or carry a + * secret. + */ +const GUARD_FAILED_JSON_LINE = + '{"severity":"ERROR","message":"[runner] Process guard handler failed (continuing)"}\n'; + +/** + * The fallback when a guard's own logging throws: never a re-entrant + * `deps.logger`. Text mode prints the error with bare `console.error`. + */ +function reportGuardFailure(guardError: unknown, deps: ProcessGuardDeps): void { + if (deps.format?.() === 'json') process.stderr.write(GUARD_FAILED_JSON_LINE); + else console.error('Process guard handler failed (continuing):', guardError); +} + +/** + * Handle a Node process warning (`process.on('warning')`, for example a + * `MetadataLookupWarning`): log it at WARN through the logger, masked and + * in the flow's format. Never throws (it is a registered listener). + */ +export function handleProcessWarning( + warning: Error, + deps: ProcessGuardDeps, +): void { + try { + deps.logger.warn(`${warning.name}: ${warning.message}`); + } catch (guardError) { + reportGuardFailure(guardError, deps); + } +} + /** * Handle an `unhandledRejection`: log the reason into the error ring (via the * logger) and keep serving. A stray rejection is treated as non-fatal, the @@ -365,7 +412,7 @@ export interface ProcessGuardDeps { * throw: if the logger or the degrade chain throws synchronously, the escape * would itself become an uncaught error and crash the process, the exact * failure this guard exists to prevent. The whole body is therefore wrapped; - * the fallback uses bare `console.error` (never a re-entrant `deps.logger`). + * the fallback never re-enters `deps.logger` (see `reportGuardFailure`). */ export function handleUnhandledRejection( reason: unknown, @@ -379,7 +426,7 @@ export function handleUnhandledRejection( ); deps.onOutOfBandError?.(); } catch (guardError) { - console.error('Process guard handler failed (continuing):', guardError); + reportGuardFailure(guardError, deps); } } @@ -391,7 +438,7 @@ export function handleUnhandledRejection( * * This IS the registered `uncaughtException` listener, so the body must never * throw (see `handleUnhandledRejection`): a throw escaping here is fatal. The - * whole body is wrapped; the fallback uses bare `console.error`. + * whole body is wrapped; the fallback never re-enters `deps.logger`. */ export function handleUncaughtException( error: Error, @@ -401,7 +448,7 @@ export function handleUncaughtException( deps.logger.error(`Uncaught exception (continuing): ${error.message}`); deps.onOutOfBandError?.(); } catch (guardError) { - console.error('Process guard handler failed (continuing):', guardError); + reportGuardFailure(guardError, deps); } } @@ -443,6 +490,7 @@ const noopGuardHandle: ProcessGuardHandle = { export function registerProcessGuards( logger: Logger.Instance, onOutOfBandError?: () => void, + format?: () => Logger.Format, ): ProcessGuardHandle { if (processGuardsRegistered) { if (processGuardDeps && onOutOfBandError) @@ -455,6 +503,7 @@ export function registerProcessGuards( logger, exit: (code) => process.exit(code), onOutOfBandError, + format, }; processGuardDeps = deps; @@ -464,6 +513,7 @@ export function registerProcessGuards( process.on('uncaughtException', (error) => handleUncaughtException(error, deps), ); + process.on('warning', (warning) => handleProcessWarning(warning, deps)); return { setOnOutOfBandError(next) { diff --git a/packages/runner/src/run.ts b/packages/runner/src/run.ts index 37f9bf7d9..e87176bef 100644 --- a/packages/runner/src/run.ts +++ b/packages/runner/src/run.ts @@ -14,7 +14,6 @@ import { Level, createTimer, getErrorMessage } from '@walkeros/core'; import type { Logger } from '@walkeros/core'; import { createCLILogger, - createCLILoggerConfig, scrubSecrets, type CLILoggerColors, } from '@walkeros/core/node'; @@ -32,6 +31,7 @@ import { } from './resolve-bundle.js'; import { validatePort } from './validators.js'; import { runPipeline, type PipelineOptions } from './pipeline.js'; +import { createRuntimeLogging } from './runtime-logging.js'; import type { RunCommandOptions } from './types.js'; /** The runtime's terminal colours, injected into the shared logger factory. */ @@ -131,6 +131,10 @@ export async function runCommand(options: RunCommandOptions): Promise { [Level.DEBUG]: 'debug', } as const; + // Every line, the runner's and the collector's, is tapped here. The text + // line feeds the rings (heartbeat, error sink); the tap also receives the + // line as a structured entry (scope, and the context with code, status, + // retryable, suppressed) for a forwarder. const onLine = (level: Level, message: string) => { if (level === Level.ERROR) errorRing.add(message); logRing.add({ time: Date.now(), level: LEVEL_NAME[level], message }); @@ -142,21 +146,19 @@ export async function runCommand(options: RunCommandOptions): Promise { const knownSecrets: string[] = []; const readKnownSecrets = (): readonly string[] => knownSecrets; - const logger = createCLILogger( - { ...options, onLine, knownSecrets: readKnownSecrets }, - RUNTIME_COLORS, - ); - - // The deployed bundle's collector builds its own logger from this config - // (`context.logger`), so its destination errors flow through the SAME - // `onLine` ring tap as the runtime logger above. Without this, production - // (no --verbose) passes no `context.logger`, the collector's createLogger has - // no handler, and destination "Push failed" errors never reach the ErrorRing - // (the heartbeat would report "No errors reported" despite failed deliveries). - const collectorLoggerConfig = createCLILoggerConfig( + // The runner's logger and the config the deployed bundle's collector builds + // its own logger from (`context.logger`), so its destination errors flow + // through the SAME `onLine` ring tap as the runner's lines. Without it, + // production (no --verbose) passes no `context.logger`, the collector's + // createLogger has no handler, and destination "Push failed" errors never + // reach the ErrorRing (the heartbeat would report "No errors reported" + // despite failed deliveries). Both use the flow's own `logger.format` once + // the bundle reports it. + const logging = createRuntimeLogging( { ...options, onLine, knownSecrets: readKnownSecrets }, RUNTIME_COLORS, ); + const { logger } = logging; try { // Resolve port @@ -230,7 +232,9 @@ export async function runCommand(options: RunCommandOptions): Promise { bundlePath, port, logger: logger.scope('runner'), - loggerConfig: collectorLoggerConfig, + loggerConfig: logging.collectorConfig, + flowLogger: logging.flowLogger, + format: logging.format, api: apiConfig, errorRing, logRing, diff --git a/packages/runner/src/runner.ts b/packages/runner/src/runner.ts index 28db7ee5b..6d48b6a0c 100644 --- a/packages/runner/src/runner.ts +++ b/packages/runner/src/runner.ts @@ -14,8 +14,14 @@ import type { } from '@walkeros/core'; import type { HealthServer } from './health-server.js'; import { loadBundle } from './load-bundle.js'; +import type { FlowLogger } from './runtime-logging.js'; -export interface RuntimeConfig { +/** + * What the runner hands the bundle factory as its context, beside the logger + * config: the server settings and the flow logger contract (`logLevel`, + * `onLogger`). + */ +export interface RuntimeConfig extends Partial { port?: number; host?: string; } diff --git a/packages/runner/src/runtime-logging.ts b/packages/runner/src/runtime-logging.ts new file mode 100644 index 000000000..330e4b685 --- /dev/null +++ b/packages/runner/src/runtime-logging.ts @@ -0,0 +1,68 @@ +/** + * The runner's loggers and the flow logger contract it offers a bundle. + * + * The runner's own lines and the flow's collector lines go through one + * handler setup (masking, the ring tap) and one output format. The format is + * the flow's own `logger.format`: the bundle entry reports the flow's logger + * settings through `onLogger` before its collector starts, and every line + * from then on, the runner's own included, uses it. No environment variable + * sets it. + */ + +import { Level, type Logger } from '@walkeros/core'; +import { + createCLILogger, + createCLILoggerConfig, + type CLILoggerColors, + type CLILoggerOptions, +} from '@walkeros/core/node'; + +/** The flow's own logger settings, as the bundle entry reports them. */ +export type FlowLoggerSettings = Pick; + +/** + * What the runner hands a bundle beside `context.logger` (see the CLI's + * generated server entry): a level it forces over the flow's own, and the + * hook that receives the flow's settings. + */ +export interface FlowLogger { + /** Forced over the flow's level: DEBUG with `--verbose`. */ + logLevel?: Logger.Config['level']; + /** Receives the flow's own settings before its collector starts. */ + onLogger: (settings: FlowLoggerSettings) => void; +} + +export interface RuntimeLogging { + /** The runner's own lines. */ + logger: Logger.Instance; + /** Handed to the bundle as `context.logger`: the same masking and tap. */ + collectorConfig: Logger.Config; + /** Handed to the bundle beside `context.logger`. */ + flowLogger: FlowLogger; + /** The output format in effect: the flow's, once the bundle reported it. */ + format: () => Logger.Format; +} + +/** + * Build the runner's logger, the collector's logger config and the flow + * logger hook around one shared format. `options` are the terminal options + * (`verbose`, `silent`, `json`), `onLine` and `knownSecrets`. + */ +export function createRuntimeLogging( + options: Omit, + colors: CLILoggerColors = {}, +): RuntimeLogging { + let format: Logger.Format = 'text'; + const shared: CLILoggerOptions = { ...options, format: () => format }; + return { + logger: createCLILogger(shared, colors), + collectorConfig: createCLILoggerConfig(shared, colors), + flowLogger: { + ...(options.verbose ? { logLevel: Level.DEBUG } : {}), + onLogger: (settings) => { + if (settings.format) format = settings.format; + }, + }, + format: () => format, + }; +} diff --git a/packages/server/sources/aws/src/sqs/__mocks__/@aws-sdk/client-sqs.ts b/packages/server/sources/aws/src/sqs/__mocks__/@aws-sdk/client-sqs.ts index 93a433e06..00afb1217 100644 --- a/packages/server/sources/aws/src/sqs/__mocks__/@aws-sdk/client-sqs.ts +++ b/packages/server/sources/aws/src/sqs/__mocks__/@aws-sdk/client-sqs.ts @@ -59,6 +59,7 @@ interface MockState { receiveBatches: ReceiveBatch[]; createQueueError?: Error; getQueueUrlError?: Error; + receiveError?: Error; deletedReceiptHandles: string[]; } @@ -73,6 +74,7 @@ export function __resetMockCalls(): void { state.receiveBatches = []; state.createQueueError = undefined; state.getQueueUrlError = undefined; + state.receiveError = undefined; state.deletedReceiptHandles = []; calls.mockClear(); } @@ -102,6 +104,11 @@ export function __setGetQueueUrlError(err: Error | undefined): void { state.getQueueUrlError = err; } +/** The next ReceiveMessage rejects with `err`, once. */ +export function __setReceiveError(err: Error | undefined): void { + state.receiveError = err; +} + export function __getDeletedReceiptHandles(): string[] { return state.deletedReceiptHandles; } @@ -265,6 +272,11 @@ export class SQSClient { } private async handleReceive(): Promise { + const error = state.receiveError; + if (error) { + state.receiveError = undefined; + throw error; + } const batch = state.receiveBatches.shift(); if (!batch) { // Simulate the long-poll wait so the loop yields control. Without this, diff --git a/packages/server/sources/aws/src/sqs/__tests__/index.test.ts b/packages/server/sources/aws/src/sqs/__tests__/index.test.ts index 865b971a0..264ac3bef 100644 --- a/packages/server/sources/aws/src/sqs/__tests__/index.test.ts +++ b/packages/server/sources/aws/src/sqs/__tests__/index.test.ts @@ -6,6 +6,7 @@ import { __setQueueHarness, __setReceiveMessagesHarness, __setGetQueueUrlError, + __setReceiveError, __getMockCalls, __getDeletedReceiptHandles, } from '@aws-sdk/client-sqs'; @@ -155,6 +156,25 @@ describe('SQS source', () => { await expect(sourceSqs(ctx)).rejects.toThrow(/walkeros setup source\.sqs/); }); + it('logs a transient receive error at WARN: nothing was received, the loop retries', async () => { + __setQueueHarness('walkeros-events', {}); + __setReceiveError(new Error('socket hang up')); + const ctx = buildContext({}); + const warn = jest.spyOn(ctx.logger, 'warn'); + const error = jest.spyOn(ctx.logger, 'error'); + const instance = await bootSqs(ctx); + try { + for (let i = 0; i < 50 && warn.mock.calls.length === 0; i++) + await new Promise((r) => setTimeout(r, 10)); + expect(warn).toHaveBeenCalledWith('SQS receive error (will retry)', { + error: 'socket hang up', + }); + expect(error).not.toHaveBeenCalled(); + } finally { + await destroyInstance(instance, ctx); + } + }); + it('long-poll loop calls ReceiveMessage with declared maxMessages and waitTimeSeconds', async () => { __setQueueHarness('walkeros-events', {}); const ctx = buildContext({}); diff --git a/packages/server/sources/aws/src/sqs/__tests__/sqs-mock.d.ts b/packages/server/sources/aws/src/sqs/__tests__/sqs-mock.d.ts index 805ba4e5f..12a6938c5 100644 --- a/packages/server/sources/aws/src/sqs/__tests__/sqs-mock.d.ts +++ b/packages/server/sources/aws/src/sqs/__tests__/sqs-mock.d.ts @@ -33,6 +33,7 @@ declare module '@aws-sdk/client-sqs' { ): void; export function __setCreateQueueError(err: Error | undefined): void; export function __setGetQueueUrlError(err: Error | undefined): void; + export function __setReceiveError(err: Error | undefined): void; export function __getDeletedReceiptHandles(): string[]; export function __getMockCalls(): Array<{ method: string; input: unknown }>; } diff --git a/packages/server/sources/aws/src/sqs/index.ts b/packages/server/sources/aws/src/sqs/index.ts index fbc52a457..771563c30 100644 --- a/packages/server/sources/aws/src/sqs/index.ts +++ b/packages/server/sources/aws/src/sqs/index.ts @@ -251,7 +251,9 @@ export const sourceSqs: Source.Init = async (context) => { stopped = true; break; } - logger.error('SQS receive error (will retry)', { error }); + // Self-healing: nothing was received, so nothing is lost; the loop + // retries. WARN: worth acting on if it repeats, intent still met. + logger.warn('SQS receive error (will retry)', { error }); // Backoff before retry to avoid tight-looping on persistent errors. await new Promise((r) => setTimeout(r, RECEIVE_RETRY_BACKOFF_MS)); } diff --git a/packages/server/sources/gcp/src/pubsub/pull/__tests__/index.test.ts b/packages/server/sources/gcp/src/pubsub/pull/__tests__/index.test.ts index 20ef42855..0b1314af8 100644 --- a/packages/server/sources/gcp/src/pubsub/pull/__tests__/index.test.ts +++ b/packages/server/sources/gcp/src/pubsub/pull/__tests__/index.test.ts @@ -169,6 +169,21 @@ describe('Pub/Sub pull source', () => { expect(String(firstCall?.[0])).toContain('walkeros setup source.pubsub'); }); + it('logs a transient stream error at WARN: the SDK reconnects, nothing is lost', async () => { + const ctx = buildContext({}); + const warnSpy = jest.spyOn(ctx.logger, 'warn'); + const errorSpy = jest.spyOn(ctx.logger, 'error'); + await sourcePubSubPull(ctx); + const err: Error & { code?: number } = new Error('unavailable'); + err.code = 14; + __triggerError(err); + expect(warnSpy).toHaveBeenCalledWith( + 'Pub/Sub subscription stream error (SDK will retry)', + { error: 'unavailable' }, + ); + expect(errorSpy).not.toHaveBeenCalled(); + }); + it('decodeReturnsNull acks-and-drops via synthetic push', async () => { const ctx = buildContext({}); const instance = await sourcePubSubPull(ctx); diff --git a/packages/server/sources/gcp/src/pubsub/pull/index.ts b/packages/server/sources/gcp/src/pubsub/pull/index.ts index 12ab2c9af..39fb36cb8 100644 --- a/packages/server/sources/gcp/src/pubsub/pull/index.ts +++ b/packages/server/sources/gcp/src/pubsub/pull/index.ts @@ -175,7 +175,9 @@ export const sourcePubSubPull: Source.Init = async (context) => { `Run "walkeros setup source.${id}" to create it. Original: ${err.message}`, ); } else { - logger.error('Pub/Sub subscription stream error (SDK will retry)', { + // Self-healing: the SDK reconnects and no message is acked, so none + // is lost. WARN: worth acting on if it repeats, intent still met. + logger.warn('Pub/Sub subscription stream error (SDK will retry)', { error: err.message, }); } From 1a19bf4d9073dc19aa290a006520217a2bd9e4ce Mon Sep 17 00:00:00 2001 From: alexanderkirtzel Date: Wed, 21 Oct 2026 04:33:30 +0000 Subject: [PATCH 07/41] init timeout --- .../src/__tests__/init-timeout.test.ts | 237 ++++++++++++++++++ packages/collector/src/destination.ts | 26 +- packages/collector/src/transformer.ts | 24 +- 3 files changed, 273 insertions(+), 14 deletions(-) create mode 100644 packages/collector/src/__tests__/init-timeout.test.ts diff --git a/packages/collector/src/__tests__/init-timeout.test.ts b/packages/collector/src/__tests__/init-timeout.test.ts new file mode 100644 index 000000000..096314d74 --- /dev/null +++ b/packages/collector/src/__tests__/init-timeout.test.ts @@ -0,0 +1,237 @@ +import { Level, resolveDestinationTimeout } from '@walkeros/core'; +import type { + Destination, + Elb, + Logger, + Transformer, + WalkerOS, +} from '@walkeros/core'; +import { startFlow } from '..'; + +/** + * An init that never settles is bounded by the step's timeout, like a push: + * the pushes waiting for it answer `failed` with code `timeout` instead of + * waiting forever, the next push starts a new attempt, and an attempt that + * settles after its timeout installs nothing. + */ + +type Line = { level: Level; message: string; context: Logger.LogContext }; + +function capture() { + const lines: Line[] = []; + const logger: Logger.Config = { + handler: (level, message, context) => { + lines.push({ level, message, context }); + }, + }; + // A cause's first line names the counter its repeats go to. + const errors = (message: string) => + lines.filter( + (line) => line.level === Level.ERROR && line.message.startsWith(message), + ); + return { logger, errors }; +} + +/** A push's answer once it arrived, read after advancing the clock. */ +function track(push: Promise) { + let answer: Elb.PushResult | undefined; + void push.then((result) => { + answer = result; + }); + return () => answer; +} + +async function turns() { + for (let turn = 0; turn < 100; turn++) await Promise.resolve(); +} + +/** Each init call is one attempt, settled by hand. */ +function attempts() { + const started: Array<(value: T) => void> = []; + const init = () => + new Promise((resolve) => { + started.push(resolve); + }); + return { init, started }; +} + +const never = () => new Promise(() => undefined); + +describe('destination init timeout', () => { + it('answers each push failed with code timeout once the destination timeout passes, with one ERROR line', async () => { + const { logger, errors } = capture(); + const push = jest.fn(); + const destination: Destination.Instance = { + type: 'hung', + config: {}, + init: never, + push, + }; + const { collector, elb } = await startFlow({ + logger, + destinations: { d: { code: destination, config: { timeout: 200 } } }, + }); + + for (const name of ['page view', 'page read']) { + const answer = track(elb(name)); + await jest.advanceTimersByTimeAsync(200); + const result = answer(); + expect(result?.ok).toBe(false); + expect(result?.failed?.d?.reason).toEqual( + expect.objectContaining({ code: 'timeout', retryable: true }), + ); + } + + expect(push).not.toHaveBeenCalled(); + expect(collector.destinations.d.config.init).toBeFalsy(); + expect(collector.status.destinations.d.failed).toBe(2); + // One line for the cause; the repeat is counted. + const lines = errors('destination init failed'); + expect(lines).toHaveLength(1); + expect(lines[0].context.code).toBe('timeout'); + }); + + it('settles every push parked behind the hung init with the same failure', async () => { + const push = jest.fn(); + const destination: Destination.Instance = { + type: 'hung', + config: {}, + init: never, + push, + }; + const { collector, elb } = await startFlow({ + logger: { handler: () => undefined }, + destinations: { d: { code: destination, config: { timeout: 200 } } }, + }); + + const answers = ['page view', 'page read', 'page scroll'].map((name) => + track(elb(name)), + ); + await turns(); + expect(answers.map((answer) => answer())).toEqual([ + undefined, + undefined, + undefined, + ]); + + await jest.advanceTimersByTimeAsync(200); + + for (const answer of answers) { + expect(answer()?.ok).toBe(false); + expect(answer()?.failed?.d?.reason).toEqual( + expect.objectContaining({ code: 'timeout', retryable: true }), + ); + } + expect(push).not.toHaveBeenCalled(); + expect(collector.status.destinations.d.failed).toBe(3); + expect(collector.destinations.d.queuePush ?? []).toHaveLength(0); + }); + + it('starts a new attempt on the next push; the stale attempt settling late installs nothing', async () => { + const names: string[] = []; + const { init, started } = attempts(); + const destination: Destination.Instance = { + type: 'slow', + config: {}, + init, + push: (event: WalkerOS.Event) => { + names.push(event.name); + }, + }; + const { collector, elb } = await startFlow({ + logger: { handler: () => undefined }, + destinations: { + d: { + code: destination, + config: { timeout: 200, settings: { attempt: 'none' } }, + }, + }, + }); + const d = collector.destinations.d; + + const first = track(elb('page view')); + await jest.advanceTimersByTimeAsync(200); + expect(first()?.failed?.d?.reason?.code).toBe('timeout'); + expect(started).toHaveLength(1); + + const second = track(elb('page read')); + await turns(); + expect(started).toHaveLength(2); + + // The timed-out attempt settles now, with its own config. + started[0]({ timeout: 200, settings: { attempt: 'stale' } }); + await turns(); + expect(d.config.init).toBeFalsy(); + expect(d.config.settings).toEqual({ attempt: 'none' }); + expect(second()).toBeUndefined(); + + // The current attempt settles in time: its config is installed. + started[1]({ timeout: 200, settings: { attempt: 'fresh' } }); + await turns(); + expect(second()?.ok).toBe(true); + expect(d.config.init).toBe(true); + expect(d.config.settings).toEqual({ attempt: 'fresh' }); + expect(names).toEqual(['page read']); + }); +}); + +describe('transformer init timeout', () => { + // A transformer has no timeout setting: its init is bounded by the + // default destination timeout. + const timeout = resolveDestinationTimeout(); + + it('fails the push with code timeout after the default timeout, retries on the next push, and ignores the stale attempt', async () => { + const { logger, errors } = capture(); + const delivered: string[] = []; + const { init, started } = attempts(); + const enrich: Transformer.Instance = { + type: 'enrich', + config: {}, + init, + push: (event) => ({ event }), + }; + const sink: Destination.Instance = { + type: 'sink', + config: {}, + push: (event: WalkerOS.Event) => { + delivered.push(event.name); + }, + }; + const { collector, elb } = await startFlow({ + logger, + next: 'enrich', + transformers: { enrich: { code: async () => enrich } }, + destinations: { sink: { code: sink } }, + }); + + const first = track(elb('page view')); + await jest.advanceTimersByTimeAsync(timeout - 1); + expect(first()).toBeUndefined(); + await jest.advanceTimersByTimeAsync(1); + expect(first()?.ok).toBe(false); + expect(first()?.reason).toEqual( + expect.objectContaining({ code: 'timeout', retryable: true }), + ); + const lines = errors('transformer init failed'); + expect(lines).toHaveLength(1); + expect(lines[0].context.code).toBe('timeout'); + + const second = track(elb('page read')); + await turns(); + expect(started).toHaveLength(2); + + started[0]({ settings: { attempt: 'stale' } }); + await turns(); + expect(collector.transformers.enrich.config.init).toBeFalsy(); + expect(collector.transformers.enrich.config.settings).toBeUndefined(); + + started[1]({ settings: { attempt: 'fresh' } }); + await turns(); + expect(second()?.ok).toBe(true); + expect(collector.transformers.enrich.config.init).toBe(true); + expect(collector.transformers.enrich.config.settings).toEqual({ + attempt: 'fresh', + }); + expect(delivered).toEqual(['page read']); + }); +}); diff --git a/packages/collector/src/destination.ts b/packages/collector/src/destination.ts index fd4e58eda..7201b539b 100644 --- a/packages/collector/src/destination.ts +++ b/packages/collector/src/destination.ts @@ -1528,8 +1528,11 @@ function hasConsentRequirement(destination: Destination.Instance): boolean { /** * Run a destination's own `init()` and return the config to install, marked - * initialized. An `init()` that throws, or returns false, throws (the - * latter as `StepError` code `not-ready`). The caller installs the config. + * initialized. An `init()` that throws, returns false (`StepError` code + * `not-ready`), or does not settle within the destination's timeout + * (`DestinationTimeoutError`, code `timeout`) throws. The caller installs the + * config, so an attempt that settles after its timeout installs nothing; the + * next push starts a new one. */ async function runDestinationInit( collector: Collector.Instance, @@ -1561,14 +1564,21 @@ async function runDestinationInit( const initStarted = Date.now(); emitDestinationState(collector, destId, 'init', undefined, undefined); + const timeoutMs = resolveDestinationTimeout(destination.config.timeout); let configResult; try { - configResult = await useHooks( - init, - 'DestinationInit', - collector.hooks, - collector.logger, - )(context); + configResult = await withTimeout( + Promise.resolve( + useHooks( + init, + 'DestinationInit', + collector.hooks, + collector.logger, + )(context), + ), + timeoutMs, + `Destination "${destId}" init timed out after ${timeoutMs}ms`, + ); } catch (err) { emitDestinationState( collector, diff --git a/packages/collector/src/transformer.ts b/packages/collector/src/transformer.ts index 62db45126..8a128c480 100644 --- a/packages/collector/src/transformer.ts +++ b/packages/collector/src/transformer.ts @@ -68,6 +68,7 @@ import { getCacheStore } from './cache'; import { buildReportError, errorMeta } from './report-error'; import { compileStepState, runStepState, warnStateOff } from './state'; import { asStepError, initReturnedFalse } from './outcome'; +import { resolveDestinationTimeout, withTimeout } from './timeout'; /** * Extracts chain property from definition and merges into config. @@ -315,12 +316,23 @@ export async function transformerInit( transformerLogger.debug('init'); - const configResult = await useHooks( - transformer.init, - 'TransformerInit', - collector.hooks, - collector.logger, - )(context); + // A transformer has no timeout setting: its init is bounded by the + // default destination timeout. One that does not settle in time throws + // `DestinationTimeoutError` (code `timeout`); the config below is only + // installed from an attempt that settled in time. + const timeoutMs = resolveDestinationTimeout(); + const configResult = await withTimeout( + Promise.resolve( + useHooks( + transformer.init, + 'TransformerInit', + collector.hooks, + collector.logger, + )(context), + ), + timeoutMs, + `Transformer "${transformerId}" init timed out after ${timeoutMs}ms`, + ); // Check for initialization failure if (configResult === false) return false; From 4aa264457d57ebad082c1d8b910aabb212669459 Mon Sep 17 00:00:00 2001 From: alexanderkirtzel Date: Wed, 21 Oct 2026 04:35:32 +0000 Subject: [PATCH 08/41] kit fixes --- .../src/dev/__tests__/conformance.test.ts | 189 +++++++++++++++++- .../fixture-package/src/alternatives.ts | 46 +++++ .../__tests__/fixture-package/src/vendor.ts | 40 ++++ packages/collector/src/dev/conformance.ts | 127 ++++++++++-- packages/collector/src/dev/runExamples.ts | 48 ++++- 5 files changed, 423 insertions(+), 27 deletions(-) create mode 100644 packages/collector/src/dev/__tests__/fixture-package/src/alternatives.ts diff --git a/packages/collector/src/dev/__tests__/conformance.test.ts b/packages/collector/src/dev/__tests__/conformance.test.ts index 9c12f23bd..f285fdf1d 100644 --- a/packages/collector/src/dev/__tests__/conformance.test.ts +++ b/packages/collector/src/dev/__tests__/conformance.test.ts @@ -17,7 +17,13 @@ import type { Client } from './fixture-package/src/store'; import { missing, remote } from './fixture-package/src/store'; import { chatty, forwarding } from './fixture-package/src/source'; import type { Vendor } from './fixture-package/src/vendor'; -import { vendorGlobal } from './fixture-package/src/vendor'; +import { + consentAware, + consentIgnoring, + initCalling, + vendorGlobal, +} from './fixture-package/src/vendor'; +import { eitherId, ga4Only } from './fixture-package/src/alternatives'; describe('conformance kit', () => { beforeEach(() => { @@ -249,6 +255,7 @@ describe('conformance kit, vendor-global profile', () => { }); const vendor: Vendor = { + init: () => undefined, track: () => undefined, consent: () => undefined, }; @@ -294,6 +301,100 @@ describe('conformance kit, vendor-global profile', () => { ).resolves.toBeUndefined(); }); + describe('a destination whose init calls its vendor', () => { + const tracked = { + ...options.env, + simulation: [...options.env.simulation, 'call:vendor.init'], + }; + const consented = { ...event, consent: { marketing: true } }; + const initExamples: Flow.StepExamples = { + pageView: { ...examples.pageView, in: consented }, + silent: { ...examples.silent, in: consented }, + unreachable: { ...examples.unreachable, in: consented }, + // Grants marketing (the init runs inside it), revokes functional. + revoke: { + command: 'consent', + in: { marketing: true, functional: false }, + out: [['vendor.consent', true]], + }, + }; + const initOptions = { ...options, env: tracked, examples: initExamples }; + + it('leaves the init calls of a destination initialized inside an input out of its effects (items 2, 8 and 11 pass)', async () => { + await expect( + conformance('destination', initCalling, initOptions), + ).resolves.toBeUndefined(); + }); + + it('passes web-sdk and fails item 5 only under sdk-env when init starts the SDK at page load', async () => { + // Without a consent gate the collector initializes it inside startFlow. + const atLoad = { ...initCalling, config: {} }; + const atLoadOptions = { ...options, env: tracked }; + await expect( + conformance('destination', atLoad, { + ...atLoadOptions, + profile: 'web-sdk', + }), + ).resolves.toBeUndefined(); + await expect( + conformance('destination', atLoad, { + ...atLoadOptions, + profile: 'sdk-env', + }), + ).rejects.toMatchObject({ + items: [5], + message: expect.stringMatching(/item 5: init makes 1 /), + }); + }); + + it('still counts the init calls for item 5', async () => { + await expect( + conformance('destination', initCalling, { + ...initOptions, + profile: 'sdk-env', + }), + ).rejects.toMatchObject({ + items: [5], + message: expect.stringMatching(/item 5: init makes 1 /), + }); + }); + }); + + describe('item 8 runs a revoke gated on its keys, granted first (I1)', () => { + it('passes a destination that passes the revocation to its vendor', async () => { + await expect( + conformance('destination', consentAware, options), + ).resolves.toBeUndefined(); + }); + + it('fails item 8 when the revoke example leaves the vendor call out', async () => { + await expect( + conformance('destination', consentAware, { + ...options, + examples: { + ...examples, + revoke: { + ...examples.revoke, + description: 'No consent call.', + out: [], + }, + }, + }), + ).rejects.toMatchObject({ items: [8] }); + }); + + it('fails item 8 for a destination that ignores the revocation', async () => { + await expect( + conformance('destination', consentIgnoring, options), + ).rejects.toMatchObject({ + items: [8], + message: expect.stringContaining( + 'item 8: example revoke: out differs: expected [["vendor.consent",false]], got []', + ), + }); + }); + }); + it('fails a consent revoke example that declares no out on item 8 only', async () => { const { out, ...revoke } = examples.revoke; await expect( @@ -304,3 +405,89 @@ describe('conformance kit, vendor-global profile', () => { ).rejects.toMatchObject({ items: [8] }); }); }); + +describe('conformance kit, item 4 with alternatives (anyOf)', () => { + beforeEach(() => { + jest.useRealTimers(); + }); + + const event = { name: 'page view', data: { title: 'Home' } }; + const examples: Flow.StepExamples = { + pageView: { + in: event, + out: [ + [ + 'fetch', + 'https://127.0.0.1:9/collect', + { method: 'POST', body: '{"id":"G-1","name":"page view"}' }, + ], + ], + }, + silent: { + public: false, + description: 'A silent rule sends nothing.', + in: event, + mapping: { silent: true }, + out: [], + }, + }; + const options = { + profile: 'http' as const, + root: fixtures.root, + settings: { ga4: { measurementId: 'G-1' } }, + schema: { + type: 'object', + properties: { + ga4: { + type: 'object', + properties: { measurementId: { type: 'string' } }, + }, + ads: { + type: 'object', + properties: { conversionId: { type: 'string' } }, + }, + }, + anyOf: [ + { + required: ['ga4'], + properties: { ga4: { required: ['measurementId'] } }, + }, + { + required: ['ads'], + properties: { ads: { required: ['conversionId'] } }, + }, + ], + }, + examples, + }; + + it('passes an init error that names a key of every alternative', async () => { + await expect( + conformance('destination', eitherId, options), + ).resolves.toBeUndefined(); + }); + + it('fails item 4 only when the init error names no key of one alternative', async () => { + await expect( + conformance('destination', ga4Only, options), + ).rejects.toMatchObject({ + items: [4], + message: expect.stringContaining( + 'names no key of the alternative ads.conversionId', + ), + }); + }); + + it('still counts a schema whose alternative requires nothing as requiring nothing', async () => { + const anyOf = [...options.schema.anyOf, { properties: {} }]; + await expect( + conformance('destination', eitherId, { + ...options, + schema: { ...options.schema, anyOf }, + }), + ).rejects.toMatchObject({ + items: [4], + message: expect.stringContaining('although the schema requires nothing'), + }); + }); +}); diff --git a/packages/collector/src/dev/__tests__/fixture-package/src/alternatives.ts b/packages/collector/src/dev/__tests__/fixture-package/src/alternatives.ts new file mode 100644 index 000000000..4703469c6 --- /dev/null +++ b/packages/collector/src/dev/__tests__/fixture-package/src/alternatives.ts @@ -0,0 +1,46 @@ +import { StepError } from '@walkeros/core'; +import type { Destination } from '@walkeros/core'; + +/** One of the two ids is required, as gtag requires one of its tools. */ +type Settings = { + ga4?: { measurementId?: string }; + ads?: { conversionId?: string }; +}; +type Types = Destination.Types; + +const URL = 'https://127.0.0.1:9/collect'; + +function idOf(settings: Settings | undefined): string | undefined { + return settings?.ga4?.measurementId ?? settings?.ads?.conversionId; +} + +function oneOfIds(message: string): Destination.Instance { + return { + type: 'one-of', + config: {}, + init: ({ config }) => { + if (!idOf(config.settings)) + throw new StepError(message, { code: 'config' }); + }, + push: async (event, { config, rule }) => { + if (rule?.silent) return; + const res = await fetch(URL, { + method: 'POST', + body: JSON.stringify({ id: idOf(config.settings), name: event.name }), + }); + if (!res.ok) + throw new StepError(`HTTP ${res.status}`, { + code: 'transport', + status: res.status, + }); + }, + }; +} + +/** Names a key of every alternative when none is set. */ +export const eitherId = oneOfIds( + 'Config settings ga4.measurementId or ads.conversionId missing', +); + +/** Names the ga4 alternative only. */ +export const ga4Only = oneOfIds('Config settings ga4.measurementId missing'); diff --git a/packages/collector/src/dev/__tests__/fixture-package/src/vendor.ts b/packages/collector/src/dev/__tests__/fixture-package/src/vendor.ts index 89b77555d..e7bbfcbf3 100644 --- a/packages/collector/src/dev/__tests__/fixture-package/src/vendor.ts +++ b/packages/collector/src/dev/__tests__/fixture-package/src/vendor.ts @@ -2,6 +2,7 @@ import { isObject, StepError } from '@walkeros/core'; import type { Destination } from '@walkeros/core'; export interface Vendor { + init(site: string): void | Promise; track(name: string, site?: string): void | Promise; consent(granted: boolean): void; } @@ -44,3 +45,42 @@ export const vendorGlobal: Destination.Instance = { vendor.consent(data.marketing === true); }, }; + +/** + * Calls its vendor in `init`, as `fbq('init', ...)` does: the call is queued + * by the vendor, not awaited. It requires marketing consent, so the collector + * initializes it inside the first input that grants it, not at the start. + */ +export const initCalling: Destination.Instance = { + ...vendorGlobal, + type: 'init-calling', + config: { consent: { marketing: true } }, + init: ({ config, env }) => { + if (!env.vendor) return; + Promise.resolve(env.vendor.init(config.settings?.site ?? '')).catch( + () => undefined, + ); + }, +}; + +/** + * Follows its own `config.consent` at the vendor: on a consent change it + * tells the vendor whether every key it requires is still granted. + */ +export const consentAware: Destination.Instance = { + ...vendorGlobal, + type: 'consent-aware', + on: (type, { config, env, data }) => { + const vendor = env.vendor; + if (type !== 'consent' || !isVendor(vendor) || !isObject(data)) return; + const required = Object.keys(config.consent ?? {}); + vendor.consent(required.every((key) => data[key] === true)); + }, +}; + +/** Ignores a revocation: the vendor keeps collecting on its own (I1). */ +export const consentIgnoring: Destination.Instance = { + ...vendorGlobal, + type: 'consent-ignoring', + on: undefined, +}; diff --git a/packages/collector/src/dev/conformance.ts b/packages/collector/src/dev/conformance.ts index 87ec6b8cb..c799283db 100644 --- a/packages/collector/src/dev/conformance.ts +++ b/packages/collector/src/dev/conformance.ts @@ -31,6 +31,7 @@ import { openTransformer, outcomeOf, readEnv, + revokedKeys, RUN_TIMEOUT, runExamples, runStoreOperation, @@ -52,6 +53,7 @@ import { export type Profile = | 'http' | 'sdk-env' + | 'web-sdk' | 'vendor-global' | 'socket' | 'transformer' @@ -60,7 +62,10 @@ export type Profile = export interface ConformanceOptions { profile: Profile; examples: Flow.StepExamples; - /** The settings schema (JSON Schema): its properties and `required`. */ + /** + * The settings schema (JSON Schema): its properties, `required`, and + * `anyOf`/`oneOf` alternatives for a "one of these" rule (item 4). + */ schema: JSONSchema; /** Minimal valid settings, pure JSON. */ settings: Record; @@ -92,6 +97,8 @@ export class ConformanceError extends Error { const PROFILE_ITEMS: Record = { http: [1, 2, 3, 4, 5, 6, 7, 9, 10], 'sdk-env': [1, 2, 3, 4, 5, 6, 7, 9, 10], + // A web SDK starts in init at page load on purpose (autocapture, recording). + 'web-sdk': [1, 2, 3, 4, 6, 7, 9, 10], 'vendor-global': [1, 2, 4, 6, 7, 8, 10], socket: [1, 2, 3, 4, 5, 6, 7, 9], transformer: [1, 2, 4, 5, 6, 7, 10], @@ -120,7 +127,7 @@ interface Context { /** Env the run uses: the examples env except where the profile has none. */ runEnv?: ExamplesEnv; schemaKeys: string[]; - required: string[]; + requires: Requirements; events: EventExample[]; /** The first success event example that sends something. */ primary?: EventExample; @@ -138,7 +145,6 @@ const sends = (example: Flow.StepExample) => function contextOf(options: ConformanceOptions): Context { const env = readEnv(options.env); const properties = options.schema.properties; - const required = options.schema.required; const events: EventExample[] = []; for (const [name, example] of Object.entries(options.examples)) { if (example.command || example.outcome || example.respond) continue; @@ -151,9 +157,7 @@ function contextOf(options: ConformanceOptions): Context { env, runEnv: options.profile === 'http' ? undefined : env, schemaKeys: isObject(properties) ? Object.keys(properties) : [], - required: Array.isArray(required) - ? required.filter((key): key is string => typeof key === 'string') - : [], + requires: requirementsOf(options.schema), events, primary: primary ?? events[0], }; @@ -182,26 +186,111 @@ function stepError(error: unknown): StepError | undefined { return error instanceof StepError ? error : undefined; } +/** + * What a settings schema requires: keys it always requires (top-level + * `required`), and groups of alternatives (`anyOf`/`oneOf`, also inside + * `allOf`) of which at least one branch must hold. A branch's keys are its + * `required` names, as dotted paths where the named property requires keys + * of its own (`ga4.measurementId`). + */ +interface Requirements { + keys: string[]; + alternatives: string[][][]; +} + +function stringsOf(value: unknown): string[] { + return Array.isArray(value) + ? value.filter((item): item is string => typeof item === 'string') + : []; +} + +function propertyOf(properties: unknown, key: string): unknown { + return isObject(properties) ? properties[key] : undefined; +} + +/** The keys a schema node requires, nested `required` as dotted paths. */ +function requiredPaths( + node: unknown, + properties: unknown, + prefix = '', + depth = 0, +): string[] { + if (!isObject(node) || depth > 5) return []; + return stringsOf(node.required).flatMap((name) => { + const path = prefix ? `${prefix}.${name}` : name; + const child = + propertyOf(node.properties, name) ?? propertyOf(properties, name); + const inner = requiredPaths(child, undefined, path, depth + 1); + return inner.length ? inner : [path]; + }); +} + +function requirementsOf(schema: JSONSchema): Requirements { + const alternatives: string[][][] = []; + const collect = (node: unknown) => { + if (!isObject(node)) return; + for (const branches of [node.anyOf, node.oneOf]) { + if (!Array.isArray(branches) || !branches.length) continue; + const keys = branches.map((branch) => + requiredPaths(branch, schema.properties), + ); + // A branch that requires nothing lets {} pass: nothing is required. + if (keys.every((branch) => branch.length)) alternatives.push(keys); + } + if (Array.isArray(node.allOf)) node.allOf.forEach(collect); + }; + collect(schema); + return { keys: stringsOf(schema.required), alternatives }; +} + +const requiresSomething = ({ keys, alternatives }: Requirements) => + keys.length > 0 || alternatives.length > 0; + +/** The requirements as words for a message: `a, b and c or d`. */ +function describeRequired({ keys, alternatives }: Requirements): string { + return [ + ...keys, + ...alternatives.map((branches) => + branches.map((branch) => branch.join(' and ')).join(' or '), + ), + ].join(', '); +} + /** Problems of what init threw with `settings: {}` (item 4). */ function notReadyProblem(ctx: Context, thrown: unknown[]): Problem { - if (!ctx.required.length) { + const { requires } = ctx; + if (!requiresSomething(requires)) { const failed = thrown.map(stepError).find((error) => error); return failed && NOT_STARTED.includes(failed.code) ? `fails ${failed.code} with settings {} although the schema requires nothing` : undefined; } + const naming = describeRequired(requires); if (!thrown.length) - return `init does not throw with settings {}: it must throw a StepError naming ${ctx.required.join(', ')}`; + return `init does not throw with settings {}: it must throw a StepError naming ${naming}`; const error = stepError(thrown[0]); if (!error) - return `init throws a plain error with settings {}: throw a StepError with code config naming ${ctx.required.join(', ')}`; + return `init throws a plain error with settings {}: throw a StepError with code config naming ${naming}`; if (error.code !== 'config') return `init throws code ${error.code} with settings {}: the code is config`; - const named = ctx.schemaKeys.filter((key) => wordIn(error.message, key)); - const missing = ctx.required.filter((key) => !named.includes(key)); - const extra = named.filter((key) => !ctx.required.includes(key)); + const { message } = error; + const missing = requires.keys.filter((key) => !wordIn(message, key)); if (missing.length) return `the init error does not name the required ${missing.join(', ')}`; + for (const branches of requires.alternatives) { + const unnamed = branches.find( + (branch) => !branch.some((key) => wordIn(message, key)), + ); + if (unnamed) + return `the init error names no key of the alternative ${unnamed.join(' and ')}`; + } + const allowed = new Set([ + ...requires.keys, + ...requires.alternatives.flat(2).map((path) => path.split('.')[0]), + ]); + const extra = ctx.schemaKeys.filter( + (key) => !allowed.has(key) && wordIn(message, key), + ); if (extra.length) return `the init error names ${extra.join(', ')}, which the schema does not require`; return undefined; @@ -469,8 +558,8 @@ async function destinationItem( example: {}, event: getEvent('page view'), }; - if (ctx.required.length && !step.init) - return `checks no settings in init: init must throw a StepError naming ${ctx.required.join(', ')}`; + if (requiresSomething(ctx.requires) && !step.init) + return `checks no settings in init: init must throw a StepError naming ${describeRequired(ctx.requires)}`; const thrown: unknown[] = []; await runDestination(step, ctx, target, { settings: {}, @@ -528,11 +617,9 @@ async function destinationItem( } case 8: { + // Each revoke runs gated on its revoked keys, granted first (I1). const revokes = Object.entries(options.examples).filter( - ([, example]) => - example.command === 'consent' && - isObject(example.in) && - Object.values(example.in).some((value) => value === false), + ([, example]) => revokedKeys(example).length > 0, ); if (!revokes.length) return 'no walker consent revoke example (its out is the vendor consent call, or [] with a description)'; @@ -834,7 +921,7 @@ async function transformerItem( case 2: { const problems: string[] = []; - if (ctx.required.length) { + if (requiresSomething(ctx.requires)) { const run = await runTransformer(step, ctx, { settings: {} }, event); // Refusing to start with a StepError is reported too, at the start. if (run.startError !== undefined) { @@ -1233,7 +1320,7 @@ type KitArgs = | ['source', SourceStep]; const KIND_PROFILES: Record = { - destination: ['http', 'sdk-env', 'vendor-global', 'socket'], + destination: ['http', 'sdk-env', 'web-sdk', 'vendor-global', 'socket'], transformer: ['transformer'], store: ['socket'], source: ['source'], diff --git a/packages/collector/src/dev/runExamples.ts b/packages/collector/src/dev/runExamples.ts index 7fb0e7b9e..0c3516589 100644 --- a/packages/collector/src/dev/runExamples.ts +++ b/packages/collector/src/dev/runExamples.ts @@ -265,9 +265,19 @@ async function open( } else env = envBoundary.env; } + // Calls recorded while the step initializes belong to the init phase: + // item 5 reads them there, and no example's effects ever include them. + const initRecorded = new Set(); + let initFrom: number | undefined; const setPhase = (phase: Phase) => { boundary.phase = phase; if (envBoundary) envBoundary.phase = phase; + if (phase === 'init') initFrom = recorded.length; + else if (initFrom !== undefined) { + for (let index = initFrom; index < recorded.length; index++) + initRecorded.add(index); + initFrom = undefined; + } }; const handler: Logger.Handler = (level, message, context, scope) => { @@ -286,10 +296,18 @@ async function open( throw error; } + /** What the step did for its inputs: never what its init did. */ const effects = (): Flow.StepEffect[] => { if (effectsOf) return effectsOf(); - if (setup.env) return recorded.map(({ fn, args }) => [fn, ...args]); - return boundary.requests.map((request) => request.effect); + if (setup.env) + return recorded.flatMap(({ fn, args }, index) => { + if (initRecorded.has(index)) return []; + const effect: Flow.StepEffect = [fn, ...args]; + return [effect]; + }); + return boundary.requests + .filter((request) => request.phase !== 'init') + .map((request) => request.effect); }; return { @@ -625,6 +643,17 @@ function setupOf( }; } +/** + * The keys a `walker consent` example revokes (sets to `false`). An example + * without any is not a revoke example. + */ +export function revokedKeys(example: Flow.StepExample): string[] { + if (example.command !== 'consent' || !isObject(example.in)) return []; + return Object.entries(example.in) + .filter(([, value]) => value === false) + .map(([key]) => key); +} + /** Run one destination example; its problems, empty when it holds. */ export async function checkDestinationExample( step: Destination.Instance, @@ -633,11 +662,18 @@ export async function checkDestinationExample( ): Promise { const input = inputOf(example); if (typeof input === 'string') return [input]; - const session = await openDestination( - step, - setupOf(example, options, mappingOf(example, input)), - ); + const setup = setupOf(example, options, mappingOf(example, input)); + + // A revoke runs as a revocation happens: the destination is gated on the + // revoked keys and granted them first; only the revoke's own effects + // count, the grant's stay out like the init's. + const revoked = revokedKeys(example); + const grant = Object.fromEntries(revoked.map((key) => [key, true])); + if (revoked.length) setup.config = { ...setup.config, consent: grant }; + + const session = await openDestination(step, setup); try { + if (revoked.length) await session.run({ command: 'consent', data: grant }); const { result, effects } = await session.run(input); const problems: string[] = []; const out = compareOut(effects, example); From 4ca1bc7a9464ed8819715f15916111c52e5eeb93 Mon Sep 17 00:00:00 2001 From: alexanderkirtzel Date: Wed, 21 Oct 2026 04:39:12 +0000 Subject: [PATCH 09/41] sources acknowledge --- packages/core/src/__tests__/envelope.test.ts | 65 +++++++- packages/core/src/envelope.ts | 50 ++++--- .../src/lambda/__tests__/acknowledge.test.ts | 98 ++++++++++++ .../aws/src/lambda/__tests__/index.test.ts | 8 +- .../server/sources/aws/src/lambda/index.ts | 8 +- .../server/sources/aws/src/lambda/push.ts | 30 ++-- .../aws/src/sqs/__tests__/acknowledge.test.ts | 141 ++++++++++++++++++ packages/server/sources/aws/src/sqs/index.ts | 19 ++- .../sources/aws/src/sqs/schemas/settings.ts | 2 +- .../express/src/__tests__/acknowledge.test.ts | 102 +++++++++++++ .../express/src/__tests__/index.test.ts | 1 + .../src/__tests__/input-hygiene.test.ts | 10 +- packages/server/sources/express/src/index.ts | 25 ++-- .../fetch/src/__tests__/acknowledge.test.ts | 97 ++++++++++++ .../fetch/src/__tests__/envelope.test.ts | 3 +- .../sources/fetch/src/__tests__/index.test.ts | 6 +- .../fetch/src/__tests__/logging.test.ts | 6 +- packages/server/sources/fetch/src/index.ts | 29 +--- .../__tests__/acknowledge.test.ts | 89 +++++++++++ .../src/cloudfunction/__tests__/index.test.ts | 7 +- .../sources/gcp/src/cloudfunction/index.ts | 10 +- .../sources/gcp/src/cloudfunction/push.ts | 37 +++-- .../pubsub/pull/__tests__/acknowledge.test.ts | 119 +++++++++++++++ .../sources/gcp/src/pubsub/pull/index.ts | 21 ++- .../gcp/src/pubsub/pull/schemas/settings.ts | 2 +- .../pubsub/push/__tests__/acknowledge.test.ts | 79 ++++++++++ .../sources/gcp/src/pubsub/push/index.ts | 44 +++--- .../src/__tests__/elbLayerFailure.test.ts | 97 ++++++++++++ .../browser/src/__tests__/pageHide.test.ts | 112 ++++++++++++++ packages/web/sources/browser/src/elbLayer.ts | 56 ++++++- packages/web/sources/browser/src/index.ts | 18 +++ packages/web/sources/browser/src/pageHide.ts | 21 +++ .../dataLayer/src/__tests__/noEnv.test.ts | 55 +++++++ .../sources/dataLayer/src/examples/trigger.ts | 20 +-- packages/web/sources/dataLayer/src/index.ts | 7 +- 35 files changed, 1311 insertions(+), 183 deletions(-) create mode 100644 packages/server/sources/aws/src/lambda/__tests__/acknowledge.test.ts create mode 100644 packages/server/sources/aws/src/sqs/__tests__/acknowledge.test.ts create mode 100644 packages/server/sources/express/src/__tests__/acknowledge.test.ts create mode 100644 packages/server/sources/fetch/src/__tests__/acknowledge.test.ts create mode 100644 packages/server/sources/gcp/src/cloudfunction/__tests__/acknowledge.test.ts create mode 100644 packages/server/sources/gcp/src/pubsub/pull/__tests__/acknowledge.test.ts create mode 100644 packages/server/sources/gcp/src/pubsub/push/__tests__/acknowledge.test.ts create mode 100644 packages/web/sources/browser/src/__tests__/elbLayerFailure.test.ts create mode 100644 packages/web/sources/browser/src/__tests__/pageHide.test.ts create mode 100644 packages/web/sources/browser/src/pageHide.ts create mode 100644 packages/web/sources/dataLayer/src/__tests__/noEnv.test.ts diff --git a/packages/core/src/__tests__/envelope.test.ts b/packages/core/src/__tests__/envelope.test.ts index 54e5f4bff..e8549f540 100644 --- a/packages/core/src/__tests__/envelope.test.ts +++ b/packages/core/src/__tests__/envelope.test.ts @@ -5,6 +5,8 @@ import { pushResultToOutcome, acknowledge, } from '../envelope'; +import type { EventOutcome } from '../envelope'; +import { createEvent } from '../eventGenerator'; import type { Elb } from '../types'; describe('toEventList', () => { @@ -105,22 +107,69 @@ describe('batchResponse', () => { }); describe('pushResultToOutcome', () => { - it.each([ - ['an accepted push', { ok: true, event: { id: 'a' } }, { id: 'a' }], - ['a declined push', { ok: false, error: 'nope' }, { error: 'nope' }], + const event = createEvent({ id: 'a' }); + const ref = (code: string, retryable: boolean) => ({ + type: 'api', + reason: { code, message: `${code} at api`, retryable }, + }); + + it.each<[string, Elb.PushResult, EventOutcome]>([ + ['a delivered push', { ok: true, event }, { id: 'a' }], [ - 'an invalid push with no reason', - { ok: false, invalid: true }, - { error: 'Invalid event' }, + 'a failure a resend cannot fix (ack: answered 200 with the id)', + { ok: false, event, failed: { api: ref('config', false) } }, + { id: 'a' }, ], [ - 'a failed push with no reason', + 'a failure a resend could still deliver', + { ok: false, event, failed: { api: ref('transport', true) } }, + { error: 'Event was not processed', status: 500 }, + ], + [ + 'a retryable transformer failure (its id and text stay in the log)', + { + ok: false, + event, + error: 'Transformer "enrich" failed: enrichment API answered 503', + reason: { code: 'transport', message: 'answered 503', retryable: true }, + }, + { error: 'Event was not processed', status: 500 }, + ], + [ + 'a failure without any reason', { ok: false }, - { error: 'Event was not processed' }, + { error: 'Event was not processed', status: 500 }, + ], + [ + 'invalid input, with the reason the producer can act on', + { ok: false, invalid: true, error: 'Event name is invalid' }, + { error: 'Event name is invalid', status: 400 }, + ], + [ + 'invalid input with no reason', + { ok: false, invalid: true }, + { error: 'Invalid event', status: 400 }, ], ])('%s', (_, result, expected) => { expect(pushResultToOutcome(result)).toEqual(expected); }); + + it('feeds the batch answer: a retryable failure makes it 207, no step named', () => { + const { status, body } = batchResponse([ + pushResultToOutcome({ ok: true, event }), + pushResultToOutcome({ + ok: false, + event, + error: 'Transformer "enrich" failed: enrichment API answered 503', + reason: { code: 'transport', message: 'answered 503', retryable: true }, + }), + ]); + expect(status).toBe(207); + expect(body.errors).toEqual([ + { index: 1, error: 'Event was not processed' }, + ]); + expect(JSON.stringify(body)).not.toMatch(/enrich|503/); + }); }); describe('acknowledge', () => { diff --git a/packages/core/src/envelope.ts b/packages/core/src/envelope.ts index ad958f5c5..4a2fb457e 100644 --- a/packages/core/src/envelope.ts +++ b/packages/core/src/envelope.ts @@ -48,14 +48,29 @@ export function isBatchBody(body: unknown): boolean { } /** - * The outcome of delivering one event of a batch. + * The answer text when a resend could still deliver the event. The step and + * its error stay in the collector's cause line. + */ +export const EVENT_NOT_PROCESSED = 'Event was not processed'; + +/** + * The answer text of a fault that threw, the push included. Its message can + * carry paths, dependency names or config details, so it goes to the log only. + */ +export const INTERNAL_SERVER_ERROR = 'Internal server error'; + +/** + * The outcome of delivering one event, as a producer is answered. * * `id` is present when the event was accepted and minted one. `error` is - * present when it was not accepted. + * present when it was not accepted, with the HTTP `status` a single event is + * answered with: 400 for invalid input, 500 when a resend could still + * deliver it. A batch answers through `batchResponse`. */ export interface EventOutcome { id?: string; error?: string; + status?: 400 | 500; } /** A batch response: the status to answer with, and the body to send. */ @@ -109,25 +124,24 @@ export function batchResponse(outcomes: EventOutcome[]): BatchResponse { } /** - * Maps a collector push result onto a batch event outcome. + * Maps a push result onto the outcome a producer is answered with, the one + * place an HTTP source decides it (`acknowledge`): + * - `ack`: accepted, with the event id. This includes a failure a resend + * cannot fix: the event is in the DLQ and its cause line is logged. + * - `retry`: 500 with a fixed text. The step and its error stay in the + * collector's cause line; a public client cannot act on them. + * - `reject`: 400 with the reason, which describes the producer's own input. * - * @param result The push result, or undefined when the push returned none. + * @param result The push result. * @returns The outcome for this event. */ -export function pushResultToOutcome(result: { - ok?: boolean; - invalid?: boolean; - error?: string; - event?: { id?: string }; -}): EventOutcome { - if (result?.ok === false) { - return { - error: - result.error ?? - (result.invalid === true ? 'Invalid event' : 'Event was not processed'), - }; - } - return { id: result?.event?.id }; +export function pushResultToOutcome(result: Elb.PushResult): EventOutcome { + const acknowledgement = acknowledge(result); + if (acknowledgement === 'reject') + return { error: result.error ?? 'Invalid event', status: 400 }; + if (acknowledgement === 'retry') + return { error: EVENT_NOT_PROCESSED, status: 500 }; + return { id: result.event?.id }; } /** What a producer does with the message that carried a push. */ diff --git a/packages/server/sources/aws/src/lambda/__tests__/acknowledge.test.ts b/packages/server/sources/aws/src/lambda/__tests__/acknowledge.test.ts new file mode 100644 index 000000000..d93aee73a --- /dev/null +++ b/packages/server/sources/aws/src/lambda/__tests__/acknowledge.test.ts @@ -0,0 +1,98 @@ +import { StepError } from '@walkeros/core'; +import type { Destination } from '@walkeros/core'; +import { createTrigger } from '../examples/trigger'; +import type { Result } from '../examples/trigger'; +import { sourceLambda } from '../index'; +import { createMockEventV2 } from './fixtures'; + +// The HTTP answer to a push, proven on a real collector: 200 unless a resend +// could still deliver the event (500, a batch 207) or the input is invalid +// (400). The body never names a destination. + +const failing = (error: Error): Destination.Instance => ({ + type: 'failing', + config: {}, + push: () => { + throw error; + }, +}); + +const delivering: Destination.Instance = { + type: 'delivering', + config: {}, + push: () => undefined, +}; + +const deterministic = () => + new StepError('setting apiKey is missing', { code: 'config' }); +const retryable = () => + new StepError('vendor answered 503', { code: 'transport', status: 503 }); + +async function post( + destination: Destination.Instance, + body: unknown, +): Promise { + const instance = await createTrigger({ + logger: { handler: () => undefined }, + sources: { lambda: { code: sourceLambda } }, + destinations: { vendorX: { code: destination } }, + }); + try { + return await instance.trigger()({ + ...createMockEventV2('POST', JSON.stringify(body)), + }); + } finally { + await instance.flow?.collector.command('shutdown'); + } +} + +describe('Lambda acknowledgement', () => { + it('answers 200 for a delivered event', async () => { + const result = await post(delivering, { name: 'page view' }); + expect(result.statusCode).toBe(200); + expect(result.body).toMatchObject({ success: true }); + }); + + it('answers 200 with the success body for a failure a resend cannot fix', async () => { + const result = await post(failing(deterministic()), { name: 'page view' }); + expect(result.statusCode).toBe(200); + expect(result.body).toEqual({ + success: true, + id: expect.any(String), + requestId: expect.any(String), + }); + expect(JSON.stringify(result.body)).not.toContain('vendorX'); + }); + + it('answers 500 when a resend could still deliver the event', async () => { + const result = await post(failing(retryable()), { name: 'page view' }); + expect(result.statusCode).toBe(500); + expect(result.body).toMatchObject({ + success: false, + error: 'Event was not processed', + }); + expect(JSON.stringify(result.body)).not.toContain('vendorX'); + }); + + it('answers 400 for invalid input', async () => { + const result = await post(delivering, { name: 'invalid' }); + expect(result.statusCode).toBe(400); + }); + + it('answers a batch of deterministic failures with 200', async () => { + const result = await post(failing(deterministic()), { + batch: [{ name: 'page view' }, { name: 'order complete' }], + }); + expect(result.statusCode).toBe(200); + expect(result.body).toMatchObject({ success: true, processed: 2 }); + }); + + it('answers a batch with a retryable failure with 207', async () => { + const result = await post(failing(retryable()), { + batch: [{ name: 'page view' }, { name: 'order complete' }], + }); + expect(result.statusCode).toBe(207); + expect(result.body).toMatchObject({ success: false, failed: 2 }); + expect(JSON.stringify(result.body)).not.toContain('vendorX'); + }); +}); diff --git a/packages/server/sources/aws/src/lambda/__tests__/index.test.ts b/packages/server/sources/aws/src/lambda/__tests__/index.test.ts index ee4a255b5..085b4e28f 100644 --- a/packages/server/sources/aws/src/lambda/__tests__/index.test.ts +++ b/packages/server/sources/aws/src/lambda/__tests__/index.test.ts @@ -537,12 +537,18 @@ describe('sourceLambda', () => { // A rejected push is a server fault, not client input: the collector // resolves pipeline failures, so a rejection is exceptional. + // The thrown message goes to the log, never into the answer. expect(result.statusCode).toBe(500); expect(JSON.parse(result.body)).toMatchObject({ success: false, - error: 'Processing failed', + error: 'Internal server error', requestId: expect.any(String), }); + expect(result.body).not.toContain('Processing failed'); + expect(mockLogger.error).toHaveBeenCalledWith( + 'Event processing failed', + expect.objectContaining({ error: expect.any(Error) }), + ); }); it('should reject unsupported methods', async () => { diff --git a/packages/server/sources/aws/src/lambda/index.ts b/packages/server/sources/aws/src/lambda/index.ts index 1b3388179..f665ff117 100644 --- a/packages/server/sources/aws/src/lambda/index.ts +++ b/packages/server/sources/aws/src/lambda/index.ts @@ -1,6 +1,7 @@ import type { LambdaSource, Types } from './types'; import type { Source } from '@walkeros/core'; import { + INTERNAL_SERVER_ERROR, batchResponse, isBatchBody, requestToData, @@ -173,12 +174,7 @@ export const sourceLambda: Source.Init = async (context) => { }); return createResponse( 500, - { - success: false, - error: - error instanceof Error ? error.message : 'Internal server error', - requestId, - }, + { success: false, error: INTERNAL_SERVER_ERROR, requestId }, {}, requestId, ); diff --git a/packages/server/sources/aws/src/lambda/push.ts b/packages/server/sources/aws/src/lambda/push.ts index 2d6792249..1c650b68e 100644 --- a/packages/server/sources/aws/src/lambda/push.ts +++ b/packages/server/sources/aws/src/lambda/push.ts @@ -1,5 +1,9 @@ -import type { Collector, Logger } from '@walkeros/core'; -import { isObject } from '@walkeros/core'; +import type { Collector, EventOutcome, Logger } from '@walkeros/core'; +import { + INTERNAL_SERVER_ERROR, + isObject, + pushResultToOutcome, +} from '@walkeros/core'; import type { EventRequest } from './types'; export async function processEvent( @@ -7,7 +11,7 @@ export async function processEvent( push: Collector.PushFn, logger?: Logger.Instance, requestId?: string, -): Promise<{ id?: string; error?: string; status?: number }> { +): Promise { try { // A batch element that is not an object carries no event fields. Treat it // as an empty event so the collector's gate classifies it as invalid @@ -15,17 +19,7 @@ export async function processEvent( const { event, ...rest } = isObject(eventReq) ? eventReq : {}; const result = await push({ ...rest, name: rest.name ?? event }); - if (result?.ok === false) { - if (result.invalid === true) { - return { error: result.error ?? 'Invalid event', status: 400 }; - } - return { - error: result.error ?? 'Event was not processed', - status: 500, - }; - } - - return { id: result?.event?.id }; + return pushResultToOutcome(result); } catch (error) { // Log with structured context - per using-logger skill logger?.error('Event processing failed', { @@ -34,10 +28,8 @@ export async function processEvent( requestId, }); // A rejected push promise is a server fault: the collector resolves - // pipeline failures, so a rejection is exceptional by construction. - return { - error: error instanceof Error ? error.message : 'Unknown error', - status: 500, - }; + // pipeline failures, so a rejection is exceptional by construction. Its + // message is logged above, never answered. + return { error: INTERNAL_SERVER_ERROR, status: 500 }; } } diff --git a/packages/server/sources/aws/src/sqs/__tests__/acknowledge.test.ts b/packages/server/sources/aws/src/sqs/__tests__/acknowledge.test.ts new file mode 100644 index 000000000..3648e0178 --- /dev/null +++ b/packages/server/sources/aws/src/sqs/__tests__/acknowledge.test.ts @@ -0,0 +1,141 @@ +import type { + CreateQueueCommandOutput, + DeleteMessageCommandOutput, + GetQueueAttributesCommandOutput, + GetQueueUrlCommandOutput, + ReceiveMessageCommandOutput, +} from '@aws-sdk/client-sqs'; +import { Level, StepError } from '@walkeros/core'; +import type { Collector, Destination } from '@walkeros/core'; +import { createTrigger } from '../examples/trigger'; +import { sourceSqs } from '../index'; +import type { SqsClient } from '../types'; + +// Acknowledgement of a consumed message, proven on a real collector: the +// source deletes a message unless a redelivery could still deliver it. + +type SqsResponse = GetQueueUrlCommandOutput & + GetQueueAttributesCommandOutput & + ReceiveMessageCommandOutput & + DeleteMessageCommandOutput & + CreateQueueCommandOutput; + +/** Answers every command with an empty response; the loop idles. */ +class IdleClient implements SqsClient { + async send(): Promise { + await new Promise((resolve) => setTimeout(resolve, 10)); + return { $metadata: {}, Messages: [] }; + } + + destroy(): void {} +} + +const failing = (error: Error): Destination.Instance => ({ + type: 'failing', + config: {}, + push: () => { + throw error; + }, +}); + +const delivering: Destination.Instance = { + type: 'delivering', + config: {}, + push: () => undefined, +}; + +const deterministic = () => + new StepError('setting apiKey is missing', { code: 'config' }); +const retryable = () => + new StepError('vendor answered 503', { code: 'transport', status: 503 }); + +const lines: Array<[Level, string]> = []; + +async function consume( + destination: Destination.Instance, + body: unknown, + onPushError?: 'ack' | 'nack', +): Promise<{ + result: Array<[string, ...unknown[]]>; + status: Collector.Status; +}> { + const instance = await createTrigger({ + logger: { handler: (level, message) => void lines.push([level, message]) }, + sources: { + sqs: { + code: sourceSqs, + config: { + settings: { + queueName: 'walkeros-events', + queueUrl: + 'https://sqs.eu-central-1.amazonaws.com/000000000000/walkeros-events', + client: new IdleClient(), + ...(onPushError ? { onPushError } : {}), + }, + }, + }, + }, + destinations: { vendorX: { code: destination } }, + }); + try { + const result = await instance.trigger()({ + MessageId: 'm1', + Body: JSON.stringify(body), + ReceiptHandle: 'r1', + }); + const status = instance.flow?.collector.status; + if (!status) throw new Error('flow did not start'); + return { result, status }; + } finally { + await instance.flow?.collector.command('shutdown'); + } +} + +describe('SQS acknowledgement', () => { + beforeEach(() => { + lines.length = 0; + }); + + it('deletes a delivered message', async () => { + const { result } = await consume(delivering, { name: 'page view' }); + expect(result).toEqual([['message.ack', 'm1']]); + }); + + it('deletes a message whose failure a redelivery cannot fix', async () => { + const { result, status } = await consume(failing(deterministic()), { + name: 'page view', + }); + expect(result).toEqual([['message.ack', 'm1']]); + expect(status.destinations.vendorX.failed).toBe(1); + // The failure is visible at the default level through the collector. + expect(lines.some(([level]) => level === Level.ERROR)).toBe(true); + }); + + it('keeps a message a redelivery could still deliver', async () => { + const { result } = await consume(failing(retryable()), { + name: 'page view', + }); + expect(result).toEqual([['message.nack', 'm1']]); + // One cause line from the collector, no per-message handler error. + expect(lines).toContainEqual([Level.ERROR, expect.any(String)]); + expect(lines).not.toContainEqual([ + Level.ERROR, + 'SQS message handler failed', + ]); + }); + + it("deletes a retryable failure when onPushError is 'ack'", async () => { + const { result } = await consume( + failing(retryable()), + { name: 'page view' }, + 'ack', + ); + expect(result).toEqual([['message.ack', 'm1']]); + }); + + it('deletes invalid input and counts it as rejected', async () => { + const { result, status } = await consume(delivering, { name: 'invalid' }); + expect(result).toEqual([['message.ack', 'm1']]); + expect(status.sources.sqs.rejected).toBe(1); + }); +}); diff --git a/packages/server/sources/aws/src/sqs/index.ts b/packages/server/sources/aws/src/sqs/index.ts index 771563c30..5af1e757b 100644 --- a/packages/server/sources/aws/src/sqs/index.ts +++ b/packages/server/sources/aws/src/sqs/index.ts @@ -4,6 +4,7 @@ import { GetQueueUrlCommand, ReceiveMessageCommand, } from '@aws-sdk/client-sqs'; +import { acknowledge } from '@walkeros/core'; import type { Collector, Logger, Source, WalkerOS } from '@walkeros/core'; import type { Settings, @@ -48,6 +49,9 @@ async function handleMessage(args: { logger: Logger.Instance; }): Promise { const { message, settings, pushFn, logger } = args; + // The error disposition: nack (redeliver) by default, ack if configured. + const dispose = () => + settings.onPushError === 'ack' ? message.ack() : message.nack(); try { const decoded = decodeBody( message.id, @@ -62,11 +66,15 @@ async function handleMessage(args: { return; } const result = await pushFn(toDeepPartialEvent(decoded)); - if (result?.ok === false) { - // Never delete a message that was not delivered. Route through the - // existing error disposition (nack by default, ack if configured). - throw new Error('push rejected the message (ok:false)'); + if (acknowledge(result) === 'retry') { + // Never delete a message a redelivery could still deliver. The + // collector logged the cause; this line is the per-message trace. + logger.debug('SQS message not delivered, retryable', { id: message.id }); + await dispose(); + return; } + // Delivered, a deterministic failure (in the DLQ, its cause logged) or + // rejected input (counted): a redelivery cannot change the outcome. await message.ack(); } catch (err) { const error = err instanceof Error ? err.message : String(err); @@ -75,8 +83,7 @@ async function handleMessage(args: { } else { logger.error('SQS message handler failed', { id: message.id, error }); } - if (settings.onPushError === 'ack') await message.ack(); - else await message.nack(); + await dispose(); } } diff --git a/packages/server/sources/aws/src/sqs/schemas/settings.ts b/packages/server/sources/aws/src/sqs/schemas/settings.ts index 6b5c2ea54..ff9b2d134 100644 --- a/packages/server/sources/aws/src/sqs/schemas/settings.ts +++ b/packages/server/sources/aws/src/sqs/schemas/settings.ts @@ -75,7 +75,7 @@ export const SettingsSchema = z.object({ onPushError: z .enum(['nack', 'ack']) .describe( - 'Behavior when forwarding to the collector throws. nack (default) skips DeleteMessage so the message redelivers; ack drops it.', + 'Behavior when a message is not delivered and a redelivery could still deliver it (a retryable failure), or when decoding or forwarding throws. nack (default) skips DeleteMessage so the message redelivers; ack drops it.', ) .optional(), }); diff --git a/packages/server/sources/express/src/__tests__/acknowledge.test.ts b/packages/server/sources/express/src/__tests__/acknowledge.test.ts new file mode 100644 index 000000000..e2a78742a --- /dev/null +++ b/packages/server/sources/express/src/__tests__/acknowledge.test.ts @@ -0,0 +1,102 @@ +import { StepError } from '@walkeros/core'; +import type { Destination, Trigger } from '@walkeros/core'; +import { examples } from '../dev'; +import type { Content, Result } from '../examples/trigger'; +import { sourceExpress } from '../index'; + +// The synchronous HTTP answer to a push, proven on a real collector and a +// real socket: 200 unless a resend could still deliver the event (500, a +// batch 207) or the input is invalid (400). The body never names a +// destination. + +const failing = (error: Error): Destination.Instance => ({ + type: 'failing', + config: {}, + push: () => { + throw error; + }, +}); + +const delivering: Destination.Instance = { + type: 'delivering', + config: {}, + push: () => undefined, +}; + +const deterministic = () => + new StepError('setting apiKey is missing', { code: 'config' }); +const retryable = () => + new StepError('vendor answered 503', { code: 'transport', status: 503 }); + +describe('Express acknowledgement', () => { + let instance: Trigger.Instance | undefined; + + afterEach(async () => { + await instance?.flow?.collector.command('shutdown'); + instance = undefined; + }); + + const post = async ( + destination: Destination.Instance, + body: unknown, + ): Promise => { + instance = await examples.createTrigger({ + logger: { handler: () => undefined }, + sources: { + express: { code: sourceExpress, config: { async: false } }, + }, + destinations: { vendorX: { code: destination } }, + }); + return instance.trigger()({ method: 'POST', path: '/collect', body }); + }; + + it('answers 200 for a delivered event', async () => { + const result = await post(delivering, { name: 'page view' }); + expect(result.status).toBe(200); + expect(result.body).toEqual({ + success: true, + timestamp: expect.any(Number), + }); + }); + + it('answers 200 with the success body for a failure a resend cannot fix', async () => { + const result = await post(failing(deterministic()), { name: 'page view' }); + expect(result.status).toBe(200); + expect(result.body).toEqual({ + success: true, + timestamp: expect.any(Number), + }); + }); + + it('answers 500 when a resend could still deliver the event', async () => { + const result = await post(failing(retryable()), { name: 'page view' }); + expect(result.status).toBe(500); + expect(result.body).toEqual({ + success: false, + error: 'Event was not processed', + timestamp: expect.any(Number), + }); + }); + + it('answers 400 for invalid input', async () => { + const result = await post(delivering, { name: 'invalid' }); + expect(result.status).toBe(400); + }); + + it('answers a batch of deterministic failures with 200', async () => { + const result = await post(failing(deterministic()), { + batch: [{ name: 'page view' }, { name: 'order complete' }], + }); + expect(result.status).toBe(200); + expect(result.body).toMatchObject({ success: true, processed: 2 }); + }); + + it('answers a batch with a retryable failure with 207', async () => { + const result = await post(failing(retryable()), { + batch: [{ name: 'page view' }, { name: 'order complete' }], + }); + expect(result.status).toBe(207); + expect(result.body).toMatchObject({ success: false, failed: 2 }); + expect(JSON.stringify(result.body)).not.toContain('vendorX'); + }); +}); diff --git a/packages/server/sources/express/src/__tests__/index.test.ts b/packages/server/sources/express/src/__tests__/index.test.ts index 7e003fd8a..95056c88a 100644 --- a/packages/server/sources/express/src/__tests__/index.test.ts +++ b/packages/server/sources/express/src/__tests__/index.test.ts @@ -451,6 +451,7 @@ describe('sourceExpress', () => { success: false, error: 'Internal server error', }); + expect(JSON.stringify(res.responseBody)).not.toContain('Collector error'); }); }); diff --git a/packages/server/sources/express/src/__tests__/input-hygiene.test.ts b/packages/server/sources/express/src/__tests__/input-hygiene.test.ts index 856781ef6..7317298bd 100644 --- a/packages/server/sources/express/src/__tests__/input-hygiene.test.ts +++ b/packages/server/sources/express/src/__tests__/input-hygiene.test.ts @@ -171,22 +171,22 @@ describe('express input hygiene', () => { } }); - it('logs a rejected body once at debug with a truncated sample, never at error', async () => { + it('logs a rejected body once at debug without its content, never at error', async () => { const logger = createMockLogger(); const { url, close } = await startServer(liveContext(logger)); try { await fetch(`${url}/collect`, { method: 'POST', headers: { 'content-type': 'application/json' }, - body: 'a'.repeat(1000), + body: `{"email":"${'a'.repeat(1000)}`, }); expect(logger.error).not.toHaveBeenCalled(); expect(logger.debug).toHaveBeenCalledTimes(1); - const [message, context]: [string, { status: number; sample?: string }] = + const [message, context]: [string, { status: number }] = logger.debug.mock.calls[0]; expect(message).toBe('Request body rejected'); - expect(context.status).toBe(400); - expect(context.sample).toHaveLength(200); + expect(context).toEqual({ status: 400, type: 'entity.parse.failed' }); + expect(JSON.stringify(context)).not.toMatch(/email|aaaa/); } finally { await close(); } diff --git a/packages/server/sources/express/src/index.ts b/packages/server/sources/express/src/index.ts index f25fa6a29..fa2a9ba5c 100644 --- a/packages/server/sources/express/src/index.ts +++ b/packages/server/sources/express/src/index.ts @@ -5,6 +5,7 @@ import express, { } from 'express'; import cors from 'cors'; import { + INTERNAL_SERVER_ERROR, requestToData, createRespond, toEventList, @@ -305,7 +306,9 @@ export const sourceExpress = async ( // Synchronous ack: wait for delivery to settle before responding, // and reflect the outcome. Invalid input is the client's fault - // (400 with the reason); anything else that failed is ours (500). + // (400 with the reason); a failure a resend could still deliver is + // ours (500); a failure a resend cannot fix is answered 200, so the + // client does not resend it to every healthy destination. const results = []; for (const event of events) { results.push(await env.push(event)); @@ -319,16 +322,13 @@ export const sourceExpress = async ( return; } - const result = results[0]; - if (result?.ok === false) { - const invalid = result.invalid === true; + const outcome = pushResultToOutcome(results[0]); + if (outcome.error) { respond({ - status: invalid ? 400 : 500, + status: outcome.status, body: { success: false, - error: - result.error ?? - (invalid ? 'Invalid event' : 'Event was not processed'), + error: outcome.error, timestamp: Date.now(), }, }); @@ -353,7 +353,7 @@ export const sourceExpress = async ( if (!res.headersSent) { res.status(500).json({ success: false, - error: 'Internal server error', + error: INTERNAL_SERVER_ERROR, }); } } @@ -398,18 +398,17 @@ export const sourceExpress = async ( } if (isRequestBodyError(err)) { countRejected(); + // Status and type only: the body and the parser's message (which + // quotes the body) can carry personal data. env.logger.debug('Request body rejected', { status: err.status, type: err.type, - error: err.message, - sample: - typeof err.body === 'string' ? err.body.slice(0, 200) : undefined, }); res.status(err.status).json({ success: false, error: err.message }); return; } env.logger.error(toError(err)); - res.status(500).json({ success: false, error: 'Internal server error' }); + res.status(500).json({ success: false, error: INTERNAL_SERVER_ERROR }); }); // Source owns the HTTP server (if port configured) diff --git a/packages/server/sources/fetch/src/__tests__/acknowledge.test.ts b/packages/server/sources/fetch/src/__tests__/acknowledge.test.ts new file mode 100644 index 000000000..96da1b020 --- /dev/null +++ b/packages/server/sources/fetch/src/__tests__/acknowledge.test.ts @@ -0,0 +1,97 @@ +import { StepError } from '@walkeros/core'; +import type { Destination } from '@walkeros/core'; +import { examples } from '../dev'; +import type { Result } from '../examples/trigger'; +import { sourceFetch } from '../index'; + +// The HTTP answer to a push, proven on a real collector: 200 unless a resend +// could still deliver the event (500, a batch 207) or the input is invalid +// (400). The body never names a destination. + +const failing = (error: Error): Destination.Instance => ({ + type: 'failing', + config: {}, + push: () => { + throw error; + }, +}); + +const delivering: Destination.Instance = { + type: 'delivering', + config: {}, + push: () => undefined, +}; + +const deterministic = () => + new StepError('setting apiKey is missing', { code: 'config' }); +const retryable = () => + new StepError('vendor answered 503', { code: 'transport', status: 503 }); + +async function post( + destination: Destination.Instance, + body: unknown, +): Promise { + const instance = await examples.createTrigger({ + logger: { handler: () => undefined }, + sources: { fetch: { code: sourceFetch } }, + destinations: { vendorX: { code: destination } }, + }); + try { + return await instance.trigger()({ + method: 'POST', + url: 'https://collect.example/collect', + body, + }); + } finally { + await instance.flow?.collector.command('shutdown'); + } +} + +describe('Fetch acknowledgement', () => { + it('answers 200 for a delivered event', async () => { + const result = await post(delivering, { name: 'page view' }); + expect(result.status).toBe(200); + expect(result.body).toMatchObject({ success: true }); + }); + + it('answers 200 with the success body for a failure a resend cannot fix', async () => { + const result = await post(failing(deterministic()), { name: 'page view' }); + expect(result.status).toBe(200); + expect(result.body).toEqual({ + success: true, + id: expect.any(String), + timestamp: expect.any(Number), + }); + }); + + it('answers 500 when a resend could still deliver the event', async () => { + const result = await post(failing(retryable()), { name: 'page view' }); + expect(result.status).toBe(500); + expect(result.body).toEqual({ + success: false, + error: 'Event was not processed', + }); + }); + + it('answers 400 for invalid input', async () => { + const result = await post(delivering, { name: 'invalid' }); + expect(result.status).toBe(400); + }); + + it('answers a batch of deterministic failures with 200', async () => { + const result = await post(failing(deterministic()), { + batch: [{ name: 'page view' }, { name: 'order complete' }], + }); + expect(result.status).toBe(200); + expect(result.body).toMatchObject({ success: true, processed: 2 }); + }); + + it('answers a batch with a retryable failure with 207', async () => { + const result = await post(failing(retryable()), { + batch: [{ name: 'page view' }, { name: 'order complete' }], + }); + expect(result.status).toBe(207); + expect(result.body).toMatchObject({ success: false, failed: 2 }); + expect(JSON.stringify(result.body)).not.toContain('vendorX'); + }); +}); diff --git a/packages/server/sources/fetch/src/__tests__/envelope.test.ts b/packages/server/sources/fetch/src/__tests__/envelope.test.ts index 53fff016f..2d72ad914 100644 --- a/packages/server/sources/fetch/src/__tests__/envelope.test.ts +++ b/packages/server/sources/fetch/src/__tests__/envelope.test.ts @@ -126,7 +126,8 @@ describe('fetch envelope contract', () => { success: false, processed: 1, failed: 1, - errors: [{ index: 1, error: 'nope' }], + // A retryable failure answers a fixed text; its own lives in the log. + errors: [{ index: 1, error: 'Event was not processed' }], }); }); diff --git a/packages/server/sources/fetch/src/__tests__/index.test.ts b/packages/server/sources/fetch/src/__tests__/index.test.ts index 555161a09..78a039edc 100644 --- a/packages/server/sources/fetch/src/__tests__/index.test.ts +++ b/packages/server/sources/fetch/src/__tests__/index.test.ts @@ -248,11 +248,13 @@ describe('sourceFetch', () => { // A rejected push is a server fault, not client input: the collector // resolves pipeline failures, so a rejection is exceptional. + // The thrown message goes to the log, never into the answer. expect(response.status).toBe(500); expect(responseBody).toMatchObject({ success: false, - error: 'Collector error', + error: 'Internal server error', }); + expect(JSON.stringify(responseBody)).not.toContain('Collector error'); }); it('answers 400 with the reason when the push settles an invalid-input rejection', async () => { @@ -506,7 +508,7 @@ describe('sourceFetch', () => { it('should process examples.inputs.pageView', async () => { const mockPush = jest .fn() - .mockResolvedValue({ event: { id: 'test-id' } }); + .mockResolvedValue({ ok: true, event: { id: 'test-id' } }); const source = await sourceFetch( createSourceContext( diff --git a/packages/server/sources/fetch/src/__tests__/logging.test.ts b/packages/server/sources/fetch/src/__tests__/logging.test.ts index d93726164..2434899cd 100644 --- a/packages/server/sources/fetch/src/__tests__/logging.test.ts +++ b/packages/server/sources/fetch/src/__tests__/logging.test.ts @@ -34,7 +34,7 @@ describe('logger usage', () => { const mockLogger = createMockLogger(); const mockPush = jest .fn() - .mockResolvedValue({ event: { id: 'forwarded' } }); + .mockResolvedValue({ ok: true, event: { id: 'forwarded' } }); const source = await sourceFetch( createSourceContext( @@ -69,7 +69,9 @@ describe('logger usage', () => { createSourceContext( {}, { - push: jest.fn().mockResolvedValue({ event: { id: 'test' } }) as never, + push: jest + .fn() + .mockResolvedValue({ ok: true, event: { id: 'test' } }) as never, command: jest.fn() as never, elb: jest.fn() as never, logger: mockLogger, diff --git a/packages/server/sources/fetch/src/index.ts b/packages/server/sources/fetch/src/index.ts index 662ff324b..153173f15 100644 --- a/packages/server/sources/fetch/src/index.ts +++ b/packages/server/sources/fetch/src/index.ts @@ -1,4 +1,5 @@ import { + INTERNAL_SERVER_ERROR, normalizeBody, requestToData, isObject, @@ -227,11 +228,7 @@ export const sourceFetch: Source.Init = async (context) => { logger.error('Internal server error', error); const corsHeaders = createCorsHeaders(settings.cors); return createJsonResponse( - { - success: false, - error: - error instanceof Error ? error.message : 'Internal server error', - }, + { success: false, error: INTERNAL_SERVER_ERROR }, 500, corsHeaders, ); @@ -245,19 +242,9 @@ async function processEvent( event: WalkerOS.DeepPartialEvent, push: Collector.PushFn, logger: Logger.Instance, -): Promise<{ id?: string; error?: string; status?: number }> { +): Promise { try { - const result = await push(event); - if (result?.ok === false) { - if (result.invalid === true) { - return { error: result.error ?? 'Invalid event', status: 400 }; - } - return { - error: result.error ?? 'Event was not processed', - status: 500, - }; - } - return { id: result?.event?.id }; + return pushResultToOutcome(await push(event)); } catch (error) { // A rejected push promise is a server fault: the collector resolves // pipeline failures, so a rejection is exceptional by construction. @@ -266,7 +253,7 @@ async function processEvent( // in the error channel. const message = error instanceof Error ? error.message : 'Unknown error'; logger.error('Event processing failed', { error: message }); - return { error: message, status: 500 }; + return { error: INTERNAL_SERVER_ERROR, status: 500 }; } } @@ -282,13 +269,11 @@ async function processBatch( try { const result = await push(event as WalkerOS.DeepPartialEvent); - const outcome = pushResultToOutcome(result ?? {}); + const outcome = pushResultToOutcome(result); if (outcome.error) logger.warn(`Batch event ${i} not processed`); outcomes.push(outcome); } catch (error) { - outcomes.push({ - error: error instanceof Error ? error.message : 'Unknown error', - }); + outcomes.push({ error: INTERNAL_SERVER_ERROR }); logger.warn(`Batch event ${i} processing failed`, error); } } diff --git a/packages/server/sources/gcp/src/cloudfunction/__tests__/acknowledge.test.ts b/packages/server/sources/gcp/src/cloudfunction/__tests__/acknowledge.test.ts new file mode 100644 index 000000000..e675ce344 --- /dev/null +++ b/packages/server/sources/gcp/src/cloudfunction/__tests__/acknowledge.test.ts @@ -0,0 +1,89 @@ +import { StepError } from '@walkeros/core'; +import type { Destination } from '@walkeros/core'; +import { createTrigger } from '../examples/trigger'; +import type { Result } from '../examples/trigger'; +import { sourceCloudFunction } from '../index'; + +// The HTTP answer to a push, proven on a real collector: 200 unless a resend +// could still deliver the event (500, a batch 207) or the input is invalid +// (400). The body never names a destination. + +const failing = (error: Error): Destination.Instance => ({ + type: 'failing', + config: {}, + push: () => { + throw error; + }, +}); + +const delivering: Destination.Instance = { + type: 'delivering', + config: {}, + push: () => undefined, +}; + +const deterministic = () => + new StepError('setting apiKey is missing', { code: 'config' }); +const retryable = () => + new StepError('vendor answered 503', { code: 'transport', status: 503 }); + +async function post( + destination: Destination.Instance, + body: Record, +): Promise { + const instance = await createTrigger({ + logger: { handler: () => undefined }, + sources: { cloudfunction: { code: sourceCloudFunction } }, + destinations: { vendorX: { code: destination } }, + }); + try { + return await instance.trigger()({ method: 'POST', body }); + } finally { + await instance.flow?.collector.command('shutdown'); + } +} + +describe('Cloud Function acknowledgement', () => { + it('answers 200 for a delivered event', async () => { + const result = await post(delivering, { name: 'page view' }); + expect(result.status).toBe(200); + expect(result.body).toMatchObject({ success: true }); + }); + + it('answers 200 with the success body for a failure a resend cannot fix', async () => { + const result = await post(failing(deterministic()), { name: 'page view' }); + expect(result.status).toBe(200); + expect(result.body).toEqual({ success: true, id: expect.any(String) }); + }); + + it('answers 500 when a resend could still deliver the event', async () => { + const result = await post(failing(retryable()), { name: 'page view' }); + expect(result.status).toBe(500); + expect(result.body).toEqual({ + success: false, + error: 'Event was not processed', + }); + }); + + it('answers 400 for invalid input', async () => { + const result = await post(delivering, { name: 'invalid' }); + expect(result.status).toBe(400); + }); + + it('answers a batch of deterministic failures with 200', async () => { + const result = await post(failing(deterministic()), { + batch: [{ name: 'page view' }, { name: 'order complete' }], + }); + expect(result.status).toBe(200); + expect(result.body).toMatchObject({ success: true, processed: 2 }); + }); + + it('answers a batch with a retryable failure with 207', async () => { + const result = await post(failing(retryable()), { + batch: [{ name: 'page view' }, { name: 'order complete' }], + }); + expect(result.status).toBe(207); + expect(result.body).toMatchObject({ success: false, failed: 2 }); + expect(JSON.stringify(result.body)).not.toContain('vendorX'); + }); +}); diff --git a/packages/server/sources/gcp/src/cloudfunction/__tests__/index.test.ts b/packages/server/sources/gcp/src/cloudfunction/__tests__/index.test.ts index bdbe695fd..916d5499e 100644 --- a/packages/server/sources/gcp/src/cloudfunction/__tests__/index.test.ts +++ b/packages/server/sources/gcp/src/cloudfunction/__tests__/index.test.ts @@ -365,11 +365,16 @@ describe('sourceCloudFunction', () => { // A rejected push is a server fault, not client input: the collector // resolves pipeline failures, so a rejection is exceptional. + // The thrown message goes to the log, never into the answer. expect(res.status).toHaveBeenCalledWith(500); expect(res.json).toHaveBeenCalledWith({ success: false, - error: 'Processing failed', + error: 'Internal server error', }); + expect(mockLogger.error).toHaveBeenCalledWith( + 'Event processing failed', + expect.objectContaining({ error: expect.any(Error) }), + ); }); }); diff --git a/packages/server/sources/gcp/src/cloudfunction/index.ts b/packages/server/sources/gcp/src/cloudfunction/index.ts index 53d90d816..6844aa0ab 100644 --- a/packages/server/sources/gcp/src/cloudfunction/index.ts +++ b/packages/server/sources/gcp/src/cloudfunction/index.ts @@ -8,6 +8,7 @@ import type { } from './types'; import type { Source } from '@walkeros/core'; import { + INTERNAL_SERVER_ERROR, batchResponse, isBatchBody, requestToData, @@ -103,7 +104,7 @@ export const sourceCloudFunction: Source.Init = async (context) => { const results = []; for (const event of events) { - results.push(await processEvent(event, envPush)); + results.push(await processEvent(event, envPush, context.logger)); } if (batched) { @@ -127,10 +128,9 @@ export const sourceCloudFunction: Source.Init = async (context) => { } as EventResponse); }); } catch (error) { - res.status(500).json({ - success: false, - error: error instanceof Error ? error.message : 'Internal server error', - }); + // The reason is logged, not answered (see INTERNAL_SERVER_ERROR). + context.logger.error('Cloud Function handler error', { error }); + res.status(500).json({ success: false, error: INTERNAL_SERVER_ERROR }); } }; diff --git a/packages/server/sources/gcp/src/cloudfunction/push.ts b/packages/server/sources/gcp/src/cloudfunction/push.ts index 17db5538f..bb35a7b79 100644 --- a/packages/server/sources/gcp/src/cloudfunction/push.ts +++ b/packages/server/sources/gcp/src/cloudfunction/push.ts @@ -1,11 +1,16 @@ -import type { Collector } from '@walkeros/core'; -import { isObject } from '@walkeros/core'; +import type { Collector, EventOutcome, Logger } from '@walkeros/core'; +import { + INTERNAL_SERVER_ERROR, + isObject, + pushResultToOutcome, +} from '@walkeros/core'; import type { EventRequest } from './types'; export async function processEvent( eventReq: EventRequest, push: Collector.PushFn, -): Promise<{ id?: string; error?: string; status?: number }> { + logger: Logger.Instance, +): Promise { try { // A batch element that is not an object carries no event fields. Treat it // as an empty event so the collector's gate classifies it as invalid @@ -13,23 +18,17 @@ export async function processEvent( const { event, ...rest } = isObject(eventReq) ? eventReq : {}; const result = await push({ ...rest, name: rest.name ?? event }); - if (result?.ok === false) { - if (result.invalid === true) { - return { error: result.error ?? 'Invalid event', status: 400 }; - } - return { - error: result.error ?? 'Event was not processed', - status: 500, - }; - } - - return { id: result?.event?.id }; + return pushResultToOutcome(result); } catch (error) { // A rejected push promise is a server fault: the collector resolves - // pipeline failures, so a rejection is exceptional by construction. - return { - error: error instanceof Error ? error.message : 'Unknown error', - status: 500, - }; + // pipeline failures, so a rejection is exceptional by construction. Its + // message is logged, never answered. + logger.error('Event processing failed', { + error, + eventName: isObject(eventReq) + ? (eventReq.name ?? eventReq.event) + : undefined, + }); + return { error: INTERNAL_SERVER_ERROR, status: 500 }; } } diff --git a/packages/server/sources/gcp/src/pubsub/pull/__tests__/acknowledge.test.ts b/packages/server/sources/gcp/src/pubsub/pull/__tests__/acknowledge.test.ts new file mode 100644 index 000000000..09eafdf22 --- /dev/null +++ b/packages/server/sources/gcp/src/pubsub/pull/__tests__/acknowledge.test.ts @@ -0,0 +1,119 @@ +import { Level, StepError } from '@walkeros/core'; +import type { Collector, Destination } from '@walkeros/core'; +import { createTrigger } from '../examples/trigger'; +import type { Result } from '../examples/trigger'; +import { sourcePubSubPull } from '../index'; +import type { PubSubPullClient } from '../types'; + +// Acknowledgement of a pulled message, proven on a real collector: the source +// acks a message unless a redelivery could still deliver it. + +const client: PubSubPullClient = { + subscription: () => ({ on: () => undefined, close: async () => {} }), + close: async () => {}, +}; + +const failing = (error: Error): Destination.Instance => ({ + type: 'failing', + config: {}, + push: () => { + throw error; + }, +}); + +const delivering: Destination.Instance = { + type: 'delivering', + config: {}, + push: () => undefined, +}; + +const deterministic = () => + new StepError('setting apiKey is missing', { code: 'config' }); +const retryable = () => + new StepError('vendor answered 503', { code: 'transport', status: 503 }); + +const lines: Array<[Level, string]> = []; + +async function consume( + destination: Destination.Instance, + body: unknown, + onPushError?: 'ack' | 'nack', +): Promise<{ result: Result; status: Collector.Status }> { + const instance = await createTrigger({ + logger: { handler: (level, message) => void lines.push([level, message]) }, + sources: { + pubsub: { + code: sourcePubSubPull, + config: { + settings: { + projectId: 'project', + subscription: 'events', + client, + ...(onPushError ? { onPushError } : {}), + }, + }, + }, + }, + destinations: { vendorX: { code: destination } }, + }); + try { + const result = await instance.trigger()({ + id: 'm1', + dataString: JSON.stringify(body), + }); + const status = instance.flow?.collector.status; + if (!status) throw new Error('flow did not start'); + return { result, status }; + } finally { + await instance.flow?.collector.command('shutdown'); + } +} + +describe('Pub/Sub pull acknowledgement', () => { + beforeEach(() => { + lines.length = 0; + }); + + it('acks a delivered message', async () => { + const { result } = await consume(delivering, { name: 'page view' }); + expect(result).toEqual([['message.ack', 'm1']]); + }); + + it('acks a message whose failure a redelivery cannot fix', async () => { + const { result, status } = await consume(failing(deterministic()), { + name: 'page view', + }); + expect(result).toEqual([['message.ack', 'm1']]); + expect(status.destinations.vendorX.failed).toBe(1); + // The failure is visible at the default level through the collector. + expect(lines.some(([level]) => level === Level.ERROR)).toBe(true); + }); + + it('nacks a message a redelivery could still deliver', async () => { + const { result } = await consume(failing(retryable()), { + name: 'page view', + }); + expect(result).toEqual([['message.nack', 'm1']]); + // One cause line from the collector, no per-message handler error. + expect(lines).toContainEqual([Level.ERROR, expect.any(String)]); + expect(lines).not.toContainEqual([ + Level.ERROR, + 'Pub/Sub message handler failed', + ]); + }); + + it("acks a retryable failure when onPushError is 'ack'", async () => { + const { result } = await consume( + failing(retryable()), + { name: 'page view' }, + 'ack', + ); + expect(result).toEqual([['message.ack', 'm1']]); + }); + + it('acks invalid input and counts it as rejected', async () => { + const { result, status } = await consume(delivering, { name: 'invalid' }); + expect(result).toEqual([['message.ack', 'm1']]); + expect(status.sources.pubsub.rejected).toBe(1); + }); +}); diff --git a/packages/server/sources/gcp/src/pubsub/pull/index.ts b/packages/server/sources/gcp/src/pubsub/pull/index.ts index 39fb36cb8..9f7be49aa 100644 --- a/packages/server/sources/gcp/src/pubsub/pull/index.ts +++ b/packages/server/sources/gcp/src/pubsub/pull/index.ts @@ -1,3 +1,4 @@ +import { acknowledge } from '@walkeros/core'; import type { Collector, Logger, Source, WalkerOS } from '@walkeros/core'; import type { Message, SubscriptionOptions } from '@google-cloud/pubsub'; import { Duration } from '@google-cloud/pubsub'; @@ -74,6 +75,9 @@ async function handleMessage(args: { logger: Logger.Instance; }): Promise { const { message, settings, pushFn, logger } = args; + // The error disposition: nack (redeliver) by default, ack if configured. + const dispose = () => + settings.onPushError === 'ack' ? message.ack() : message.nack(); try { const decoded = decodeMessage(message, settings.decoder ?? 'json'); if (decoded === null || decoded === undefined) { @@ -87,11 +91,17 @@ async function handleMessage(args: { return; } const result = await pushFn(toDeepPartialEvent(decoded)); - if (result?.ok === false) { - // Never delete a message that was not delivered. Route through the - // existing error disposition (nack by default, ack if configured). - throw new Error('push rejected the message (ok:false)'); + if (acknowledge(result) === 'retry') { + // Never ack a message a redelivery could still deliver. The collector + // logged the cause; this line is the per-message trace. + logger.debug('Pub/Sub message not delivered, retryable', { + id: message.id, + }); + dispose(); + return; } + // Delivered, a deterministic failure (in the DLQ, its cause logged) or + // rejected input (counted): a redelivery cannot change the outcome. message.ack(); } catch (err) { const error = err instanceof Error ? err.message : String(err); @@ -100,8 +110,7 @@ async function handleMessage(args: { } else { logger.error('Pub/Sub message handler failed', { id: message.id, error }); } - if (settings.onPushError === 'ack') message.ack(); - else message.nack(); + dispose(); } } diff --git a/packages/server/sources/gcp/src/pubsub/pull/schemas/settings.ts b/packages/server/sources/gcp/src/pubsub/pull/schemas/settings.ts index 96cb3d416..23fe59205 100644 --- a/packages/server/sources/gcp/src/pubsub/pull/schemas/settings.ts +++ b/packages/server/sources/gcp/src/pubsub/pull/schemas/settings.ts @@ -73,7 +73,7 @@ export const SettingsSchema = z.object({ onPushError: z .enum(['nack', 'ack']) .describe( - 'Behavior when forwarding to the collector throws. nack (default) redelivers the message; ack drops it.', + 'Behavior when a message is not delivered and a redelivery could still deliver it (a retryable failure), or when decoding or forwarding throws. nack (default) redelivers the message; ack drops it.', ) .optional(), }); diff --git a/packages/server/sources/gcp/src/pubsub/push/__tests__/acknowledge.test.ts b/packages/server/sources/gcp/src/pubsub/push/__tests__/acknowledge.test.ts new file mode 100644 index 000000000..ea48b789b --- /dev/null +++ b/packages/server/sources/gcp/src/pubsub/push/__tests__/acknowledge.test.ts @@ -0,0 +1,79 @@ +import { StepError } from '@walkeros/core'; +import type { Destination } from '@walkeros/core'; +import { createTrigger } from '../examples/trigger'; +import { sourcePubSubPush } from '../index'; + +// The HTTP answer to a pushed message, proven on a real collector: 200 unless +// a redelivery could still deliver the event (500) or the input is invalid +// (400). The body never names a destination. + +const failing = (error: Error): Destination.Instance => ({ + type: 'failing', + config: {}, + push: () => { + throw error; + }, +}); + +const delivering: Destination.Instance = { + type: 'delivering', + config: {}, + push: () => undefined, +}; + +const deterministic = () => + new StepError('setting apiKey is missing', { code: 'config' }); +const retryable = () => + new StepError('vendor answered 503', { code: 'transport', status: 503 }); + +async function deliver( + destination: Destination.Instance, + event: unknown, +): Promise<{ status: unknown; body: unknown }> { + const instance = await createTrigger({ + logger: { handler: () => undefined }, + sources: { pubsub: { code: sourcePubSubPush } }, + destinations: { vendorX: { code: destination } }, + }); + try { + const [[, status, body]] = await instance.trigger()({ + message: { + messageId: 'm1', + data: Buffer.from(JSON.stringify(event)).toString('base64'), + }, + subscription: 'projects/project/subscriptions/events', + }); + return { status, body }; + } finally { + await instance.flow?.collector.command('shutdown'); + } +} + +describe('Pub/Sub push acknowledgement', () => { + it('answers 200 for a delivered message', async () => { + const { status, body } = await deliver(delivering, { name: 'page view' }); + expect(status).toBe(200); + expect(body).toEqual({ success: true, id: 'm1' }); + }); + + it('answers 200 with the success body for a failure a redelivery cannot fix', async () => { + const { status, body } = await deliver(failing(deterministic()), { + name: 'page view', + }); + expect(status).toBe(200); + expect(body).toEqual({ success: true, id: 'm1' }); + }); + + it('answers 500 when a redelivery could still deliver the message', async () => { + const { status, body } = await deliver(failing(retryable()), { + name: 'page view', + }); + expect(status).toBe(500); + expect(body).toEqual({ success: false, error: 'Event was not processed' }); + }); + + it('answers 400 for invalid input', async () => { + const { status } = await deliver(delivering, { name: 'invalid' }); + expect(status).toBe(400); + }); +}); diff --git a/packages/server/sources/gcp/src/pubsub/push/index.ts b/packages/server/sources/gcp/src/pubsub/push/index.ts index bc35b0102..39b6d0535 100644 --- a/packages/server/sources/gcp/src/pubsub/push/index.ts +++ b/packages/server/sources/gcp/src/pubsub/push/index.ts @@ -1,5 +1,9 @@ import type { Source } from '@walkeros/core'; -import { isObject } from '@walkeros/core'; +import { + INTERNAL_SERVER_ERROR, + isObject, + pushResultToOutcome, +} from '@walkeros/core'; import { decodeMessage, DecoderError } from '../shared/decoder'; import type { DecodableMessage } from '../shared/decoder'; import { getConfig } from './config'; @@ -161,26 +165,21 @@ export const sourcePubSubPush: Source.Init = async (context) => { return; } - const result = await env.push(decoded); - if (result?.ok === false) { - if (result.invalid === true) { - // Permanently-invalid message: a retry can never fix it. Answer - // 400 (a nack, same as the decoder-failure path) and leave poison - // handling to the subscription's dead-letter policy. The collector - // already warned and counted the rejection. - res.status(400).json({ - success: false, - error: result.error ?? 'Invalid event', - }); - return; - } - logger.warn( - 'Pub/Sub push: event not processed, answering 500 for redelivery', - { id: body.message.messageId }, - ); - res.status(500).json({ + // 400 for a permanently invalid message (a nack, as on the decoder + // path; poison handling is the subscription's dead-letter policy), 500 + // when a redelivery could still deliver it, else 200: delivered, or + // failed for a reason a redelivery cannot fix (in the DLQ, its cause + // line logged by the collector). + const outcome = pushResultToOutcome(await env.push(decoded)); + if (outcome.error) { + if (outcome.status === 500) + logger.debug( + 'Pub/Sub push: event not processed, answering 500 for redelivery', + { id: body.message.messageId }, + ); + res.status(outcome.status ?? 500).json({ success: false, - error: 'Event was not processed.', + error: outcome.error, }); return; } @@ -189,10 +188,7 @@ export const sourcePubSubPush: Source.Init = async (context) => { logger.error('Pub/Sub push handler failed', { error: error instanceof Error ? error.message : String(error), }); - res.status(500).json({ - success: false, - error: error instanceof Error ? error.message : 'Internal server error', - }); + res.status(500).json({ success: false, error: INTERNAL_SERVER_ERROR }); } }; diff --git a/packages/web/sources/browser/src/__tests__/elbLayerFailure.test.ts b/packages/web/sources/browser/src/__tests__/elbLayerFailure.test.ts new file mode 100644 index 000000000..53131755c --- /dev/null +++ b/packages/web/sources/browser/src/__tests__/elbLayerFailure.test.ts @@ -0,0 +1,97 @@ +import { Level } from '@walkeros/core'; +import type { Collector, Destination } from '@walkeros/core'; +import { startFlow } from '@walkeros/collector'; +import { sourceBrowser } from '../index'; +import { flushChain } from './test-utils'; + +// An elbLayer entry whose routing throws, queued before the source starts as +// a page script ahead of walker.js does: the entry's event is lost, so the +// line is an ERROR at the default level, naming the entry by its event and +// never by its payload, and the entries after it still run. + +interface Line { + level: Level; + message: string; + context: unknown; +} + +let current: Collector.Instance | undefined; + +afterEach(async () => { + await current?.command('shutdown'); + current = undefined; + Reflect.deleteProperty(window, 'elbLayer'); + Reflect.deleteProperty(window, 'elb'); +}); + +async function start(layer: unknown[]) { + const lines: Line[] = []; + const delivered: string[] = []; + const probe: Destination.Instance = { + type: 'probe', + config: {}, + push: (event) => void delivered.push(event.name), + }; + Reflect.set(window, 'elbLayer', layer); + const { collector } = await startFlow({ + logger: { + handler: (level, message, context) => + void lines.push({ level, message, context }), + }, + sources: { + browser: { + code: sourceBrowser, + config: { settings: { pageview: false } }, + }, + }, + destinations: { probe: { code: probe } }, + }); + current = collector; + await flushChain(); + return { collector, lines, delivered }; +} + +describe('elbLayer entry failure', () => { + it('logs one ERROR without the payload and keeps routing the queue', async () => { + const { collector, lines, delivered } = await start([ + [ + 'order complete', + { + email: 'jane@example.com', + get total() { + throw new Error('bad total getter'); + }, + }, + ], + ['product view', { id: 'p1' }], + ]); + + const errors = lines.filter(({ level }) => level === Level.ERROR); + expect(errors).toHaveLength(1); + expect(errors[0].context).toMatchObject({ + step: 'source.browser', + event: 'order complete', + }); + expect(JSON.stringify(errors[0].context)).not.toContain('jane@example.com'); + expect(collector.status.failed).toBe(1); + expect(delivered).toEqual(['product view']); + }); + + it('names an event object by a name it can read, never reading a throwing one', async () => { + const { lines, delivered } = await start([ + { + get name(): string { + throw new Error('bad event getter'); + }, + data: { email: 'jane@example.com' }, + }, + ['product view', { id: 'p1' }], + ]); + + const errors = lines.filter(({ level }) => level === Level.ERROR); + expect(errors).toHaveLength(1); + expect(errors[0].context).toMatchObject({ step: 'source.browser' }); + expect(JSON.stringify(errors[0].context)).not.toContain('jane@example.com'); + expect(delivered).toEqual(['product view']); + }); +}); diff --git a/packages/web/sources/browser/src/__tests__/pageHide.test.ts b/packages/web/sources/browser/src/__tests__/pageHide.test.ts new file mode 100644 index 000000000..ca35be5e9 --- /dev/null +++ b/packages/web/sources/browser/src/__tests__/pageHide.test.ts @@ -0,0 +1,112 @@ +import type { Collector, Destination, WalkerOS } from '@walkeros/core'; +import { startFlow } from '@walkeros/collector'; +import { sourceBrowser } from '../index'; +import { flushChain } from './test-utils'; + +// Leaving the page flushes every destination batch, so events waiting in a +// batch window are sent before the page goes away: a real collector, the +// browser source and a destination whose batch waits longer than the test. + +let current: Collector.Instance | undefined; +let batches: WalkerOS.Events[] = []; + +const batching: Destination.Instance = { + type: 'batching', + config: { batch: { wait: 60000 } }, + push: () => undefined, + pushBatch: (batch) => { + batches.push(batch.events); + }, +}; + +async function start(): Promise { + const { collector } = await startFlow({ + sources: { + browser: { + code: sourceBrowser, + config: { settings: { pageview: false } }, + }, + }, + destinations: { batching: { code: batching } }, + }); + current = collector; + await flushChain(); + return collector; +} + +async function pushTwo(collector: Collector.Instance) { + await collector.push({ name: 'product view' }); + await collector.push({ name: 'product add' }); +} + +const setVisibility = (state: DocumentVisibilityState) => + Object.defineProperty(document, 'visibilityState', { + value: state, + configurable: true, + }); + +beforeEach(() => { + batches = []; + setVisibility('visible'); +}); + +afterEach(async () => { + await current?.command('shutdown'); + current = undefined; + Reflect.deleteProperty(window, 'elbLayer'); + Reflect.deleteProperty(window, 'elb'); + setVisibility('visible'); +}); + +const names = () => batches.map((events) => events.map(({ name }) => name)); + +describe('flush when the page goes away', () => { + it('sends the pending batch on pagehide', async () => { + const collector = await start(); + await pushTwo(collector); + expect(batches).toHaveLength(0); + + window.dispatchEvent(new Event('pagehide')); + await flushChain(); + + expect(names()).toEqual([['product view', 'product add']]); + }); + + it('sends the pending batch when the document turns hidden, not visible', async () => { + const collector = await start(); + await pushTwo(collector); + + document.dispatchEvent(new Event('visibilitychange')); + await flushChain(); + expect(batches).toHaveLength(0); + + setVisibility('hidden'); + document.dispatchEvent(new Event('visibilitychange')); + await flushChain(); + + expect(names()).toEqual([['product view', 'product add']]); + }); + + it('removes both listeners on destroy', async () => { + const collector = await start(); + await pushTwo(collector); + await collector.sources.browser.destroy?.({ + id: 'browser', + config: collector.sources.browser.config, + env: { + push: collector.push, + command: collector.command, + elb: collector.elb, + logger: collector.logger, + }, + logger: collector.logger, + }); + + window.dispatchEvent(new Event('pagehide')); + setVisibility('hidden'); + document.dispatchEvent(new Event('visibilitychange')); + await flushChain(); + + expect(batches).toHaveLength(0); + }); +}); diff --git a/packages/web/sources/browser/src/elbLayer.ts b/packages/web/sources/browser/src/elbLayer.ts index 9954deafa..a91273fcb 100644 --- a/packages/web/sources/browser/src/elbLayer.ts +++ b/packages/web/sources/browser/src/elbLayer.ts @@ -1,5 +1,11 @@ -import type { Elb, Logger } from '@walkeros/core'; -import { isString, isObject, tryCatch } from '@walkeros/core'; +import type { Collector, Elb, Logger } from '@walkeros/core'; +import { + isString, + isObject, + reportCause, + toFailureReason, + tryCatch, +} from '@walkeros/core'; import { createPushResult } from '@walkeros/collector'; import type { Context } from './types'; import { translateToCoreCollector } from './translation'; @@ -85,13 +91,32 @@ const readOrCreateDrainMarker = (arr: unknown[]): DrainMarker => { return fresh; }; +// The entry's event or command name, never its payload. An event object is +// read inside a guard: its own `name` may be the getter that threw. +const entryName = (item: unknown[]): string | undefined => { + const [first] = item; + if (isString(first)) return first; + return tryCatch(() => + isObject(first) && isString(first.name) ? first.name : undefined, + )(); +}; + export function createElbLayer( context: Context, - config: { name?: string; window?: Window; logger?: Logger.Instance } = {}, + config: { + name?: string; + window?: Window; + logger?: Logger.Instance; + /** Reports a failed entry through its cause log and status. */ + collector?: Collector.Instance; + /** The source id, which names the step of a failed entry. */ + id?: string; + } = {}, ): ElbLayerController { const layerName = config.name || 'elbLayer'; const windowObj = config.window; const logger = config.logger; + const collector = config.collector; const ok = (): Elb.PushResult => createPushResult({ ok: true }); @@ -166,13 +191,32 @@ export function createElbLayer( }; // One bad entry must not break the chain: isolate each route so a throw - // resolves ok:false and the FIFO continues. + // resolves ok:false and the FIFO continues. The entry's event is lost, so + // it is an ERROR through the collector's cause log and counted as failed; + // the line names the entry, never carries it. const routeSafe = async (item: unknown[]): Promise => { try { return await route(item); } catch (error) { - logger?.warn('elbLayer entry failed', { error, item }); - return createPushResult({ ok: false }); + const reason = toFailureReason(error); + if (collector) collector.status.failed++; + if (logger) + reportCause( + collector, + logger, + { + step: config.id ? `source.${config.id}` : 'source', + code: reason.code, + status: reason.status, + }, + 'elbLayer entry failed', + { + event: entryName(item), + retryable: reason.retryable, + error: reason.message, + }, + ); + return createPushResult({ ok: false, reason, error: reason.message }); } }; diff --git a/packages/web/sources/browser/src/index.ts b/packages/web/sources/browser/src/index.ts index b6ca11834..0046a67ce 100644 --- a/packages/web/sources/browser/src/index.ts +++ b/packages/web/sources/browser/src/index.ts @@ -17,6 +17,7 @@ import { getPageViewData, getUser } from './walker'; import { getConfig } from './config'; import { mark, owner, release } from './ownership'; import { watchHistory } from './history'; +import { watchPageHide } from './pageHide'; export * as SourceBrowser from './types'; @@ -115,6 +116,8 @@ export const sourceBrowser: Source.Init = async (context) => { // it started, in order: a run that takes one is a history run. let stopHistory: (() => void) | undefined; const historyHrefs: string[] = []; + // The unsubscribe of the page-hide watcher, set in init. + let stopPageHide: (() => void) | undefined; // The URL of the last page view, the referrer of the next one. let previousHref: string | undefined; @@ -159,6 +162,16 @@ export const sourceBrowser: Source.Init = async (context) => { if (initialized) return; initialized = true; + // Leaving the page flushes every destination batch through the flush + // shutdown uses, so events waiting in a batch window are sent. The + // collector keeps running, for a page restored from the back-forward + // cache. Not awaited: the page is going away. + stopPageHide = watchPageHide(actualWindow, actualDocument, () => { + elb('walker flush').catch((error: unknown) => { + logger.error('walker flush failed', { error }); + }); + }); + // Build the append-only layer controller. It routes through the real // translationContext, so `walker init` reaches initScope and events flow // through the same funnel as direct window.elb calls. @@ -171,6 +184,8 @@ export const sourceBrowser: Source.Init = async (context) => { name: layerName, window: actualWindow, logger, + collector: context.collector, + id: context.id, }) : undefined; @@ -298,6 +313,9 @@ export const sourceBrowser: Source.Init = async (context) => { stopHistory = undefined; historyHrefs.length = 0; + stopPageHide?.(); + stopPageHide = undefined; + // Restore the layer's native push and forget controller state. controller?.destroy(); diff --git a/packages/web/sources/browser/src/pageHide.ts b/packages/web/sources/browser/src/pageHide.ts new file mode 100644 index 000000000..f713c6d8c --- /dev/null +++ b/packages/web/sources/browser/src/pageHide.ts @@ -0,0 +1,21 @@ +// The page is going away: `pagehide` on unload and on entering the +// back-forward cache, and the document turning hidden, the last moment a +// mobile browser reliably runs code. Either may fire without the other, and +// both may fire for one departure. +export function watchPageHide( + win: Window, + doc: Document, + onHide: () => void, +): () => void { + const hidden = () => { + if (doc.visibilityState === 'hidden') onHide(); + }; + + win.addEventListener('pagehide', onHide); + doc.addEventListener('visibilitychange', hidden); + + return () => { + win.removeEventListener('pagehide', onHide); + doc.removeEventListener('visibilitychange', hidden); + }; +} diff --git a/packages/web/sources/dataLayer/src/__tests__/noEnv.test.ts b/packages/web/sources/dataLayer/src/__tests__/noEnv.test.ts new file mode 100644 index 000000000..6532b7936 --- /dev/null +++ b/packages/web/sources/dataLayer/src/__tests__/noEnv.test.ts @@ -0,0 +1,55 @@ +import { resolve } from 'path'; +import { startFlow } from '@walkeros/collector'; +import { conformance } from '@walkeros/collector/dev'; +import type { Destination, WalkerOS } from '@walkeros/core'; +import sourceDataLayer from '../index'; +import { examples, schemas } from '../dev'; + +// The source as the docs integrate it: the published default export with no +// env. The collector passes no `window`, so the source finds the page's own. + +describe('DataLayer source without env', () => { + beforeEach(() => { + jest.useRealTimers(); + window.dataLayer = undefined; + }); + + afterEach(() => { + window.dataLayer = undefined; + }); + + it('captures a dataLayer.push', async () => { + const delivered: WalkerOS.Event[] = []; + const spy: Destination.Instance = { + type: 'spy', + config: {}, + push: (event) => void delivered.push(event), + }; + const { collector } = await startFlow({ + sources: { dataLayer: { code: sourceDataLayer } }, + destinations: { spy: { code: spy } }, + }); + try { + window.dataLayer?.push({ event: 'purchase', value: 25.42 }); + await new Promise((settle) => setTimeout(settle, 0)); + expect(delivered.map((event) => event.name)).toEqual([ + 'dataLayer purchase', + ]); + } finally { + await collector.command('shutdown'); + } + }); + + it('passes the source conformance profile', async () => { + await expect( + conformance('source', sourceDataLayer, { + profile: 'source', + examples: examples.step, + schema: schemas.settings, + settings: {}, + root: resolve(__dirname, '../..'), + createTrigger: examples.createTrigger, + }), + ).resolves.toBeUndefined(); + }); +}); diff --git a/packages/web/sources/dataLayer/src/examples/trigger.ts b/packages/web/sources/dataLayer/src/examples/trigger.ts index c5b8f4af5..3f84eb9b7 100644 --- a/packages/web/sources/dataLayer/src/examples/trigger.ts +++ b/packages/web/sources/dataLayer/src/examples/trigger.ts @@ -8,24 +8,12 @@ const createTrigger: Trigger.CreateFn = async ( let flow: Trigger.FlowHandle | undefined; const trigger: Trigger.Fn = () => async (content: unknown) => { - // Lazy startFlow — source patches window.dataLayer.push during init + // Lazy startFlow: the source patches window.dataLayer.push during init + // and finds the page's window itself, as in a real integration. if (!flow) { - // Ensure dataLayer sources get the window env for interception - const enrichedConfig = { ...config }; - if (enrichedConfig.sources) { - enrichedConfig.sources = { ...enrichedConfig.sources }; - for (const [id, source] of Object.entries(enrichedConfig.sources)) { - if (!source.env?.window) { - enrichedConfig.sources[id] = { - ...source, - env: { ...source.env, window }, - }; - } - } - } const result = await startFlow({ - ...enrichedConfig, - run: enrichedConfig.run ?? true, + ...config, + run: config.run ?? true, }); flow = { collector: result.collector, elb: result.elb }; } diff --git a/packages/web/sources/dataLayer/src/index.ts b/packages/web/sources/dataLayer/src/index.ts index e5432df16..334787053 100644 --- a/packages/web/sources/dataLayer/src/index.ts +++ b/packages/web/sources/dataLayer/src/index.ts @@ -16,7 +16,12 @@ export const sourceDataLayer: Source.Init = async (context) => { // Events exit via `push` (the collector's source pipeline, so this source's // `next`/`before`/mapping/cache/state and status apply); `elb` stays the // outward-facing slot on the instance below. This source emits no commands. - const { elb, push, window: envWindow } = env; + const { elb, push } = env; + // The collector passes no `window`: an injected one (tests, simulation) + // wins, else the page's own. + const envWindow = + env.window || + (typeof globalThis.window !== 'undefined' ? globalThis.window : undefined); const settings: Source.Settings = { name: 'dataLayer', From 5bf850acb078451896194c82a14065bad65a730f Mon Sep 17 00:00:00 2001 From: alexanderkirtzel Date: Wed, 21 Oct 2026 04:48:12 +0000 Subject: [PATCH 10/41] kit env phases --- .../src/dev/__tests__/conformance.test.ts | 91 +++++++++++++++++-- .../__tests__/fixture-package/src/store.ts | 15 ++- .../fixture-package/src/transformer.ts | 25 +++++ .../__tests__/fixture-package/src/vendor.ts | 33 ++++++- .../src/dev/__tests__/runExamples.test.ts | 14 +++ packages/collector/src/dev/conformance.ts | 53 +++++++++-- packages/collector/src/dev/runExamples.ts | 7 +- 7 files changed, 214 insertions(+), 24 deletions(-) diff --git a/packages/collector/src/dev/__tests__/conformance.test.ts b/packages/collector/src/dev/__tests__/conformance.test.ts index f285fdf1d..77e5f498e 100644 --- a/packages/collector/src/dev/__tests__/conformance.test.ts +++ b/packages/collector/src/dev/__tests__/conformance.test.ts @@ -21,6 +21,7 @@ import { consentAware, consentIgnoring, initCalling, + swallowingSdk, vendorGlobal, } from './fixture-package/src/vendor'; import { eitherId, ga4Only } from './fixture-package/src/alternatives'; @@ -169,11 +170,14 @@ describe('conformance kit, other step kinds', () => { describe('store (socket profile)', () => { const values = new Map(); + // Counted, so a test cannot pass without the store reaching its client. + const read = jest.fn(async (key: string) => values.get(key)); + const write = jest.fn(async (key: string, value: Store.StoreValue) => { + values.set(key, value); + }); const client: Client = { - read: async (key) => values.get(key), - write: async (key, value) => { - values.set(key, value); - }, + read, + write, remove: async (key) => { values.delete(key); }, @@ -193,16 +197,56 @@ describe('conformance kit, other step kinds', () => { }, }; - it('passes a store that reports a failed read', async () => { + it('passes a store that reports a failed read, through its env client', async () => { + read.mockClear(); + write.mockClear(); await expect( conformance('store', remote, options), ).resolves.toBeUndefined(); + expect(read).toHaveBeenCalled(); + expect(write).toHaveBeenCalled(); }); - it('fails a store that turns a failed read into a miss (item 2)', async () => { + it('judges items 2 and 3 by the examples that reach the SDK (a refused key reaches none)', async () => { + read.mockClear(); + await expect( + conformance('store', remote, { + ...options, + examples: { + ...options.examples, + refused: { in: { operation: 'get', key: '../../etc/passwd' } }, + }, + }), + ).resolves.toBeUndefined(); + expect(read).toHaveBeenCalled(); + }); + + it('fails items 2 and 3 when no example of an operation reaches the SDK', async () => { + await expect( + conformance('store', remote, { + ...options, + examples: { + write: options.examples.write, + refused: { in: { operation: 'get', key: '../../etc/passwd' } }, + }, + }), + ).rejects.toMatchObject({ + items: [2, 3], + message: expect.stringContaining('no example of get reaches the SDK'), + }); + }); + + it('fails a store that turns its client rejection into a miss (item 2)', async () => { + read.mockClear(); await expect( conformance('store', missing, options), - ).rejects.toMatchObject({ items: [2] }); + ).rejects.toMatchObject({ + items: [2], + message: expect.stringContaining( + 'a rejected SDK call on get is not reported', + ), + }); + expect(read).toHaveBeenCalled(); }); }); @@ -310,7 +354,14 @@ describe('conformance kit, vendor-global profile', () => { const initExamples: Flow.StepExamples = { pageView: { ...examples.pageView, in: consented }, silent: { ...examples.silent, in: consented }, - unreachable: { ...examples.unreachable, in: consented }, + // The vendor does not start: init fails before any push call. + unreachable: { + ...examples.unreachable, + in: consented, + description: 'The vendor does not start: init reports it.', + out: [], + outcome: { status: 'failed', code: 'vendor-not-ready' }, + }, // Grants marketing (the init runs inside it), revokes functional. revoke: { command: 'consent', @@ -347,6 +398,30 @@ describe('conformance kit, vendor-global profile', () => { }); }); + it('counts item 2 over the session: an init that fails inside startFlow writes the one ERROR line', async () => { + const atLoad = { ...initCalling, config: {} }; + await expect( + conformance('destination', atLoad, { + ...options, + env: tracked, + profile: 'web-sdk', + }), + ).resolves.toBeUndefined(); + }); + + it('fails item 2 for a destination that swallows a failed start and a failed call', async () => { + await expect( + conformance('destination', swallowingSdk, { + ...options, + env: tracked, + profile: 'web-sdk', + }), + ).rejects.toMatchObject({ + items: [2], + message: expect.stringContaining('a rejected SDK call is not reported'), + }); + }); + it('still counts the init calls for item 5', async () => { await expect( conformance('destination', initCalling, { diff --git a/packages/collector/src/dev/__tests__/fixture-package/src/store.ts b/packages/collector/src/dev/__tests__/fixture-package/src/store.ts index 3e1d7095f..1598ab6c8 100644 --- a/packages/collector/src/dev/__tests__/fixture-package/src/store.ts +++ b/packages/collector/src/dev/__tests__/fixture-package/src/store.ts @@ -15,8 +15,11 @@ function clientOf(env: { client?: Client }): Client { return env.client; } -/** Reports a failed read as a StepError, never as a miss. */ -export const remote: Store.Init = ({ config, env }) => { +/** + * Reports a failed read as a StepError, never as a miss. Like the fs store, + * it refuses a key that leaves the store before any client call: a miss. + */ +export const remote: Store.Init = ({ config, env, logger }) => { if (!config.settings?.bucket) throw new StepError('Config settings bucket missing', { code: 'config' }); const fail = (error: unknown): never => { @@ -28,7 +31,13 @@ export const remote: Store.Init = ({ config, env }) => { return { type: 'remote', config, - get: (key) => clientOf(env).read(key).catch(fail), + get: (key) => { + if (key.includes('..')) { + logger.warn('Store refused a key outside the store'); + return undefined; + } + return clientOf(env).read(key).catch(fail); + }, set: (key, value) => clientOf(env).write(key, value).catch(fail), delete: (key) => clientOf(env).remove(key).catch(fail), }; diff --git a/packages/collector/src/dev/__tests__/fixture-package/src/transformer.ts b/packages/collector/src/dev/__tests__/fixture-package/src/transformer.ts index 226d8d846..ed079ad6a 100644 --- a/packages/collector/src/dev/__tests__/fixture-package/src/transformer.ts +++ b/packages/collector/src/dev/__tests__/fixture-package/src/transformer.ts @@ -36,3 +36,28 @@ export const dropping: Transformer.Init = async (context) => { event.user?.id ? instance.push(event, pushContext) : false, }; }; + +type EnvTypes = Transformer.Types; + +/** Needs `env.prefix` in its factory, its init and its push. */ +export const envReading: Transformer.Init = ({ config, env }) => { + if (!env.prefix) + throw new StepError('Env prefix missing in the factory', { + code: 'config', + }); + return { + type: 'env-reading', + config, + init: ({ env: initEnv }) => { + if (!initEnv.prefix) + throw new StepError('Env prefix missing in init', { code: 'config' }); + }, + push: (event, { env: pushEnv }) => { + if (!pushEnv.prefix) + throw new StepError('Env prefix missing in push', { + code: 'missing-input', + }); + return { event }; + }, + }; +}; diff --git a/packages/collector/src/dev/__tests__/fixture-package/src/vendor.ts b/packages/collector/src/dev/__tests__/fixture-package/src/vendor.ts index e7bbfcbf3..66180db95 100644 --- a/packages/collector/src/dev/__tests__/fixture-package/src/vendor.ts +++ b/packages/collector/src/dev/__tests__/fixture-package/src/vendor.ts @@ -47,20 +47,43 @@ export const vendorGlobal: Destination.Instance = { }; /** - * Calls its vendor in `init`, as `fbq('init', ...)` does: the call is queued - * by the vendor, not awaited. It requires marketing consent, so the collector - * initializes it inside the first input that grants it, not at the start. + * Starts its vendor SDK in `init` and awaits it, as fullstory's `fs.init`; + * a failed start throws `vendor-not-ready`. It requires marketing consent, + * so the collector initializes it inside the first input that grants it. */ export const initCalling: Destination.Instance = { ...vendorGlobal, type: 'init-calling', config: { consent: { marketing: true } }, - init: ({ config, env }) => { + init: async ({ config, env }) => { if (!env.vendor) return; - Promise.resolve(env.vendor.init(config.settings?.site ?? '')).catch( + try { + await env.vendor.init(config.settings?.site ?? ''); + } catch (error) { + throw new StepError('Vendor did not start', { + code: 'vendor-not-ready', + cause: error, + }); + } + }, +}; + +/** Swallows a failed start and a failed call: nothing is reported (item 2). */ +export const swallowingSdk: Destination.Instance = { + ...vendorGlobal, + type: 'swallowing-sdk', + config: {}, + init: async ({ config, env }) => { + await Promise.resolve(env.vendor?.init(config.settings?.site ?? '')).catch( () => undefined, ); }, + push: async (event, { config, env, rule }) => { + if (rule?.silent) return; + await Promise.resolve( + env.vendor?.track(event.name, config.settings?.site), + ).catch(() => undefined); + }, }; /** diff --git a/packages/collector/src/dev/__tests__/runExamples.test.ts b/packages/collector/src/dev/__tests__/runExamples.test.ts index fa9281d6a..b1b088c87 100644 --- a/packages/collector/src/dev/__tests__/runExamples.test.ts +++ b/packages/collector/src/dev/__tests__/runExamples.test.ts @@ -14,6 +14,7 @@ import { startFlow } from '../../flow'; import { runExamples } from '../runExamples'; import { fixtures } from './fixtures'; import { good } from './fixture-package/src'; +import { envReading } from './fixture-package/src/transformer'; const URL = 'https://127.0.0.1:9/e'; const fetchOut = (name: string): Flow.StepOut => [ @@ -291,6 +292,19 @@ describe('runExamples', () => { }); }); + describe('transformer env', () => { + it("gives the examples env to the factory, init and push, where a flow's entry carries it", async () => { + await expect( + runExamples( + 'transformer', + envReading, + { tagged: { in: { name: 'page view' } } }, + { env: { push: { prefix: 'kit' } } }, + ), + ).resolves.toBeUndefined(); + }); + }); + describe('store', () => { const memory: Store.Init = ({ config }) => { const values = new Map(); diff --git a/packages/collector/src/dev/conformance.ts b/packages/collector/src/dev/conformance.ts index c799283db..56951168f 100644 --- a/packages/collector/src/dev/conformance.ts +++ b/packages/collector/src/dev/conformance.ts @@ -419,6 +419,8 @@ const failureEnvAnswers: Array<{ label: string; answer: EnvAnswer }> = [ type DestinationStep = Destination.Instance; interface DestinationRun extends Step { + /** Every log line of the session, from startFlow through the run. */ + sessionLogs: LogLine[]; initCalls: number; scripts: { marked: number; unmarked: number }; dlq: WalkerOS.Event[]; @@ -448,6 +450,7 @@ async function runDestination( const run = await session.run({ event, ingest }); return { ...run, + sessionLogs: [...session.logs], initCalls: session.initCalls(), scripts: session.scripts(), dlq: (session.collector.destinations[STEP_ID]?.dlq ?? []).map( @@ -469,7 +472,9 @@ function reportProblem(label: string, run: DestinationRun): Problem { return `${label} is reported without its status`; if (!reason.retryable) return `${label} is reported as not retryable`; if (!run.dlq.length) return `${label} leaves the DLQ empty`; - const errors = errorsIn(run.logs); + // The whole session: a destination initialized inside startFlow writes + // its one line there, and the run's retry of the same cause is suppressed. + const errors = errorsIn(run.sessionLogs); if (errors !== 1) return `${label} writes ${errors} ERROR lines instead of the collector's one`; return undefined; @@ -734,7 +739,7 @@ async function vendorFailureProblem( return `a push after the vendor script failed is not reported vendor-not-loaded (${show(reason)})`; const dlq = session.collector.destinations[STEP_ID]?.dlq ?? []; if (!dlq.length) return 'the vendor-not-loaded event is not in the DLQ'; - const errors = errorsIn(run.logs); + const errors = errorsIn(session.logs); return errors === 1 ? undefined : `vendor-not-loaded writes ${errors} ERROR lines instead of the collector's one`; @@ -1029,6 +1034,34 @@ async function withStore( } } +/** + * Which operation examples reach the SDK boundary in a normal run (at least + * one call on the examples env), from the session's own call record. Items 2 + * and 3 judge those only; an operation that no example reaches fails them, + * so a store cannot pass by never calling its client. + */ +async function reachingOperations( + step: StoreStep, + base: Setup, + operations: StoreOperation[], +): Promise<{ reaching: StoreOperation[]; problems: string[] }> { + const reached = await withStore(step, base, async (session) => { + const flags: boolean[] = []; + for (const operation of operations) { + const before = session.envBoundary?.calls.length ?? 0; + await runStoreOperation(session, operation).catch(() => undefined); + flags.push((session.envBoundary?.calls.length ?? 0) > before); + } + return flags; + }); + const reaching = operations.filter((_, index) => reached[index]); + const kinds = [...new Set(operations.map(({ operation }) => operation))]; + const problems = kinds + .filter((kind) => !reaching.some(({ operation }) => operation === kind)) + .map((kind) => `no example of ${kind} reaches the SDK`); + return { reaching, problems }; +} + async function storeItem( item: number, step: StoreStep, @@ -1055,7 +1088,11 @@ async function storeItem( case 2: { if (!ctx.env) return storeDeclaredFailures(step, ctx); if (!operations.length) return 'no store operation example to fail'; - const problems: string[] = []; + const { reaching, problems } = await reachingOperations( + step, + base, + operations, + ); for (const { label, answer } of failureEnvAnswers) { // A store call has no collector timeout around it here: a call that // never settles is item 3's late answer, not a failure to wait for. @@ -1064,7 +1101,7 @@ async function storeItem( step, { ...base, envAnswers: () => answer }, async (session) => { - for (const operation of operations) { + for (const operation of reaching) { const logsBefore = session.logs.length; const error = await runStoreOperation(session, operation).then( () => undefined, @@ -1087,12 +1124,16 @@ async function storeItem( case 3: { if (!ctx.env) return 'no SDK boundary to answer late (pass env)'; - const problems: string[] = []; + const { reaching, problems } = await reachingOperations( + step, + base, + operations, + ); await withStore( step, { ...base, envAnswers: () => ({ kind: 'late' }) }, async (session) => { - for (const operation of operations) { + for (const operation of reaching) { await runStoreOperation(session, operation).catch(() => undefined); if (session.envBoundary?.pending()) problems.push( diff --git a/packages/collector/src/dev/runExamples.ts b/packages/collector/src/dev/runExamples.ts index 0c3516589..4513edbc7 100644 --- a/packages/collector/src/dev/runExamples.ts +++ b/packages/collector/src/dev/runExamples.ts @@ -423,8 +423,10 @@ export function openTransformer( ...(setup.settings !== undefined ? { settings: settingsOf(setup) } : {}), - ...(env ? { env } : {}), }, + // Where a flow's transformer entry carries it: the factory gets + // it as context.env, init and push through the config it returns. + ...(env ? { env } : {}), }, }, // Under the step's id too, so its failure reads like a destination's. @@ -449,8 +451,9 @@ export function openStore( ...(setup.settings !== undefined ? { settings: settingsOf(setup) } : {}), - ...(env ? { env } : {}), }, + // Where a flow's store entry carries it: the store's context.env. + ...(env ? { env } : {}), }, }, })); From 5c7d2b7e8e6fc5a8920be9e81fa8e7201015303b Mon Sep 17 00:00:00 2001 From: alexanderkirtzel Date: Wed, 21 Oct 2026 04:54:35 +0000 Subject: [PATCH 11/41] kit ingest transformers --- .../src/dev/__tests__/conformance.test.ts | 56 ++++++++++++++++++- .../fixture-package/src/transformer.ts | 25 +++++++++ packages/collector/src/dev/conformance.ts | 31 ++++++++++ 3 files changed, 111 insertions(+), 1 deletion(-) diff --git a/packages/collector/src/dev/__tests__/conformance.test.ts b/packages/collector/src/dev/__tests__/conformance.test.ts index 77e5f498e..497f8897e 100644 --- a/packages/collector/src/dev/__tests__/conformance.test.ts +++ b/packages/collector/src/dev/__tests__/conformance.test.ts @@ -12,7 +12,12 @@ import { rawHttps, swallowing, } from './fixture-package/src'; -import { dropping, hashing } from './fixture-package/src/transformer'; +import { + dropping, + hashing, + ingestDecoding, + ingestPassing, +} from './fixture-package/src/transformer'; import type { Client } from './fixture-package/src/store'; import { missing, remote } from './fixture-package/src/store'; import { chatty, forwarding } from './fixture-package/src/source'; @@ -168,6 +173,55 @@ describe('conformance kit, other step kinds', () => { }); }); + describe('transformer whose input is the ingest (item 10)', () => { + const noRequest: Flow.StepExample = { + public: false, + description: 'An event without a request in its ingest is reported.', + in: { name: 'page view' }, + out: [], + outcome: { status: 'failed', code: 'missing-input' }, + }; + const ingestOptions = { + profile: 'transformer' as const, + root, + schema: { type: 'object', properties: {} }, + settings: {}, + examples: { + request: { in: { url: 'https://example.com/g/collect?en=page_view' } }, + noRequest, + }, + }; + + it('passes with a declared missing-input example the kit confirms', async () => { + await expect( + conformance('transformer', ingestDecoding, ingestOptions), + ).resolves.toBeUndefined(); + }); + + it('fails item 10 without that example', async () => { + await expect( + conformance('transformer', ingestDecoding, { + ...ingestOptions, + examples: { request: ingestOptions.examples.request }, + }), + ).rejects.toMatchObject({ + items: [10], + message: expect.stringContaining('no missing-input example'), + }); + }); + + it('fails a transformer that passes an event without the ingest on unchanged (items 2 and 10)', async () => { + await expect( + conformance('transformer', ingestPassing, ingestOptions), + ).rejects.toMatchObject({ + items: [2, 10], + message: expect.stringContaining( + 'item 10: example noRequest: out differs', + ), + }); + }); + }); + describe('store (socket profile)', () => { const values = new Map(); // Counted, so a test cannot pass without the store reaching its client. diff --git a/packages/collector/src/dev/__tests__/fixture-package/src/transformer.ts b/packages/collector/src/dev/__tests__/fixture-package/src/transformer.ts index ed079ad6a..2100c8e75 100644 --- a/packages/collector/src/dev/__tests__/fixture-package/src/transformer.ts +++ b/packages/collector/src/dev/__tests__/fixture-package/src/transformer.ts @@ -61,3 +61,28 @@ export const envReading: Transformer.Init = ({ config, env }) => { }, }; }; + +type IngestTypes = Transformer.Types; + +/** Decodes the request in the ingest; an event without one is missing-input. */ +export const ingestDecoding: Transformer.Init = ({ config }) => ({ + type: 'ingest-decoding', + config, + push: (event, { ingest }) => { + if (typeof ingest.url !== 'string') + throw new StepError('No request in the ingest', { + code: 'missing-input', + }); + return { event: { ...event, name: 'page view' } }; + }, +}); + +/** Passes an event on unchanged when the ingest has no request (silent). */ +export const ingestPassing: Transformer.Init = ({ config }) => ({ + type: 'ingest-passing', + config, + push: (event, { ingest }) => + typeof ingest.url === 'string' + ? { event: { ...event, name: 'page view' } } + : { event }, +}); diff --git a/packages/collector/src/dev/conformance.ts b/packages/collector/src/dev/conformance.ts index 56951168f..4d37326d0 100644 --- a/packages/collector/src/dev/conformance.ts +++ b/packages/collector/src/dev/conformance.ts @@ -998,6 +998,7 @@ async function transformerItem( } case 10: + if (!ctx.events.length) return ingestInputProblem(step, ctx); return strippedProblem(ctx, async (_, stripped) => { const run = await runTransformer(step, ctx, base, stripped); return { @@ -1009,6 +1010,36 @@ async function transformerItem( return undefined; } +/** + * Item 10 for a transformer whose input is the ingest (no success example + * carries an event `in`, e.g. a request to decode): an event without that + * ingest must be reported. Its declared `missing-input` example (an event + * `in`, `out: []`, a description) runs and must hold. + */ +async function ingestInputProblem( + step: TransformerStep, + ctx: Context, +): Promise { + const declared = Object.entries(ctx.options.examples).filter( + ([, example]) => + isEvent(example.in) && + example.outcome?.status === 'failed' && + example.outcome.code === 'missing-input' && + Array.isArray(example.out) && + example.out.length === 0 && + Boolean(example.description), + ); + if (!declared.length) + return 'no example carries an event, and no missing-input example (an event without the ingest, out: [], a description) shows it is reported'; + return runExamples('transformer', step, Object.fromEntries(declared), { + settings: ctx.options.settings, + env: ctx.options.env, + }).then( + () => undefined, + (error: unknown) => textOf(error), + ); +} + /* ------------------------------------------------------------------------ */ /* Stores */ /* ------------------------------------------------------------------------ */ From d69edd46fce327fbff270a970cd8cffff4691821 Mon Sep 17 00:00:00 2001 From: alexanderkirtzel Date: Wed, 21 Oct 2026 05:16:12 +0000 Subject: [PATCH 12/41] kit cause count --- .../src/dev/__tests__/conformance.test.ts | 28 ++++++++++++ .../__tests__/fixture-package/src/index.ts | 14 ++++++ .../__tests__/fixture-package/src/vendor.ts | 45 +++++++++++++++++++ packages/collector/src/dev/conformance.ts | 26 ++++++++--- 4 files changed, 108 insertions(+), 5 deletions(-) diff --git a/packages/collector/src/dev/__tests__/conformance.test.ts b/packages/collector/src/dev/__tests__/conformance.test.ts index 497f8897e..dfde8fb65 100644 --- a/packages/collector/src/dev/__tests__/conformance.test.ts +++ b/packages/collector/src/dev/__tests__/conformance.test.ts @@ -9,6 +9,7 @@ import { echoing, floating, good, + loud, rawHttps, swallowing, } from './fixture-package/src'; @@ -26,6 +27,7 @@ import { consentAware, consentIgnoring, initCalling, + scriptVendor, swallowingSdk, vendorGlobal, } from './fixture-package/src/vendor'; @@ -121,6 +123,17 @@ describe('conformance kit', () => { ).rejects.toMatchObject({ items: [11] }); }); + it("fails item 2 for a destination that logs its own line next to the collector's", async () => { + await expect( + conformance('destination', loud, { profile: 'http', ...fixtures }), + ).rejects.toMatchObject({ + items: [2], + message: expect.stringContaining( + 'HTTP 500 writes 2 ERROR lines for transport', + ), + }); + }); + it('refuses fake timers with a clear message', async () => { jest.useFakeTimers(); await expect( @@ -489,6 +502,21 @@ describe('conformance kit, vendor-global profile', () => { }); }); + it('counts item 2 per cause: a vendor-timeout line before the script error does not count', async () => { + await expect( + conformance('destination', scriptVendor, { + ...options, + schema: { type: 'object', properties: {} }, + settings: {}, + examples: { + pageView: { in: event }, + silent: examples.silent, + revoke: examples.revoke, + }, + }), + ).resolves.toBeUndefined(); + }); + describe('item 8 runs a revoke gated on its keys, granted first (I1)', () => { it('passes a destination that passes the revocation to its vendor', async () => { await expect( diff --git a/packages/collector/src/dev/__tests__/fixture-package/src/index.ts b/packages/collector/src/dev/__tests__/fixture-package/src/index.ts index 2c5459a90..c5ff8b2bd 100644 --- a/packages/collector/src/dev/__tests__/fixture-package/src/index.ts +++ b/packages/collector/src/dev/__tests__/fixture-package/src/index.ts @@ -127,3 +127,17 @@ export const swallowing: Destination.Instance = { ); }, }; + +/** Logs its own ERROR line next to the collector's: two lines for one cause (item 2). */ +export const loud: Destination.Instance = { + ...good, + type: 'loud', + push: async (event, context) => { + try { + await good.push(event, context); + } catch (error) { + context.logger.error('Vendor request failed'); + throw error; + } + }, +}; diff --git a/packages/collector/src/dev/__tests__/fixture-package/src/vendor.ts b/packages/collector/src/dev/__tests__/fixture-package/src/vendor.ts index 66180db95..a278294f5 100644 --- a/packages/collector/src/dev/__tests__/fixture-package/src/vendor.ts +++ b/packages/collector/src/dev/__tests__/fixture-package/src/vendor.ts @@ -1,4 +1,5 @@ import { isObject, StepError } from '@walkeros/core'; +import type { Collector } from '@walkeros/core'; import type { Destination } from '@walkeros/core'; export interface Vendor { @@ -107,3 +108,47 @@ export const consentIgnoring: Destination.Instance = { type: 'consent-ignoring', on: undefined, }; + +/** Whether a page's vendor script fired `error`, per collector (one page). */ +const scriptFailed = new WeakMap(); + +/** + * Loads its vendor script as loadVendor does (marked, awaited): a push + * waits for the script and fails `vendor-timeout` at a short deadline (jsdom + * never loads it), and once the script fired `error` every push fails + * `vendor-not-loaded`: two causes, one line each. + */ +export const scriptVendor: Destination.Instance = { + type: 'script-vendor', + config: {}, + init: ({ collector }) => { + const script = document.createElement('script'); + script.src = 'https://vendor.example/sdk.js'; + script.setAttribute('data-walkeros-vendor', ''); + script.addEventListener('error', () => scriptFailed.set(collector, true)); + document.head.appendChild(script); + }, + push: async (_, { collector, rule }) => { + if (rule?.silent) return; + if (scriptFailed.get(collector)) + throw new StepError('Vendor script failed to load', { + code: 'vendor-not-loaded', + }); + await new Promise((__, reject) => + setTimeout( + () => + reject( + new StepError('Vendor script did not load within 50ms', { + code: 'vendor-timeout', + }), + ), + 50, + ), + ); + }, + on: (type, { env, data }) => { + const vendor = env.vendor; + if (type !== 'consent' || !isVendor(vendor) || !isObject(data)) return; + vendor.consent(data.marketing === true); + }, +}; diff --git a/packages/collector/src/dev/conformance.ts b/packages/collector/src/dev/conformance.ts index 4d37326d0..57afd3e30 100644 --- a/packages/collector/src/dev/conformance.ts +++ b/packages/collector/src/dev/conformance.ts @@ -9,7 +9,7 @@ import type { WalkerOS, } from '@walkeros/core'; import type { JSONSchema } from '@walkeros/core/dev'; -import { getEvent, isObject, Level, StepError } from '@walkeros/core'; +import { getEvent, isObject, Level, StepError, stepId } from '@walkeros/core'; import type { Answer, EnvAnswer } from './boundary'; import type { ExamplesEnv, @@ -166,6 +166,22 @@ function contextOf(options: ConformanceOptions): Context { const errorsIn = (logs: LogLine[]) => logs.filter((line) => line.level === Level.ERROR).length; +/** + * ERROR lines of the cause under test (item 2): the step's cause-log lines + * with that code, read from their structured context, plus lines without a + * code (the cause log always writes one, so such a line is the step's own + * report of the failure). Lines of other causes do not count. + */ +function causeLines(logs: LogLine[], code: string): number { + const step = stepId('destination', STEP_ID); + return logs.filter( + ({ level, context }) => + level === Level.ERROR && + (context.code === undefined || + (context.code === code && context.step === step)), + ).length; +} + function textOf(value: unknown): string { if (typeof value === 'string') return value; if (value instanceof Error) return value.message; @@ -474,9 +490,9 @@ function reportProblem(label: string, run: DestinationRun): Problem { if (!run.dlq.length) return `${label} leaves the DLQ empty`; // The whole session: a destination initialized inside startFlow writes // its one line there, and the run's retry of the same cause is suppressed. - const errors = errorsIn(run.sessionLogs); + const errors = causeLines(run.sessionLogs, reason.code); if (errors !== 1) - return `${label} writes ${errors} ERROR lines instead of the collector's one`; + return `${label} writes ${errors} ERROR lines for ${reason.code} instead of the collector's one`; return undefined; } @@ -739,10 +755,10 @@ async function vendorFailureProblem( return `a push after the vendor script failed is not reported vendor-not-loaded (${show(reason)})`; const dlq = session.collector.destinations[STEP_ID]?.dlq ?? []; if (!dlq.length) return 'the vendor-not-loaded event is not in the DLQ'; - const errors = errorsIn(session.logs); + const errors = causeLines(session.logs, reason.code); return errors === 1 ? undefined - : `vendor-not-loaded writes ${errors} ERROR lines instead of the collector's one`; + : `vendor-not-loaded writes ${errors} ERROR lines for ${reason.code} instead of the collector's one`; } finally { await session.close(); } From 09fbb60b38e946448ed0ca25215e5c2f21510640 Mon Sep 17 00:00:00 2001 From: alexanderkirtzel Date: Wed, 21 Oct 2026 05:36:25 +0000 Subject: [PATCH 13/41] web vendors --- packages/web/destinations/amplitude/README.md | 8 + .../src/__tests__/conformance.test.ts | 42 ++ .../amplitude/src/__tests__/index.test.ts | 41 ++ .../amplitude/src/__tests__/realSdk.test.ts | 71 +++ .../src/__tests__/stepExamples.test.ts | 15 +- .../amplitude/src/examples/env.ts | 7 +- .../amplitude/src/examples/step.ts | 31 + .../web/destinations/amplitude/src/index.ts | 533 +++++++++++------- .../amplitude/src/schemas/settings.ts | 6 +- .../destinations/amplitude/src/types/index.ts | 97 ++-- .../web/destinations/clarity/jest.config.mjs | 7 +- .../clarity/src/__tests__/conformance.test.ts | 23 + .../clarity/src/__tests__/index.test.ts | 75 +++ .../src/__tests__/stepExamples.test.ts | 23 +- .../destinations/clarity/src/examples/step.ts | 49 +- .../web/destinations/clarity/src/index.ts | 318 ++++++++--- .../clarity/src/schemas/settings.ts | 42 +- .../destinations/clarity/src/types/index.ts | 19 +- .../src/__tests__/conformance.test.ts | 21 + .../fullstory/src/__tests__/index.test.ts | 48 ++ .../src/__tests__/stepExamples.test.ts | 7 +- .../fullstory/src/examples/step.ts | 60 ++ .../web/destinations/fullstory/src/index.ts | 254 ++++++--- .../fullstory/src/schemas/settings.ts | 104 ++-- .../gtag/src/__tests__/conformance.test.ts | 21 + .../gtag/src/__tests__/droppedParams.test.ts | 44 ++ .../gtag/src/__tests__/ga4.test.ts | 126 +++-- .../gtag/src/__tests__/gtm.test.ts | 27 +- .../gtag/src/__tests__/index.test.ts | 74 ++- .../gtag/src/__tests__/reservedParams.test.ts | 19 +- .../gtag/src/__tests__/shared.test.ts | 72 ++- .../gtag/src/__tests__/stepExamples.test.ts | 5 +- .../web/destinations/gtag/src/ads/config.ts | 28 +- .../web/destinations/gtag/src/ads/push.ts | 17 +- .../web/destinations/gtag/src/examples/env.ts | 16 - .../destinations/gtag/src/examples/step.ts | 35 ++ .../web/destinations/gtag/src/ga4/config.ts | 26 +- .../web/destinations/gtag/src/ga4/push.ts | 30 +- .../web/destinations/gtag/src/gtm/config.ts | 49 +- .../web/destinations/gtag/src/gtm/push.ts | 4 +- packages/web/destinations/gtag/src/index.ts | 179 +++--- .../destinations/gtag/src/schemas/settings.ts | 25 +- .../web/destinations/gtag/src/shared/gtag.ts | 72 ++- .../web/destinations/gtag/src/shared/tools.ts | 34 ++ .../web/destinations/gtag/src/types/index.ts | 11 - .../heap/src/__tests__/conformance.test.ts | 21 + .../heap/src/__tests__/index.test.ts | 73 +++ .../heap/src/__tests__/stepExamples.test.ts | 103 ++-- .../destinations/heap/src/examples/step.ts | 41 +- packages/web/destinations/heap/src/index.ts | 248 ++++---- .../destinations/heap/src/schemas/settings.ts | 83 ++- .../web/destinations/heap/src/types/index.ts | 29 +- .../matomo/src/__tests__/conformance.test.ts | 21 + .../matomo/src/__tests__/stepExamples.test.ts | 4 +- .../destinations/matomo/src/examples/env.ts | 22 +- .../destinations/matomo/src/examples/step.ts | 36 ++ .../web/destinations/matomo/src/index.test.ts | 181 +++--- packages/web/destinations/matomo/src/index.ts | 192 ++++--- .../matomo/src/schemas/settings.ts | 72 +-- .../web/destinations/matomo/src/skip.test.ts | 28 - packages/web/destinations/matomo/src/skip.ts | 23 - .../destinations/matomo/src/types/index.ts | 11 - .../meta/src/__tests__/conformance.test.ts | 21 + .../meta/src/__tests__/stepExamples.test.ts | 3 +- .../web/destinations/meta/src/examples/env.ts | 21 - .../destinations/meta/src/examples/step.ts | 25 + .../web/destinations/meta/src/index.test.ts | 43 +- packages/web/destinations/meta/src/index.ts | 109 +++- .../destinations/meta/src/schemas/settings.ts | 12 +- packages/web/destinations/meta/src/setup.ts | 12 +- .../web/destinations/meta/src/types/index.ts | 9 - .../piano/src/__tests__/conformance.test.ts | 21 + .../piano/src/__tests__/script.test.ts | 53 ++ .../piano/src/__tests__/stepExamples.test.ts | 3 +- .../destinations/piano/src/examples/step.ts | 22 + .../web/destinations/piano/src/index.test.ts | 117 ++-- packages/web/destinations/piano/src/index.ts | 94 +-- .../piano/src/schemas/settings.ts | 30 +- .../src/__tests__/conformance.test.ts | 24 + .../piwikpro/src/__tests__/skip.test.ts | 28 - .../src/__tests__/stepExamples.test.ts | 4 +- .../destinations/piwikpro/src/examples/env.ts | 18 - .../piwikpro/src/examples/step.ts | 37 ++ .../destinations/piwikpro/src/index.test.ts | 116 ++-- .../web/destinations/piwikpro/src/index.ts | 171 ++++-- .../piwikpro/src/schemas/settings.ts | 64 ++- .../web/destinations/piwikpro/src/skip.ts | 23 - .../destinations/piwikpro/src/types/index.ts | 11 - .../segment/src/__tests__/conformance.test.ts | 41 ++ .../segment/src/__tests__/index.test.ts | 45 ++ .../src/__tests__/stepExamples.test.ts | 3 +- .../destinations/segment/src/examples/step.ts | 49 +- .../web/destinations/segment/src/index.ts | 319 ++++++----- .../destinations/segment/src/types/index.ts | 6 +- 94 files changed, 3594 insertions(+), 1963 deletions(-) create mode 100644 packages/web/destinations/amplitude/src/__tests__/conformance.test.ts create mode 100644 packages/web/destinations/amplitude/src/__tests__/index.test.ts create mode 100644 packages/web/destinations/amplitude/src/__tests__/realSdk.test.ts create mode 100644 packages/web/destinations/clarity/src/__tests__/conformance.test.ts create mode 100644 packages/web/destinations/clarity/src/__tests__/index.test.ts create mode 100644 packages/web/destinations/fullstory/src/__tests__/conformance.test.ts create mode 100644 packages/web/destinations/fullstory/src/__tests__/index.test.ts create mode 100644 packages/web/destinations/gtag/src/__tests__/conformance.test.ts create mode 100644 packages/web/destinations/gtag/src/__tests__/droppedParams.test.ts create mode 100644 packages/web/destinations/gtag/src/shared/tools.ts create mode 100644 packages/web/destinations/heap/src/__tests__/conformance.test.ts create mode 100644 packages/web/destinations/heap/src/__tests__/index.test.ts create mode 100644 packages/web/destinations/matomo/src/__tests__/conformance.test.ts delete mode 100644 packages/web/destinations/matomo/src/skip.test.ts delete mode 100644 packages/web/destinations/matomo/src/skip.ts create mode 100644 packages/web/destinations/meta/src/__tests__/conformance.test.ts create mode 100644 packages/web/destinations/piano/src/__tests__/conformance.test.ts create mode 100644 packages/web/destinations/piano/src/__tests__/script.test.ts create mode 100644 packages/web/destinations/piwikpro/src/__tests__/conformance.test.ts delete mode 100644 packages/web/destinations/piwikpro/src/__tests__/skip.test.ts delete mode 100644 packages/web/destinations/piwikpro/src/skip.ts create mode 100644 packages/web/destinations/segment/src/__tests__/conformance.test.ts create mode 100644 packages/web/destinations/segment/src/__tests__/index.test.ts diff --git a/packages/web/destinations/amplitude/README.md b/packages/web/destinations/amplitude/README.md index f5db863fb..6094d9be1 100644 --- a/packages/web/destinations/amplitude/README.md +++ b/packages/web/destinations/amplitude/README.md @@ -45,6 +45,14 @@ npm install @walkeros/web-destination-amplitude } ``` +## Plugins + +With only `apiKey` and the analytics options, only Amplitude analytics starts. +Setting `sessionReplay`, `experiment` (with a `deploymentKey`) or `engagement` +starts the plugins through `@amplitude/unified`, which sets up all three +together: enabling one also sets up Session Replay, Experiment and Guides & +Surveys. + ## Documentation Full configuration, mapping, and examples live in the docs: diff --git a/packages/web/destinations/amplitude/src/__tests__/conformance.test.ts b/packages/web/destinations/amplitude/src/__tests__/conformance.test.ts new file mode 100644 index 000000000..0e04752b7 --- /dev/null +++ b/packages/web/destinations/amplitude/src/__tests__/conformance.test.ts @@ -0,0 +1,42 @@ +import path from 'path'; +import { conformance } from '@walkeros/collector/dev'; +import { destinationAmplitude } from '..'; +import { examples, schemas } from '../dev'; + +// Amplitude BrowserOptions are forwarded into the SDK as a whole. +const passThrough = [ + 'serverZone', + 'flushIntervalMillis', + 'flushQueueSize', + 'flushMaxRetries', + 'transport', + 'useBatch', + 'appVersion', +]; + +describe('amplitude conformance', () => { + beforeEach(() => { + jest.useRealTimers(); + }); + + it('conforms', async () => { + await conformance('destination', destinationAmplitude, { + profile: 'web-sdk', + examples: examples.step, + schema: schemas.settings, + settings: { + apiKey: 'a1b2c3d4e5f6789012345678abcdef12', + serverZone: 'EU', + flushIntervalMillis: 1000, + flushQueueSize: 30, + flushMaxRetries: 5, + transport: 'fetch', + useBatch: false, + appVersion: '1.0.0', + }, + env: { push: examples.env.push, simulation: examples.env.simulation }, + root: path.join(__dirname, '..', '..'), + passThrough, + }); + }); +}); diff --git a/packages/web/destinations/amplitude/src/__tests__/index.test.ts b/packages/web/destinations/amplitude/src/__tests__/index.test.ts new file mode 100644 index 000000000..8bb1e4abf --- /dev/null +++ b/packages/web/destinations/amplitude/src/__tests__/index.test.ts @@ -0,0 +1,41 @@ +import { startFlow } from '@walkeros/collector'; +import { clone, getEvent } from '@walkeros/core'; +import { destinationAmplitude } from '..'; +import { examples } from '../dev'; +import type { Settings } from '../types'; + +async function initCalls(settings: Settings): Promise { + const calls: string[] = []; + const env = clone(examples.env.push); + const amplitude = env.amplitude; + if (!amplitude) throw new Error('env.amplitude missing'); + amplitude.init = () => { + calls.push('init'); + return { promise: Promise.resolve() }; + }; + amplitude.initAll = () => { + calls.push('initAll'); + return Promise.resolve(); + }; + const { elb } = await startFlow({ + destinations: { + amplitude: { code: destinationAmplitude, config: { settings, env } }, + }, + }); + await elb(getEvent('page view')); + return calls; +} + +describe('amplitude destination', () => { + it('starts analytics alone when no plugin is enabled', async () => { + expect(await initCalls({ apiKey: 'key' })).toEqual(['init']); + }); + + it.each<[string, Partial]>([ + ['sessionReplay', { sessionReplay: { sampleRate: 1 } }], + ['experiment', { experiment: { deploymentKey: 'deployment' } }], + ['engagement', { engagement: true }], + ])('starts the plugins when %s is set', async (_name, plugin) => { + expect(await initCalls({ apiKey: 'key', ...plugin })).toEqual(['initAll']); + }); +}); diff --git a/packages/web/destinations/amplitude/src/__tests__/realSdk.test.ts b/packages/web/destinations/amplitude/src/__tests__/realSdk.test.ts new file mode 100644 index 000000000..3e04389bf --- /dev/null +++ b/packages/web/destinations/amplitude/src/__tests__/realSdk.test.ts @@ -0,0 +1,71 @@ +import type { Logger } from '@walkeros/core'; +import type * as Unified from '@amplitude/unified'; +import type { UnifiedClient } from '@amplitude/unified'; +import { startFlow } from '@walkeros/collector'; +import { getEvent, Level } from '@walkeros/core'; +import { destinationAmplitude } from '..'; + +// The real @amplitude/unified, with the client the destination creates kept. +const mockClients: UnifiedClient[] = []; +jest.mock('@amplitude/unified', () => { + const actual = jest.requireActual('@amplitude/unified'); + return { + ...actual, + createInstance: () => { + const client = actual.createInstance(); + mockClients.push(client); + return client; + }, + }; +}); + +describe('amplitude destination on the real SDK', () => { + const fetchBefore = globalThis.fetch; + const requests: string[] = []; + + beforeEach(() => { + // The SDK's own timers (remote config, flush) run for real; the network + // is a stub that answers every request with an empty 200. + jest.useRealTimers(); + globalThis.fetch = async (input) => { + requests.push(String(input)); + return new Response('{}', { status: 200 }); + }; + }); + + afterEach(() => { + globalThis.fetch = fetchBefore; + }); + + it('starts analytics alone with apiKey only: no ERROR line, its options applied, no plugin', async () => { + const levels: Logger.Level[] = []; + const { elb } = await startFlow({ + // The default level: only ERROR lines reach the handler. + logger: { handler: (level) => levels.push(level) }, + destinations: { + amplitude: { + code: destinationAmplitude, + config: { settings: { apiKey: 'a1b2c3d4e5f6789012345678abcdef12' } }, + }, + }, + }); + + expect((await elb(getEvent('page view'))).ok).toBe(true); + + expect(levels.filter((level) => level === Level.ERROR)).toEqual([]); + expect(mockClients).toHaveLength(1); + const [client] = mockClients; + // Options passed as the user id would show up here. + expect(client.getUserId()).toBeUndefined(); + expect(client.experiment()).toBeUndefined(); + expect(client.sessionReplay()).toBeUndefined(); + // Only analytics hosts are asked: its remote config and its event API. + expect( + requests.filter( + (url) => + !url.startsWith('https://sr-client-cfg.amplitude.com/config/') && + !url.startsWith('https://api2.amplitude.com/'), + ), + ).toEqual([]); + }); +}); diff --git a/packages/web/destinations/amplitude/src/__tests__/stepExamples.test.ts b/packages/web/destinations/amplitude/src/__tests__/stepExamples.test.ts index 06f97191c..90121fa7b 100644 --- a/packages/web/destinations/amplitude/src/__tests__/stepExamples.test.ts +++ b/packages/web/destinations/amplitude/src/__tests__/stepExamples.test.ts @@ -119,6 +119,10 @@ function spyEnv(env: Env): { env: Env; collected: () => CallRecord[] } { calls.push([`amplitude.${name}`, ...args]); }; env.amplitude = { + init: (...args: unknown[]) => { + calls.push(['amplitude.init', ...args]); + return { promise: Promise.resolve() }; + }, initAll: (async (...args: unknown[]) => { calls.push(['amplitude.initAll', ...args]); }) as NonNullable['initAll'], @@ -150,8 +154,6 @@ function spyEnv(env: Env): { env: Env; collected: () => CallRecord[] } { groupIdentify: ((type: string, name: string, id: RecordingIdentify) => { calls.push(['amplitude.groupIdentify', type, name, id.ops]); }) as NonNullable['groupIdentify'], - flush: () => ({ promise: Promise.resolve() }), - add: () => ({ promise: Promise.resolve() }), Identify: RecordingIdentify, Revenue: RecordingRevenue, }; @@ -159,7 +161,12 @@ function spyEnv(env: Env): { env: Env; collected: () => CallRecord[] } { } describe('amplitude destination — step examples', () => { - it.each<[string, AmplitudeStepExample]>(Object.entries(examples.step))( + // Non-public examples (silent, failures) run in the conformance test. + const published = Object.entries(examples.step).filter( + ([, example]) => example.public !== false, + ); + + it.each<[string, AmplitudeStepExample]>(published)( '%s', async (name, example) => { const env = clone(examples.env.push) as Env; @@ -210,7 +217,7 @@ describe('amplitude destination — step examples', () => { // Drop init — every example triggers init once; it's not part of `out`. const expected = (example.out ?? []) as ReadonlyArray; const actual = collected().filter( - ([path]) => path !== 'amplitude.initAll', + ([path]) => path !== 'amplitude.initAll' && path !== 'amplitude.init', ); expect(actual).toEqual(expected); diff --git a/packages/web/destinations/amplitude/src/examples/env.ts b/packages/web/destinations/amplitude/src/examples/env.ts index 68974b4e6..e71d10df1 100644 --- a/packages/web/destinations/amplitude/src/examples/env.ts +++ b/packages/web/destinations/amplitude/src/examples/env.ts @@ -112,6 +112,7 @@ class MockRevenue implements RevenueInstance { */ export const init: Env | undefined = { amplitude: { + init: noopPromise, initAll: () => Promise.resolve(), track: noop, identify: noop, @@ -123,8 +124,6 @@ export const init: Env | undefined = { setSessionId: noop, setGroup: noop, groupIdentify: noop, - flush: noopPromise, - add: noopPromise, Identify: MockIdentify, Revenue: MockRevenue, }, @@ -136,6 +135,7 @@ export const init: Env | undefined = { */ export const push: Env = { amplitude: { + init: noopPromise, initAll: () => Promise.resolve(), track: noop, identify: noop, @@ -147,8 +147,6 @@ export const push: Env = { setSessionId: noop, setGroup: noop, groupIdentify: noop, - flush: noopPromise, - add: noopPromise, Identify: MockIdentify, Revenue: MockRevenue, }, @@ -156,6 +154,7 @@ export const push: Env = { /** Simulation tracking paths for CLI --simulate. */ export const simulation = [ + 'call:amplitude.init', 'call:amplitude.initAll', 'call:amplitude.track', 'call:amplitude.identify', diff --git a/packages/web/destinations/amplitude/src/examples/step.ts b/packages/web/destinations/amplitude/src/examples/step.ts index f1cd56f0e..cf525484c 100644 --- a/packages/web/destinations/amplitude/src/examples/step.ts +++ b/packages/web/destinations/amplitude/src/examples/step.ts @@ -435,3 +435,34 @@ export const consentGrantOptIn: AmplitudeStepExample = { settings: {} as Partial, out: [['amplitude.setOptOut', false]], }; + +/** A silent rule without settings of its own calls nothing. */ +export const silentRule: AmplitudeStepExample = { + public: false, + description: 'A rule with silent: true and no settings calls nothing.', + in: getEvent('page view', { timestamp: 1700000120 }), + mapping: { silent: true }, + out: [], +}; + +/** + * The silent group rule of groupAssignmentWithProperties on an update + * without data: no group name resolves and the rule suppresses the track, + * so nothing is sent. + */ +export const groupAssignmentWithoutData: AmplitudeStepExample = { + public: false, + description: + 'A silent rule whose group mapping finds no company sends nothing, so the push fails with missing-input instead of counting as delivered.', + in: { name: 'company update', entity: 'company', action: 'update' }, + mapping: { + silent: true, + settings: { + group: { + map: { type: { value: 'company' }, name: 'data.company' }, + }, + }, + }, + out: [], + outcome: { status: 'failed', code: 'missing-input' }, +}; diff --git a/packages/web/destinations/amplitude/src/index.ts b/packages/web/destinations/amplitude/src/index.ts index 29a3388f9..b4b4a797e 100644 --- a/packages/web/destinations/amplitude/src/index.ts +++ b/packages/web/destinations/amplitude/src/index.ts @@ -1,17 +1,21 @@ -import type { WalkerOS } from '@walkeros/core'; +import type { Collector, WalkerOS } from '@walkeros/core'; import { + getGrantedConsent, getMappingValue, isArray, isDefined, isNumber, isObject, isString, + StepError, } from '@walkeros/core'; import * as amplitudeSDK from '@amplitude/unified'; import type { AmplitudeSDK, Destination, - Env, + IdentifyInstance, + Mapping, + RevenueInstance, RuntimeState, Settings, } from './types'; @@ -19,16 +23,94 @@ import type { // Types export export * as DestinationAmplitude from './types'; +type RealSDK = AmplitudeSDK; + +let real: RealSDK | undefined; + +/** + * The real SDK: one @amplitude/unified client, as the module's own default + * client is one per page. Created on first use; its `init` sets up analytics + * alone, `initAll` adds the three plugins. + */ +function realAmplitude(): RealSDK { + if (real) return real; + const client = amplitudeSDK.createInstance(); + real = { + // The unified client forwards three arguments, and with three the + // analytics SDK reads the second as a user id: the options go third. + init: (apiKey, options) => client.init(apiKey, undefined, options), + initAll: (apiKey, options) => client.initAll(apiKey, options), + track: (eventType, eventProperties) => + client.track(eventType, eventProperties), + identify: (identify) => client.identify(identify), + revenue: (revenue) => client.revenue(revenue), + reset: () => client.reset(), + setOptOut: (optOut) => client.setOptOut(optOut), + setUserId: (userId) => client.setUserId(userId), + setDeviceId: (deviceId) => client.setDeviceId(deviceId), + setSessionId: (sessionId) => client.setSessionId(sessionId), + setGroup: (groupType, groupName) => client.setGroup(groupType, groupName), + groupIdentify: (groupType, groupName, identify) => + client.groupIdentify(groupType, groupName, identify), + Identify: amplitudeSDK.Identify, + Revenue: amplitudeSDK.Revenue, + }; + return real; +} + +/** The caller-provided mock (tests), if the env carries one. */ +function mockOf(env: unknown): AmplitudeSDK | undefined { + const amplitude = isObject(env) ? env.amplitude : undefined; + if (amplitude === undefined || isAmplitudeSDK(amplitude)) return amplitude; + throw new StepError('env.amplitude is not an Amplitude SDK', { + code: 'config', + }); +} + +function isAmplitudeSDK(value: unknown): value is AmplitudeSDK { + return ( + isObject(value) && + [ + 'init', + 'initAll', + 'track', + 'identify', + 'revenue', + 'reset', + 'setOptOut', + 'setUserId', + 'setDeviceId', + 'setSessionId', + 'setGroup', + 'groupIdentify', + 'Identify', + 'Revenue', + ].every((key) => typeof value[key] === 'function') + ); +} + /** - * Resolve the Amplitude SDK: use the caller-provided mock when available - * (tests), otherwise fall back to the real @amplitude/unified - * namespace. Matches @walkeros/web-destination-clarity and - * @walkeros/server-destination-gcp (BigQuery). + * Waits for one SDK call: its promise, or the `promise` an AmplitudeReturn + * carries. A throw or a rejection becomes a StepError. */ -function getAmplitude(env: Env | undefined): AmplitudeSDK { - return env?.amplitude ?? (amplitudeSDK as unknown as AmplitudeSDK); +async function call( + fn: () => unknown, + code: string, + message: string, + settle = false, +): Promise { + try { + const result = await fn(); + if (settle && isObject(result) && result.promise instanceof Promise) + await result.promise; + } catch (error) { + throw new StepError(message, { code, cause: error }); + } } +const send = (fn: () => unknown) => + call(fn, 'transport', 'Amplitude call failed'); + /** * Deterministic string → positive integer hash (djb2). * Same input always produces the same output, so Amplitude's @@ -47,20 +129,24 @@ function hashStringToInt(value: string): number { * Compare resolved identity against runtime state, call only the setters * that actually changed. Returns the updated state snapshot. */ -function applyIdentitySetters( - amp: AmplitudeSDK, +async function applyIdentitySetters< + I extends IdentifyInstance, + R extends RevenueInstance, +>( + amp: AmplitudeSDK, + sent: (fn: () => unknown) => Promise, resolved: Record, lastIdentity: RuntimeState['lastIdentity'] = {}, -): NonNullable { +): Promise> { const updated = { ...lastIdentity }; if ('user' in resolved) { const user = resolved.user; if (user === null || user === '') { - amp.setUserId(undefined); + await sent(() => amp.setUserId(undefined)); delete updated.user; } else if (isString(user) && user !== lastIdentity.user) { - amp.setUserId(user); + await sent(() => amp.setUserId(user)); updated.user = user; } } @@ -68,7 +154,7 @@ function applyIdentitySetters( if ('device' in resolved) { const device = resolved.device; if (isString(device) && device !== lastIdentity.device) { - amp.setDeviceId(device); + await sent(() => amp.setDeviceId(device)); updated.device = device; } } @@ -88,7 +174,7 @@ function applyIdentitySetters( ? hashStringToInt(session) : undefined; if (asNumber !== undefined && asNumber !== lastIdentity.session) { - amp.setSessionId(asNumber); + await sent(() => amp.setSessionId(asNumber)); updated.session = asNumber; } } @@ -122,12 +208,12 @@ const IDENTIFY_OP_KEYS: IdentifyOpKey[] = [ * keys. Returns undefined if no operations are present (so the caller * can skip amplitude.identify() entirely). */ -function buildIdentify( - amp: AmplitudeSDK, +function buildIdentify( + amp: AmplitudeSDK, resolved: Record, -): InstanceType | undefined { - let identify: InstanceType | undefined; - const ensure = () => { +): I | undefined { + let identify: I | undefined; + const ensure = (): I => { if (!identify) identify = new amp.Identify(); return identify; }; @@ -158,210 +244,261 @@ function buildIdentify( return identify; } -type RevenueFieldSetter = - | 'setProductId' - | 'setPrice' - | 'setQuantity' - | 'setRevenueType' - | 'setCurrency' - | 'setRevenue' - | 'setReceipt' - | 'setReceiptSig'; - -const REVENUE_FIELD_MAP: Record = { - productId: 'setProductId', - price: 'setPrice', - quantity: 'setQuantity', - revenueType: 'setRevenueType', - currency: 'setCurrency', - revenue: 'setRevenue', - receipt: 'setReceipt', - receiptSig: 'setReceiptSig', -}; - -function buildRevenue( - amp: AmplitudeSDK, +/** A Revenue instance from one resolved item, each field by its type. */ +function buildRevenue( + amp: AmplitudeSDK, item: Record, -): InstanceType { +): R { const rev = new amp.Revenue(); - for (const [key, setter] of Object.entries(REVENUE_FIELD_MAP)) { - const value = item[key]; - if (!isDefined(value)) continue; - (rev[setter] as (v: unknown) => InstanceType)( - value, - ); - } - if (isObject(item.eventProperties)) { - rev.setEventProperties(item.eventProperties as Record); - } + const { productId, price, quantity, revenueType, currency, revenue } = item; + const { receipt, receiptSig, eventProperties } = item; + if (isString(productId)) rev.setProductId(productId); + if (isNumber(price)) rev.setPrice(price); + if (isNumber(quantity)) rev.setQuantity(quantity); + if (isString(revenueType)) rev.setRevenueType(revenueType); + if (isString(currency)) rev.setCurrency(currency); + if (isNumber(revenue)) rev.setRevenue(revenue); + if (isString(receipt)) rev.setReceipt(receipt); + if (isString(receiptSig)) rev.setReceiptSig(receiptSig); + if (isObject(eventProperties)) rev.setEventProperties(eventProperties); return rev; } -export const destinationAmplitude: Destination = { - type: 'amplitude', +/** Starts the SDK: analytics alone, or with the plugins the settings enable. */ +async function start( + amp: AmplitudeSDK, + settings: Settings, +): Promise { + const { + apiKey, + identify: _identify, + sessionReplay, + experiment, + engagement, + _state: _existingState, + ...browserOptions + } = settings; + + const analyticsOptions = { + autocapture: false, + identityStorage: 'none' as const, + ...browserOptions, + }; - config: {}, + // Session Replay, Experiment and Guides & Surveys are opt-in: without any + // of them only analytics starts. @amplitude/unified sets up all three + // together, so one enabled plugin brings the other two. + const plugins = Boolean( + sessionReplay || experiment?.deploymentKey || engagement, + ); + + if (!plugins) + return call( + () => amp.init(apiKey, analyticsOptions), + 'vendor-not-ready', + 'Amplitude init failed', + true, + ); - async init({ config, env }) { - const settings = config.settings; - if (!settings?.apiKey) return false; - - const amp = getAmplitude(env as Env | undefined); - - const { - apiKey, - identify: _identify, - sessionReplay, - experiment, - engagement, - _state: _existingState, - ...browserOptions - } = settings; - - const analyticsOptions = { - autocapture: false, - identityStorage: 'none' as const, - ...browserOptions, - }; - - await amp.initAll(apiKey, { - serverZone: analyticsOptions.serverZone, - analytics: analyticsOptions, - ...(sessionReplay ? { sessionReplay } : {}), - ...(experiment?.deploymentKey ? { experiment } : {}), - ...(engagement - ? { engagement: engagement === true ? undefined : engagement } - : {}), - }); + return call( + () => + amp.initAll(apiKey, { + serverZone: analyticsOptions.serverZone, + analytics: analyticsOptions, + ...(sessionReplay ? { sessionReplay } : {}), + ...(experiment?.deploymentKey ? { experiment } : {}), + ...(engagement + ? { engagement: engagement === true ? undefined : engagement } + : {}), + }), + 'vendor-not-ready', + 'Amplitude init failed', + ); +} - const _state: RuntimeState = { lastIdentity: {} }; - return { ...config, settings: { ...settings, _state } }; - }, +async function pushTo( + amp: AmplitudeSDK, + event: WalkerOS.Event, + settings: Settings, + mappingSettings: Mapping, + rule: { name?: unknown; silent?: boolean } | undefined, + data: unknown, + collector: Collector.Instance, +): Promise { + const state: RuntimeState = settings._state || {}; + let calls = 0; + const sent = (fn: () => unknown) => { + calls++; + return send(fn); + }; - async push(event, { config, rule, env, data, collector }) { - const amp = getAmplitude(env as Env | undefined); - const settings = (config.settings || {}) as Settings; - const mappingSettings = rule?.settings || {}; - const state: RuntimeState = settings._state || {}; - - // 1. Reset - fires first so subsequent identity calls start clean. - if (mappingSettings.reset !== undefined) { - const resolved = - typeof mappingSettings.reset === 'boolean' - ? mappingSettings.reset - : await getMappingValue(event, mappingSettings.reset, { - collector, - }); - if (resolved) { - amp.reset(); - state.lastIdentity = {}; - } + // 1. Reset - fires first so subsequent identity calls start clean. + if (mappingSettings.reset !== undefined) { + const resolved = + typeof mappingSettings.reset === 'boolean' + ? mappingSettings.reset + : await getMappingValue(event, mappingSettings.reset, { + collector, + }); + if (resolved) { + await sent(() => amp.reset()); + state.lastIdentity = {}; } + } - // 2. Identity setters - rule-level override wins over destination-level. - const identifyMapping = mappingSettings.identify ?? settings.identify; - let identifyResolved: Record | undefined; - if (identifyMapping !== undefined) { - const resolved = await getMappingValue(event, identifyMapping, { - collector, - }); - if (isObject(resolved)) { - identifyResolved = resolved as Record; - state.lastIdentity = applyIdentitySetters( - amp, - identifyResolved, - state.lastIdentity, - ); - } + // 2. Identity setters - rule-level override wins over destination-level. + const identifyMapping = mappingSettings.identify ?? settings.identify; + let identifyResolved: Record | undefined; + if (identifyMapping !== undefined) { + const resolved = await getMappingValue(event, identifyMapping, { + collector, + }); + if (isObject(resolved)) { + identifyResolved = resolved; + state.lastIdentity = await applyIdentitySetters( + amp, + sent, + resolved, + state.lastIdentity, + ); } + } - // 3. Identify operations - fires amplitude.identify() if the resolved - // object has any op keys (set, setOnce, add, ...). - if (identifyResolved) { - const id = buildIdentify(amp, identifyResolved); - if (id) amp.identify(id); - } + // 3. Identify operations - fires amplitude.identify() if the resolved + // object has any op keys (set, setOnce, add, ...). + if (identifyResolved) { + const id = buildIdentify(amp, identifyResolved); + if (id) await sent(() => amp.identify(id)); + } - // 4. Group assignment - if (mappingSettings.group !== undefined) { - const resolved = await getMappingValue(event, mappingSettings.group, { - collector, - }); - if (isObject(resolved)) { - const { type, name } = resolved as { - type?: unknown; - name?: unknown; - }; - if (isString(type) && (isString(name) || isArray(name))) { - amp.setGroup(type, name as string | string[]); - } - } + // 4. Group assignment + if (mappingSettings.group !== undefined) { + const resolved = await getMappingValue(event, mappingSettings.group, { + collector, + }); + if (isObject(resolved)) { + const { type, name } = resolved; + if (isString(type) && (isString(name) || isStringArray(name))) + await sent(() => amp.setGroup(type, name)); } + } - // 5. Group identify (properties on the group) - if (mappingSettings.groupIdentify !== undefined) { - const resolved = await getMappingValue( - event, - mappingSettings.groupIdentify, - { collector }, - ); - if (isObject(resolved)) { - const { type, name } = resolved as { - type?: unknown; - name?: unknown; - }; - if (isString(type) && isString(name)) { - const id = buildIdentify(amp, resolved as Record); - if (id) amp.groupIdentify(type, name, id); - } + // 5. Group identify (properties on the group) + if (mappingSettings.groupIdentify !== undefined) { + const resolved = await getMappingValue( + event, + mappingSettings.groupIdentify, + { collector }, + ); + if (isObject(resolved)) { + const { type, name } = resolved; + if (isString(type) && isString(name)) { + const id = buildIdentify(amp, resolved); + if (id) await sent(() => amp.groupIdentify(type, name, id)); } } + } - // 6. Revenue - single object or array from loop → N revenue() calls. - if (mappingSettings.revenue !== undefined) { - const resolved = await getMappingValue(event, mappingSettings.revenue, { - collector, - }); - const items = isArray(resolved) - ? (resolved as Record[]) - : isObject(resolved) - ? [resolved as Record] - : []; - for (const item of items) { - if (!isObject(item)) continue; - amp.revenue(buildRevenue(amp, item)); - } + // 6. Revenue - single object or array from loop → N revenue() calls. + if (mappingSettings.revenue !== undefined) { + const resolved = await getMappingValue(event, mappingSettings.revenue, { + collector, + }); + const items = isArray(resolved) ? resolved : [resolved]; + for (const item of items) { + if (!isObject(item)) continue; + const revenue = buildRevenue(amp, item); + await sent(() => amp.revenue(revenue)); } + } - // 7. Default track - if (rule?.silent !== true) { - const eventType = isString(rule?.name) ? rule.name : event.name; - const eventProperties = isObject(data) - ? (data as Record) - : {}; - amp.track(eventType, eventProperties); - } + // 7. Default track + if (rule?.silent !== true) { + const eventType = isString(rule?.name) ? rule.name : event.name; + const eventProperties = isObject(data) ? data : {}; + await sent(() => amp.track(eventType, eventProperties)); + } - // Persist state mutations back onto config.settings. - settings._state = state; - }, + // Persist state mutations back onto config.settings. + settings._state = state; + + // A silent rule whose own settings resolved nothing sent nothing. + const ruleSettings = + mappingSettings.reset !== undefined || + mappingSettings.identify !== undefined || + mappingSettings.group !== undefined || + mappingSettings.groupIdentify !== undefined || + mappingSettings.revenue !== undefined; + if (!calls && ruleSettings) + throw new StepError( + `Amplitude: the silent rule of "${event.name}" resolved no identity, group or revenue`, + { code: 'missing-input' }, + ); +} + +function isStringArray(value: unknown): value is string[] { + return isArray(value) && value.every(isString); +} + +export const destinationAmplitude: Destination = { + type: 'amplitude', + + config: {}, - on(type, context) { - if (type !== 'consent' || !context.data) return; - const amp = getAmplitude(context.env as Env | undefined); + // The SDK starts in init, so it collects from page load on. + async init({ config, env }) { + const apiKey = config.settings?.apiKey; + if (!apiKey) + throw new StepError('Amplitude needs settings.apiKey', { + code: 'config', + }); + const settings: Settings = { ...config.settings, apiKey }; + + const mock = mockOf(env); + if (mock) await start(mock, settings); + else + await start(realAmplitude(), settings).catch((error: unknown) => { + // The SDK answers a second init after a failed one as done without + // starting: the next attempt gets a fresh client. + real = undefined; + throw error; + }); - const consent = context.data as WalkerOS.Consent; - // Derive the consent key from config.consent - iterate every key the - // destination declared as required. If ALL required keys are granted, - // opt IN; otherwise opt OUT. A missing config.consent means nothing - // to check and we take no action. - const required = context.config?.consent; - if (!required || Object.keys(required).length === 0) return; + const _state: RuntimeState = { lastIdentity: {} }; + return { ...config, settings: { ...settings, _state } }; + }, + + async push(event, { config, rule, env, data, collector }) { + const settings: Settings = { apiKey: '', ...config.settings }; + const mappingSettings: Mapping = rule?.settings || {}; + const mock = mockOf(env); + await (mock + ? pushTo(mock, event, settings, mappingSettings, rule, data, collector) + : pushTo( + realAmplitude(), + event, + settings, + mappingSettings, + rule, + data, + collector, + )); + }, - const allGranted = Object.keys(required).every( - (key) => consent[key] === true, + // The destination's required consent decides, as it gates the pushes: a + // revocation opts out of Amplitude, a grant opts back in. + async on(type, { config, data, env, collector }) { + const required = config.consent; + if (type !== 'consent' || !required || !isObject(data)) return; + if (!Object.keys(required).some((key) => key in data)) return; + + const granted = getGrantedConsent(required, collector.consent) !== false; + const amp = mockOf(env) ?? realAmplitude(); + await call( + () => amp.setOptOut(!granted), + 'consent-not-applied', + 'Amplitude did not take the consent update', ); - amp.setOptOut(!allGranted); }, }; diff --git a/packages/web/destinations/amplitude/src/schemas/settings.ts b/packages/web/destinations/amplitude/src/schemas/settings.ts index c508171d7..ffcc83230 100644 --- a/packages/web/destinations/amplitude/src/schemas/settings.ts +++ b/packages/web/destinations/amplitude/src/schemas/settings.ts @@ -52,19 +52,19 @@ export const SettingsSchema = z.object({ sessionReplay: z .unknown() .describe( - 'Session Replay options, passed to the Session Replay plugin. @amplitude/unified sets up the plugin on every init, whether or not this is set; it records a sampled share of sessions, set by the project sampling settings in Amplitude when present, else by sampleRate (default 0).', + 'Session Replay options, passed to the Session Replay plugin. Without this, experiment.deploymentKey or engagement only analytics starts. With any of the three set, @amplitude/unified sets up all three plugins together. It records a sampled share of sessions, set by the project sampling settings in Amplitude when present, else by sampleRate (default 0).', ) .optional(), experiment: z .unknown() .describe( - 'Feature Experiment options. Passed to the Experiment plugin only when they include `deploymentKey`. @amplitude/unified sets up the plugin on every init; without a deploymentKey it uses the API key.', + 'Feature Experiment options, passed to the Experiment plugin only when they include `deploymentKey`. Without it, sessionReplay or engagement only analytics starts. With any of the three set, @amplitude/unified sets up all three plugins together.', ) .optional(), engagement: z .unknown() .describe( - 'Guides & Surveys options object. @amplitude/unified loads the Guides & Surveys SDK from cdn.amplitude.com on every init, whether or not this is set; `true` passes no options.', + 'Guides & Surveys options object; `true` passes no options. Without it, sessionReplay or experiment.deploymentKey only analytics starts. With any of the three set, @amplitude/unified sets up all three plugins together, and Guides & Surveys loads its SDK from cdn.amplitude.com.', ) .optional(), }); diff --git a/packages/web/destinations/amplitude/src/types/index.ts b/packages/web/destinations/amplitude/src/types/index.ts index d68e7392b..2784896c5 100644 --- a/packages/web/destinations/amplitude/src/types/index.ts +++ b/packages/web/destinations/amplitude/src/types/index.ts @@ -3,7 +3,10 @@ import type { Destination as CoreDestination, } from '@walkeros/core'; import type { DestinationWeb } from '@walkeros/web-core'; -import type { Types as AmplitudeTypes } from '@amplitude/unified'; +import type { + Types as AmplitudeTypes, + UnifiedOptions, +} from '@amplitude/unified'; import type { SessionReplayOptions } from '@amplitude/plugin-session-replay-browser'; import type { ExperimentPluginConfig } from '@amplitude/plugin-experiment-browser'; import type { InitOptions as EngagementInitOptions } from '@amplitude/engagement-browser'; @@ -59,57 +62,61 @@ export interface Mapping { } /** - * Amplitude SDK surface — the subset of @amplitude/unified the - * destination actually uses. Mirrors the real module's named exports so - * tests can mock each method individually via env.amplitude. + * Amplitude SDK surface: the subset of @amplitude/unified the + * destination actually uses. Mirrors the real client's methods so tests can + * mock each one via env.amplitude. Generic over the Identify and Revenue + * instances, so the real SDK gets back the instances it created. */ -export interface AmplitudeSDK { - initAll: (apiKey: string, options?: Record) => Promise; - track: (eventType: string, eventProperties?: Record) => void; - identify: (identify: IdentifyInstance) => void; - revenue: (revenue: RevenueInstance) => void; - reset: () => void; - setOptOut: (optOut: boolean) => void; - setUserId: (userId: string | undefined) => void; - setDeviceId: (deviceId: string) => void; - setSessionId: (sessionId: number) => void; - setGroup: (groupType: string, groupName: string | string[]) => void; - groupIdentify: ( - groupType: string, - groupName: string, - identify: IdentifyInstance, - ) => void; - flush: () => { promise: Promise }; - add: (plugin: unknown) => { promise: Promise }; - Identify: new () => IdentifyInstance; - Revenue: new () => RevenueInstance; +export interface AmplitudeSDK< + I extends IdentifyInstance = IdentifyInstance, + R extends RevenueInstance = RevenueInstance, +> { + /** Analytics only, the plugins none. */ + init: (apiKey: string, options?: AmplitudeBrowserOptions) => unknown; + /** Analytics plus Session Replay, Experiment and Guides & Surveys. */ + initAll: (apiKey: string, options?: UnifiedOptions) => unknown; + track: ( + eventType: string, + eventProperties?: Record, + ) => unknown; + identify: (identify: I) => unknown; + revenue: (revenue: R) => unknown; + reset: () => unknown; + setOptOut: (optOut: boolean) => unknown; + setUserId: (userId: string | undefined) => unknown; + setDeviceId: (deviceId: string) => unknown; + setSessionId: (sessionId: number) => unknown; + setGroup: (groupType: string, groupName: string | string[]) => unknown; + groupIdentify: (groupType: string, groupName: string, identify: I) => unknown; + Identify: new () => I; + Revenue: new () => R; } -/** Chainable Identify — mirrors @amplitude/unified Identify class. */ +/** Chainable Identify, as the @amplitude/unified Identify class. */ export interface IdentifyInstance { - set: (property: string, value: unknown) => IdentifyInstance; - setOnce: (property: string, value: unknown) => IdentifyInstance; - add: (property: string, value: number) => IdentifyInstance; - append: (property: string, value: unknown) => IdentifyInstance; - prepend: (property: string, value: unknown) => IdentifyInstance; - preInsert: (property: string, value: unknown) => IdentifyInstance; - postInsert: (property: string, value: unknown) => IdentifyInstance; - remove: (property: string, value: unknown) => IdentifyInstance; - unset: (property: string) => IdentifyInstance; - clearAll: () => IdentifyInstance; + set(property: string, value: unknown): IdentifyInstance; + setOnce(property: string, value: unknown): IdentifyInstance; + add(property: string, value: number): IdentifyInstance; + append(property: string, value: unknown): IdentifyInstance; + prepend(property: string, value: unknown): IdentifyInstance; + preInsert(property: string, value: unknown): IdentifyInstance; + postInsert(property: string, value: unknown): IdentifyInstance; + remove(property: string, value: unknown): IdentifyInstance; + unset(property: string): IdentifyInstance; + clearAll(): IdentifyInstance; } -/** Chainable Revenue — mirrors @amplitude/unified Revenue class. */ +/** Chainable Revenue, as the @amplitude/unified Revenue class. */ export interface RevenueInstance { - setProductId: (id: string) => RevenueInstance; - setPrice: (price: number) => RevenueInstance; - setQuantity: (quantity: number) => RevenueInstance; - setRevenueType: (type: string) => RevenueInstance; - setCurrency: (currency: string) => RevenueInstance; - setRevenue: (revenue: number) => RevenueInstance; - setReceipt: (receipt: string) => RevenueInstance; - setReceiptSig: (signature: string) => RevenueInstance; - setEventProperties: (properties: Record) => RevenueInstance; + setProductId(id: string): RevenueInstance; + setPrice(price: number): RevenueInstance; + setQuantity(quantity: number): RevenueInstance; + setRevenueType(type: string): RevenueInstance; + setCurrency(currency: string): RevenueInstance; + setRevenue(revenue: number): RevenueInstance; + setReceipt(receipt: string): RevenueInstance; + setReceiptSig(signature: string): RevenueInstance; + setEventProperties(properties: Record): RevenueInstance; } /** diff --git a/packages/web/destinations/clarity/jest.config.mjs b/packages/web/destinations/clarity/jest.config.mjs index 9f5dcec3d..7f1fb7d54 100644 --- a/packages/web/destinations/clarity/jest.config.mjs +++ b/packages/web/destinations/clarity/jest.config.mjs @@ -1,5 +1,10 @@ import baseConfig from '@walkeros/config/jest/web.config'; -const config = {}; +// @microsoft/clarity ships ESM only: transform it like the workspace packages. +const config = { + transformIgnorePatterns: [ + '/node_modules/(?!(@walkeros|@microsoft/clarity)/)', + ], +}; export default { ...baseConfig, ...config }; diff --git a/packages/web/destinations/clarity/src/__tests__/conformance.test.ts b/packages/web/destinations/clarity/src/__tests__/conformance.test.ts new file mode 100644 index 000000000..4273954e9 --- /dev/null +++ b/packages/web/destinations/clarity/src/__tests__/conformance.test.ts @@ -0,0 +1,23 @@ +import path from 'path'; +import { conformance } from '@walkeros/collector/dev'; +import { destinationClarity } from '..'; +import { examples, schemas } from '../dev'; + +describe('clarity conformance', () => { + beforeEach(() => { + jest.useRealTimers(); + }); + + it('conforms', async () => { + await conformance('destination', destinationClarity, { + profile: 'vendor-global', + examples: examples.step, + schema: schemas.settings, + // No settings.consent: the revocation reaches Clarity through the + // destination's required consent alone. + settings: { apiKey: 'test-project' }, + env: { push: examples.env.push, simulation: examples.env.simulation }, + root: path.join(__dirname, '..', '..'), + }); + }); +}); diff --git a/packages/web/destinations/clarity/src/__tests__/index.test.ts b/packages/web/destinations/clarity/src/__tests__/index.test.ts new file mode 100644 index 000000000..79e423290 --- /dev/null +++ b/packages/web/destinations/clarity/src/__tests__/index.test.ts @@ -0,0 +1,75 @@ +import { startFlow } from '@walkeros/collector'; +import { getEvent } from '@walkeros/core'; +import { destinationClarity } from '..'; +import type { ClaritySDK } from '../types'; + +function recordingClarity(calls: unknown[][]): ClaritySDK { + const record = + (name: string) => + (...args: unknown[]) => { + calls.push([name, ...args]); + }; + return { + init: record('init'), + identify: record('identify'), + setTag: record('setTag'), + event: record('event'), + consentV2: record('consentV2'), + upgrade: record('upgrade'), + }; +} + +describe('clarity destination', () => { + it('passes a revocation to Clarity without settings.consent when the destination is consent-gated', async () => { + const calls: unknown[][] = []; + const { elb } = await startFlow({ + destinations: { + clarity: { + code: { + ...destinationClarity, + env: { clarity: recordingClarity(calls) }, + }, + config: { + consent: { marketing: true }, + settings: { apiKey: 'gated' }, + }, + }, + }, + }); + + await elb('walker consent', { marketing: true }); + await elb(getEvent('page view')); + await elb('walker consent', { marketing: false }); + + expect(calls).toEqual([ + ['init', 'gated'], + ['consentV2', { analytics_Storage: 'granted', ad_Storage: 'granted' }], + ['event', 'page view'], + ['consentV2', { analytics_Storage: 'denied', ad_Storage: 'denied' }], + ]); + }); + + it('fails pushes with vendor-not-loaded once the Clarity script failed to load', async () => { + const { elb } = await startFlow({ + destinations: { + clarity: { + code: destinationClarity, + config: { settings: { apiKey: 'blocked' } }, + }, + }, + }); + + expect((await elb(getEvent('page view'))).ok).toBe(true); + const script = document.querySelector( + 'script[src="https://www.clarity.ms/tag/blocked?ref=npm"]', + ); + expect(script?.hasAttribute('data-walkeros-vendor')).toBe(true); + script?.dispatchEvent(new Event('error')); + + const result = await elb(getEvent('product view')); + expect(result.ok).toBe(false); + expect(result.failed?.clarity?.reason).toMatchObject({ + code: 'vendor-not-loaded', + }); + }); +}); diff --git a/packages/web/destinations/clarity/src/__tests__/stepExamples.test.ts b/packages/web/destinations/clarity/src/__tests__/stepExamples.test.ts index c1076565a..159ddf906 100644 --- a/packages/web/destinations/clarity/src/__tests__/stepExamples.test.ts +++ b/packages/web/destinations/clarity/src/__tests__/stepExamples.test.ts @@ -1,12 +1,3 @@ -// The real @microsoft/clarity package is ESM and can't be parsed by Jest's -// default CommonJS transformer. Tests always wire their own mock via -// `env.clarity`, so the real module is never touched; we stub the import -// with an empty default export to let Jest load the destination file. -jest.mock('@microsoft/clarity', () => ({ - __esModule: true, - default: {}, -})); - import type { WalkerOS, Mapping as WalkerOSMapping } from '@walkeros/core'; import { startFlow } from '@walkeros/collector'; import { clone } from '@walkeros/core'; @@ -57,7 +48,12 @@ describe('clarity destination -- step examples', () => { expect(unnamed).toEqual([]); }); - it.each<[string, ClarityStepExample]>(Object.entries(examples.step))( + // Non-public examples (silent, failures) run in the conformance test. + const published = Object.entries(examples.step).filter( + ([, example]) => example.public !== false, + ); + + it.each<[string, ClarityStepExample]>(published)( '%s', async (name, example) => { const env = clone(examples.env.push) as Env; @@ -66,6 +62,8 @@ describe('clarity destination -- step examples', () => { const dest = jest.requireActual('../').default; const { elb } = await startFlow(); + // Effects before this index belong to the example's `before` grant. + let before = 0; const baseSettings: Partial & { apiKey: string } = { apiKey: 'test-project', ...(example.settings || {}), @@ -87,6 +85,7 @@ describe('clarity destination -- step examples', () => { // declares one, so a revoke acts on an already-granted destination. Both // the grant and the consent-under-test effects are asserted. if (example.before) await elb('walker consent', example.before); + before = collected().length; await elb('walker consent', example.in as WalkerOS.Consent); } else { // Standard event example. @@ -110,7 +109,9 @@ describe('clarity destination -- step examples', () => { // Drop the init call: every example triggers init once, it is not part // of the declared `out`. const expected = (example.out ?? []) as ReadonlyArray; - const actual = collected().filter(([path]) => path !== 'clarity.init'); + const actual = collected() + .slice(before) + .filter(([path]) => path !== 'clarity.init'); expect(actual).toEqual(expected); }, diff --git a/packages/web/destinations/clarity/src/examples/step.ts b/packages/web/destinations/clarity/src/examples/step.ts index 625f9c7d5..5abe7512c 100644 --- a/packages/web/destinations/clarity/src/examples/step.ts +++ b/packages/web/destinations/clarity/src/examples/step.ts @@ -295,7 +295,7 @@ export const consentGrantBoth: ClarityStepExample = { export const consentRevoke: ClarityStepExample = { title: 'Consent revoked', description: - 'With settings.consent mapping analytics to analytics_Storage and marketing to ad_Storage, a grant of both and then a walker consent command denying both call Clarity.consentV2 twice, the second time with denied flags.', + 'With settings.consent mapping analytics to analytics_Storage and marketing to ad_Storage, a walker consent command denying both calls Clarity.consentV2 with denied flags.', command: 'consent', before: { analytics: true, marketing: true }, in: { analytics: false, marketing: false } as WalkerOS.Consent, @@ -306,13 +306,52 @@ export const consentRevoke: ClarityStepExample = { }, }, out: [ - [ - 'clarity.consentV2', - { analytics_Storage: 'granted', ad_Storage: 'granted' }, - ], [ 'clarity.consentV2', { analytics_Storage: 'denied', ad_Storage: 'denied' }, ], ], }; + +/** A silent rule without settings of its own calls nothing. */ +export const silentRule: ClarityStepExample = { + public: false, + description: 'A rule with silent: true and no settings calls nothing.', + in: getEvent('page view', { timestamp: 1700000110 }), + mapping: { silent: true }, + out: [], +}; + +/** The Clarity SDK is unreachable: its init fails, so the push reports it. */ +export const clarityUnreachable: ClarityStepExample = { + public: false, + description: + 'The Clarity SDK fails, so the destination does not start: the push fails with vendor-not-ready, retryable, and the event is in the DLQ.', + in: getEvent('product view', { timestamp: 1700000111 }), + respond: { error: 'network' }, + out: [], + outcome: { status: 'failed', code: 'vendor-not-ready', retryable: true }, +}; + +/** + * The silent tag rule of pageViewSilent on a page view without data: no tag + * resolves and the rule suppresses the event, so nothing is sent. + */ +export const pageViewSilentWithoutData: ClarityStepExample = { + public: false, + description: + 'A silent rule whose tag mapping finds no data sends nothing, so the push fails with missing-input instead of counting as delivered.', + in: { name: 'page view', entity: 'page', action: 'view' }, + mapping: { + silent: true, + settings: { + set: { + map: { + page_id: 'data.id', + }, + }, + }, + }, + out: [], + outcome: { status: 'failed', code: 'missing-input' }, +}; diff --git a/packages/web/destinations/clarity/src/index.ts b/packages/web/destinations/clarity/src/index.ts index db74299b7..979a198b4 100644 --- a/packages/web/destinations/clarity/src/index.ts +++ b/packages/web/destinations/clarity/src/index.ts @@ -1,21 +1,132 @@ import type { WalkerOS } from '@walkeros/core'; -import { getMappingValue, isObject, isString } from '@walkeros/core'; +import { + getGrantedConsent, + getMappingValue, + isObject, + isString, + StepError, +} from '@walkeros/core'; +import { getEnv, loadVendor } from '@walkeros/web-core'; import Clarity from '@microsoft/clarity'; -import type { ClaritySDK, Destination, Env, Mapping, Settings } from './types'; +import type { + ClarityQueue, + ClaritySDK, + ConsentV2, + Destination, + Mapping, + Settings, +} from './types'; // Types export export * as DestinationClarity from './types'; +/** The Clarity tag script, as `@microsoft/clarity` addresses it. */ +const SCRIPT_SRC = 'https://www.clarity.ms/tag/'; + +type Category = keyof ConsentV2; + +/** + * The `window.clarity` stub the tag script replays, the one + * `@microsoft/clarity` installs before it adds the script. + */ +function installStub(): ClarityQueue { + const { window } = getEnv(); + if (window.clarity) return window.clarity; + const queue: unknown[] = []; + const clarity: ClarityQueue = Object.assign( + (...args: unknown[]) => { + queue.push(args); + }, + { q: queue }, + ); + window.clarity = clarity; + return clarity; +} + +/** The tag script `loadVendor` injects; it replays the stub's queue. */ +function vendor(apiKey: string, timeout?: number) { + return loadVendor( + `${SCRIPT_SRC}${apiKey}?ref=npm`, + () => getEnv().window.clarity, + { timeout }, + ); +} + +/** + * The real SDK: `@microsoft/clarity` for the calls, with its script added + * through `loadVendor`, so a script that fails to load fails every later + * call with `vendor-not-loaded`. + */ +function realClarity(apiKey: string, timeout?: number): ClaritySDK { + const loaded = () => vendor(apiKey, timeout).check(); + return { + init: () => { + installStub(); + vendor(apiKey, timeout); + }, + identify: (...args) => { + loaded(); + Clarity.identify(...args); + }, + setTag: (key, value) => { + loaded(); + Clarity.setTag(key, value); + }, + event: (name) => { + loaded(); + Clarity.event(name); + }, + // A partial update names only the categories of the consent change, + // which the SDK's typing does not allow; it forwards the same way. + consentV2: (options) => installStub()('consentv2', options), + upgrade: (reason) => { + loaded(); + Clarity.upgrade(reason); + }, + }; +} + +function isClaritySDK(value: unknown): value is ClaritySDK { + return ( + isObject(value) && + ['init', 'identify', 'setTag', 'event', 'consentV2', 'upgrade'].every( + (key) => typeof value[key] === 'function', + ) + ); +} + /** - * Resolve the Clarity SDK: use the caller-provided mock when available - * (tests), otherwise fall back to the real `@microsoft/clarity` default - * export. Matches the pattern used by @walkeros/server-destination-gcp for - * the BigQuery SDK. + * Resolve the Clarity SDK: the caller-provided mock when available (tests), + * otherwise the real one. */ -function getClarity(env: Env | undefined): ClaritySDK { - return env?.clarity ?? (Clarity as unknown as ClaritySDK); +function getClarity( + env: unknown, + apiKey: string, + timeout?: number, +): ClaritySDK { + const mock = isObject(env) ? env.clarity : undefined; + if (mock === undefined) return realClarity(apiKey, timeout); + if (isClaritySDK(mock)) return mock; + throw new StepError('env.clarity is not a Clarity SDK', { code: 'config' }); +} + +/** Runs one Clarity call; a throw or a rejection becomes a StepError. */ +async function call( + fn: () => unknown, + code: string, + message: string, +): Promise { + try { + await fn(); + } catch (error) { + if (error instanceof StepError) throw error; + throw new StepError(message, { code, cause: error }); + } } +const send = (fn: () => unknown) => + call(fn, 'transport', 'Clarity call failed'); + function coerceTagValue(value: unknown): string | string[] | undefined { if (value == null || value === '') return undefined; if (Array.isArray(value)) return value.map(String); @@ -25,10 +136,15 @@ function coerceTagValue(value: unknown): string | string[] | undefined { return undefined; // skip nested objects } -function emitTag(clarity: ClaritySDK, key: string, raw: unknown): void { - const value = coerceTagValue(raw); - if (value === undefined) return; - clarity.setTag(key, value); +/** The tags of a resolved object, as Clarity.setTag takes them. */ +function tagsOf(resolved: unknown): Array<[string, string | string[]]> { + if (!isObject(resolved)) return []; + const tags: Array<[string, string | string[]]> = []; + for (const [key, raw] of Object.entries(resolved)) { + const value = coerceTagValue(raw); + if (value !== undefined) tags.push([key, value]); + } + return tags; } /** @@ -42,16 +158,52 @@ function callIdentify( customSessionId?: string, customPageId?: string, friendlyName?: string, -): void { - if (friendlyName !== undefined) { - clarity.identify(customId, customSessionId, customPageId, friendlyName); - } else if (customPageId !== undefined) { - clarity.identify(customId, customSessionId, customPageId); - } else if (customSessionId !== undefined) { - clarity.identify(customId, customSessionId); - } else { - clarity.identify(customId); +): unknown { + if (friendlyName !== undefined) + return clarity.identify( + customId, + customSessionId, + customPageId, + friendlyName, + ); + if (customPageId !== undefined) + return clarity.identify(customId, customSessionId, customPageId); + if (customSessionId !== undefined) + return clarity.identify(customId, customSessionId); + return clarity.identify(customId); +} + +function isCategory(value: unknown): value is Category { + return value === 'analytics_Storage' || value === 'ad_Storage'; +} + +/** + * Clarity's ConsentV2 update for one consent change, or none to send. With + * `settings.consent` each mapped key present in the change sets its + * category; without it the destination's required consent decides, as it + * gates the pushes, and both categories follow it. + */ +function consentOf( + settings: unknown, + required: WalkerOS.Consent | undefined, + change: Record, + state: WalkerOS.Consent, +): ConsentV2 | undefined { + const consentMap = isObject(settings) ? settings.consent : undefined; + if (isObject(consentMap)) { + const v2: ConsentV2 = {}; + for (const [walkerKey, category] of Object.entries(consentMap)) { + if (!isCategory(category) || !(walkerKey in change)) continue; + v2[category] = change[walkerKey] === true ? 'granted' : 'denied'; + } + return Object.keys(v2).length ? v2 : undefined; } + + if (!required || !Object.keys(required).some((key) => key in change)) + return undefined; + const granted = + getGrantedConsent(required, state) !== false ? 'granted' : 'denied'; + return { analytics_Storage: granted, ad_Storage: granted }; } export const destinationClarity: Destination = { @@ -59,19 +211,27 @@ export const destinationClarity: Destination = { config: {}, - init({ config, env }) { - const settings = config.settings; - if (!settings?.apiKey) return false; + async init({ config, env }) { + const apiKey = config.settings?.apiKey; + if (!apiKey) + throw new StepError('Clarity needs settings.apiKey', { code: 'config' }); - getClarity(env).init(settings.apiKey); - - return config; + await call( + () => getClarity(env, apiKey, config.timeout).init(apiKey), + 'vendor-not-ready', + 'Clarity init failed', + ); }, async push(event, { config, rule, env, data, collector }) { - const clarity = getClarity(env as Env | undefined); - const settings = (config.settings || {}) as Partial; - const mappingSettings = (rule?.settings || {}) as Mapping; + const settings: Partial = config.settings || {}; + const clarity = getClarity(env, settings.apiKey ?? '', config.timeout); + const mappingSettings: Mapping = rule?.settings || {}; + let calls = 0; + const sent = (fn: () => unknown) => { + calls++; + return send(fn); + }; // 1. Identify - rule-level override wins over destination-level. // Fires first per Clarity's own guidance so subsequent tag/event @@ -83,43 +243,32 @@ export const destinationClarity: Destination = { }); if (isObject(resolved)) { const { customId, customSessionId, customPageId, friendlyName } = - resolved as { - customId?: unknown; - customSessionId?: unknown; - customPageId?: unknown; - friendlyName?: unknown; - }; + resolved; if (isString(customId) && customId) { - callIdentify( - clarity, - customId, - isString(customSessionId) ? customSessionId : undefined, - isString(customPageId) ? customPageId : undefined, - isString(friendlyName) ? friendlyName : undefined, + await sent(() => + callIdentify( + clarity, + customId, + isString(customSessionId) ? customSessionId : undefined, + isString(customPageId) ? customPageId : undefined, + isString(friendlyName) ? friendlyName : undefined, + ), ); } } } // 2. Include tags - pre-flattened by core into context.data - if (isObject(data)) { - for (const [key, raw] of Object.entries( - data as Record, - )) { - emitTag(clarity, key, raw); - } - } + for (const [key, value] of tagsOf(data)) + await sent(() => clarity.setTag(key, value)); // 3. Explicit tags via mapping.settings.set if (mappingSettings.set !== undefined) { const resolved = await getMappingValue(event, mappingSettings.set, { collector, }); - if (isObject(resolved)) { - for (const [key, raw] of Object.entries(resolved)) { - emitTag(clarity, key, raw); - } - } + for (const [key, value] of tagsOf(resolved)) + await sent(() => clarity.setTag(key, value)); } // 4. Session priority upgrade @@ -127,44 +276,47 @@ export const destinationClarity: Destination = { const reason = await getMappingValue(event, mappingSettings.upgrade, { collector, }); - if (isString(reason) && reason) { - clarity.upgrade(reason); - } + if (isString(reason) && reason) await sent(() => clarity.upgrade(reason)); } // 5. Default event forwarding - unless rule.silent is set if (rule?.silent !== true) { const eventName = isString(rule?.name) ? rule.name : event.name; - clarity.event(eventName); + await sent(() => clarity.event(eventName)); } - }, - - on(type, context) { - if (type !== 'consent' || !context.data) return; - const clarity = getClarity(context.env as Env | undefined); - const consent = context.data as WalkerOS.Consent; - const settings = (context.config?.settings || {}) as Partial; - const consentMap = settings.consent; - if (!consentMap) return; // Users MUST configure settings.consent; without it we take no action. - - const v2: { - analytics_Storage?: 'granted' | 'denied'; - ad_Storage?: 'granted' | 'denied'; - } = {}; - - for (const [walkerKey, clarityKey] of Object.entries(consentMap)) { - if (!(walkerKey in consent)) continue; - v2[clarityKey] = consent[walkerKey] === true ? 'granted' : 'denied'; - } + // A silent rule whose own settings resolved nothing sent nothing. + const ruleSettings = + mappingSettings.identify !== undefined || + mappingSettings.set !== undefined || + mappingSettings.upgrade !== undefined; + if (!calls && ruleSettings) + throw new StepError( + `Clarity: the silent rule of "${event.name}" resolved no identify, tag or upgrade`, + { code: 'missing-input' }, + ); + }, - if (Object.keys(v2).length === 0) return; + async on(type, { config, data, env, collector }) { + if (type !== 'consent' || !isObject(data)) return; + const v2 = consentOf( + config.settings, + config.consent, + data, + collector.consent, + ); + if (!v2) return; - clarity.consentV2( - v2 as { - analytics_Storage: 'granted' | 'denied'; - ad_Storage: 'granted' | 'denied'; - }, + const apiKey = isObject(config.settings) ? config.settings.apiKey : ''; + await call( + () => + getClarity( + env, + isString(apiKey) ? apiKey : '', + config.timeout, + ).consentV2(v2), + 'consent-not-applied', + 'Clarity did not take the consent update', ); }, }; diff --git a/packages/web/destinations/clarity/src/schemas/settings.ts b/packages/web/destinations/clarity/src/schemas/settings.ts index 66fcb0f43..422055718 100644 --- a/packages/web/destinations/clarity/src/schemas/settings.ts +++ b/packages/web/destinations/clarity/src/schemas/settings.ts @@ -1,24 +1,26 @@ import { z } from '@walkeros/core/dev'; -export const SettingsSchema = z.object({ - apiKey: z - .string() - .min(1) - .describe( - 'Your Microsoft Clarity project ID (e.g. "3t0wlogvdz"). Find it in your Clarity dashboard under Settings → Setup.', - ), - consent: z - .record(z.string(), z.enum(['analytics_Storage', 'ad_Storage'])) - .describe( - 'Translation table from walkerOS consent keys to Clarity ConsentV2 categories. Example: { "analytics": "analytics_Storage", "marketing": "ad_Storage" }. Required to get meaningful consent behavior, since Clarity expects its own category names.', - ) - .optional(), - identify: z - .unknown() - .describe( - 'walkerOS mapping value resolving to positional arguments for Clarity.identify(). Keys: customId (required), customSessionId?, customPageId?, friendlyName?.', - ) - .optional(), -}); +export const SettingsSchema = z + .object({ + apiKey: z + .string() + .min(1) + .describe( + 'Your Microsoft Clarity project ID (e.g. "3t0wlogvdz"). Find it in your Clarity dashboard under Settings → Setup.', + ), + consent: z + .record(z.string(), z.enum(['analytics_Storage', 'ad_Storage'])) + .describe( + 'Translation table from walkerOS consent keys to Clarity ConsentV2 categories. Example: { "analytics": "analytics_Storage", "marketing": "ad_Storage" }. Required to get meaningful consent behavior, since Clarity expects its own category names.', + ) + .optional(), + identify: z + .unknown() + .describe( + 'walkerOS mapping value resolving to positional arguments for Clarity.identify(). Keys: customId (required), customSessionId?, customPageId?, friendlyName?.', + ) + .optional(), + }) + .strict(); export type Settings = z.infer; diff --git a/packages/web/destinations/clarity/src/types/index.ts b/packages/web/destinations/clarity/src/types/index.ts index 16708c83e..16032e500 100644 --- a/packages/web/destinations/clarity/src/types/index.ts +++ b/packages/web/destinations/clarity/src/types/index.ts @@ -4,6 +4,15 @@ import type { } from '@walkeros/core'; import type { DestinationWeb } from '@walkeros/web-core'; +/** The `window.clarity` queue: the stub until the tag script replaces it. */ +export type ClarityQueue = ((...args: unknown[]) => void) & { q?: unknown[] }; + +declare global { + interface Window { + clarity?: ClarityQueue; + } +} + /** * Settings (destination-level). * @@ -48,13 +57,15 @@ export interface ClaritySDK { ) => void; setTag: (key: string, value: string | string[]) => void; event: (name: string) => void; - consentV2: (consentOptions?: { - ad_Storage: 'granted' | 'denied'; - analytics_Storage: 'granted' | 'denied'; - }) => void; + consentV2: (consentOptions: ConsentV2) => void; upgrade: (reason: string) => void; } +/** A ConsentV2 update: the categories a consent change names. */ +export type ConsentV2 = Partial< + Record<'analytics_Storage' | 'ad_Storage', 'granted' | 'denied'> +>; + /** * Env: optional override for the vendor SDK. Production leaves this * undefined and the destination falls back to the real `@microsoft/clarity` diff --git a/packages/web/destinations/fullstory/src/__tests__/conformance.test.ts b/packages/web/destinations/fullstory/src/__tests__/conformance.test.ts new file mode 100644 index 000000000..3a0d3865e --- /dev/null +++ b/packages/web/destinations/fullstory/src/__tests__/conformance.test.ts @@ -0,0 +1,21 @@ +import path from 'path'; +import { conformance } from '@walkeros/collector/dev'; +import { destinationFullStory } from '..'; +import { examples, schemas } from '../dev'; + +describe('fullstory conformance', () => { + beforeEach(() => { + jest.useRealTimers(); + }); + + it('conforms', async () => { + await conformance('destination', destinationFullStory, { + profile: 'web-sdk', + examples: examples.step, + schema: schemas.settings, + settings: { orgId: 'o-TEST-na1' }, + env: { push: examples.env.push, simulation: examples.env.simulation }, + root: path.join(__dirname, '..', '..'), + }); + }); +}); diff --git a/packages/web/destinations/fullstory/src/__tests__/index.test.ts b/packages/web/destinations/fullstory/src/__tests__/index.test.ts new file mode 100644 index 000000000..200319549 --- /dev/null +++ b/packages/web/destinations/fullstory/src/__tests__/index.test.ts @@ -0,0 +1,48 @@ +import { startFlow } from '@walkeros/collector'; +import { getEvent } from '@walkeros/core'; +import { destinationFullStory } from '..'; +import type { FullStorySDK } from '../types'; + +function recordingFS(calls: unknown[][]): FullStorySDK { + const record = + (name: string) => + (...args: unknown[]) => { + calls.push([name, ...args]); + }; + return { + init: record('init'), + trackEvent: record('trackEvent'), + setIdentity: record('setIdentity'), + setProperties: record('setProperties'), + shutdown: record('shutdown'), + start: record('start'), + }; +} + +describe('fullstory destination', () => { + it('stops capture on a revocation without settings.consent when the destination is consent-gated', async () => { + const calls: unknown[][] = []; + const { elb } = await startFlow({ + destinations: { + fullstory: { + code: { + ...destinationFullStory, + env: { fullstory: recordingFS(calls) }, + }, + config: { consent: { analytics: true }, settings: { orgId: 'o-1' } }, + }, + }, + }); + + await elb('walker consent', { analytics: true }); + await elb(getEvent('page view')); + await elb('walker consent', { analytics: false }); + + expect(calls).toEqual([ + ['init', { orgId: 'o-1' }], + ['start'], + ['trackEvent', { name: 'page view', properties: {} }], + ['shutdown'], + ]); + }); +}); diff --git a/packages/web/destinations/fullstory/src/__tests__/stepExamples.test.ts b/packages/web/destinations/fullstory/src/__tests__/stepExamples.test.ts index 5ec60bc48..00da434f2 100644 --- a/packages/web/destinations/fullstory/src/__tests__/stepExamples.test.ts +++ b/packages/web/destinations/fullstory/src/__tests__/stepExamples.test.ts @@ -49,7 +49,12 @@ function spyEnv(env: Env): { } describe('fullstory destination -- step examples', () => { - it.each<[string, FullStoryStepExample]>(Object.entries(examples.step))( + // Non-public examples (silent, failures) run in the conformance test. + const published = Object.entries(examples.step).filter( + ([, example]) => example.public !== false, + ); + + it.each<[string, FullStoryStepExample]>(published)( '%s', async (name, example) => { const env = clone(examples.env.push) as Env; diff --git a/packages/web/destinations/fullstory/src/examples/step.ts b/packages/web/destinations/fullstory/src/examples/step.ts index b61847383..022d0dd0f 100644 --- a/packages/web/destinations/fullstory/src/examples/step.ts +++ b/packages/web/destinations/fullstory/src/examples/step.ts @@ -204,6 +204,7 @@ export const combinedFeatures: FullStoryStepExample = { */ export const silentWithIdentify: FullStoryStepExample = { public: false, + description: 'A silent rule still runs its identify.', in: createEvent({ name: 'user login', timestamp: 1700000108, @@ -315,3 +316,62 @@ export const consentRevokeFlag: FullStoryStepExample = { ['fullstory.setIdentity', { consent: false }], ], }; + +/** A silent rule without settings of its own calls nothing. */ +export const silentRule: FullStoryStepExample = { + public: false, + description: 'A rule with silent: true and no settings calls nothing.', + in: getEvent('page view', { timestamp: 1700000120 }), + mapping: { silent: true }, + out: [], +}; + +/** The FullStory SDK is unreachable: its init fails, so the push reports it. */ +export const fullstoryUnreachable: FullStoryStepExample = { + public: false, + description: + 'The FullStory SDK fails, so the destination does not start: the push fails with vendor-not-ready, retryable, and the event is in the DLQ.', + in: getEvent('product view', { timestamp: 1700000121 }), + respond: { error: 'network' }, + out: [], + outcome: { status: 'failed', code: 'vendor-not-ready', retryable: true }, +}; + +/** + * The silent page rule of setPageProperties on a page view without data: no + * property resolves and the rule suppresses the event, so nothing is sent. + */ +export const setPagePropertiesWithoutData: FullStoryStepExample = { + public: false, + description: + 'A silent rule whose property mapping finds no data sends nothing, so the push fails with missing-input instead of counting as delivered.', + in: { name: 'page view', entity: 'page', action: 'view' }, + mapping: { + silent: true, + settings: { + set: { map: { pageName: 'data.title' } }, + setType: 'page', + }, + }, + out: [], + outcome: { status: 'failed', code: 'missing-input' }, +}; + +/** + * The silent identify rule of silentWithIdentify on a login without data: no + * uid resolves and the rule suppresses the event, so nothing is sent. + */ +export const silentIdentifyWithoutData: FullStoryStepExample = { + public: false, + description: + 'A silent rule whose identify mapping finds no uid sends nothing, so the push fails with missing-input instead of counting as delivered.', + in: { name: 'user login', entity: 'user', action: 'login' }, + mapping: { + silent: true, + settings: { + identify: { map: { uid: 'data.id' } }, + }, + }, + out: [], + outcome: { status: 'failed', code: 'missing-input' }, +}; diff --git a/packages/web/destinations/fullstory/src/index.ts b/packages/web/destinations/fullstory/src/index.ts index d27e3220c..f716ee1b3 100644 --- a/packages/web/destinations/fullstory/src/index.ts +++ b/packages/web/destinations/fullstory/src/index.ts @@ -1,35 +1,124 @@ import type { WalkerOS } from '@walkeros/core'; -import { getMappingValue, isObject, isString } from '@walkeros/core'; +import { + getGrantedConsent, + getMappingValue, + isObject, + isString, + StepError, +} from '@walkeros/core'; import { init as fsInit, FullStory } from '@fullstory/browser'; -import type { - Destination, - Env, - FullStorySDK, - Mapping, - Settings, -} from './types'; +import type { Destination, FullStorySDK, Mapping, Settings } from './types'; // Types export export * as DestinationFullStory from './types'; +type InitOptions = Parameters[0]; + +/** The real @fullstory/browser functions. */ +const realFS: FullStorySDK = { + init: fsInit, + trackEvent: (opts) => + FullStory('trackEvent', { + name: opts.name, + properties: opts.properties ?? {}, + }), + setIdentity: (opts) => FullStory('setIdentity', opts), + setProperties: (opts) => FullStory('setProperties', opts), + shutdown: () => FullStory('shutdown'), + start: () => FullStory('start'), +}; + +function isFullStorySDK(value: unknown): value is FullStorySDK { + return ( + isObject(value) && + [ + 'init', + 'trackEvent', + 'setIdentity', + 'setProperties', + 'shutdown', + 'start', + ].every((key) => typeof value[key] === 'function') + ); +} + /** * Resolve the FullStory SDK: use the caller-provided mock when available * (tests), otherwise fall back to the real @fullstory/browser functions. */ -function getFS(env: Env | undefined): FullStorySDK { - if (env?.fullstory) return env.fullstory; - return { - init: fsInit, - trackEvent: (opts) => - FullStory('trackEvent', { - name: opts.name, - properties: opts.properties ?? {}, - }), - setIdentity: (opts) => FullStory('setIdentity', opts), - setProperties: (opts) => FullStory('setProperties', opts), - shutdown: () => FullStory('shutdown'), - start: () => FullStory('start'), - }; +function getFS(env: unknown): FullStorySDK { + const mock = isObject(env) ? env.fullstory : undefined; + if (mock === undefined) return realFS; + if (isFullStorySDK(mock)) return mock; + throw new StepError('env.fullstory is not a FullStory SDK', { + code: 'config', + }); +} + +/** The SDK's init options from the settings. */ +function initOptions(settings: unknown): InitOptions | undefined { + if (!isObject(settings) || !isString(settings.orgId) || !settings.orgId) + return undefined; + const options: InitOptions = { orgId: settings.orgId }; + if (isString(settings.host)) options.host = settings.host; + if (isString(settings.script)) options.script = settings.script; + if (isString(settings.cookieDomain)) + options.cookieDomain = settings.cookieDomain; + if (typeof settings.debug === 'boolean') options.debug = settings.debug; + if (typeof settings.devMode === 'boolean') options.devMode = settings.devMode; + if (typeof settings.startCaptureManually === 'boolean') + options.startCaptureManually = settings.startCaptureManually; + if (isString(settings.namespace)) options.namespace = settings.namespace; + if (typeof settings.recordCrossDomainIFrames === 'boolean') + options.recordCrossDomainIFrames = settings.recordCrossDomainIFrames; + return options; +} + +/** Runs one FullStory call; a throw or a rejection becomes a StepError. */ +async function call( + fn: () => unknown, + code: string, + message: string, +): Promise { + try { + await fn(); + } catch (error) { + if (error instanceof StepError) throw error; + throw new StepError(message, { code, cause: error }); + } +} + +const send = (fn: () => unknown) => + call(fn, 'transport', 'FullStory call failed'); + +/** The calls a consent change makes, or none. */ +function consentCalls( + fs: FullStorySDK, + settings: unknown, + required: WalkerOS.Consent | undefined, + change: Record, + state: WalkerOS.Consent, +): Array<() => unknown> { + const consentMap = isObject(settings) ? settings.consent : undefined; + if (isObject(consentMap)) { + // The explicit translation: each mapped key present in the change. + return Object.entries(consentMap).flatMap(([walkerKey, fsAction]) => { + if (!(walkerKey in change)) return []; + const granted = change[walkerKey] === true; + if (fsAction === 'capture') + return [() => (granted ? fs.start() : fs.shutdown())]; + if (fsAction === 'consent') + return [() => fs.setIdentity({ consent: granted })]; + return []; + }); + } + + // Without a translation the destination's required consent decides, as it + // gates the pushes: capture starts on a grant and stops on a revocation. + if (!required || !Object.keys(required).some((key) => key in change)) + return []; + const granted = getGrantedConsent(required, state) !== false; + return [() => (granted ? fs.start() : fs.shutdown())]; } export const destinationFullStory: Destination = { @@ -37,35 +126,29 @@ export const destinationFullStory: Destination = { config: {}, - init({ config, env }) { - const settings = config.settings; - if (!settings?.orgId) return false; - - const fs = getFS(env as Env | undefined); - const initOpts: Parameters[0] = { - orgId: settings.orgId, - }; - if (settings.host !== undefined) initOpts.host = settings.host; - if (settings.script !== undefined) initOpts.script = settings.script; - if (settings.cookieDomain !== undefined) - initOpts.cookieDomain = settings.cookieDomain; - if (settings.debug !== undefined) initOpts.debug = settings.debug; - if (settings.devMode !== undefined) initOpts.devMode = settings.devMode; - if (settings.startCaptureManually !== undefined) - initOpts.startCaptureManually = settings.startCaptureManually; - if (settings.namespace !== undefined) - initOpts.namespace = settings.namespace; - if (settings.recordCrossDomainIFrames !== undefined) - initOpts.recordCrossDomainIFrames = settings.recordCrossDomainIFrames; - fs.init(initOpts); - - return config; + // FullStory starts in init, so recording begins at page load. + async init({ config, env }) { + const options = initOptions(config.settings); + if (!options) + throw new StepError('FullStory needs settings.orgId', { + code: 'config', + }); + await call( + () => getFS(env).init(options), + 'vendor-not-ready', + 'FullStory init failed', + ); }, async push(event, { config, rule, env, data, collector }) { - const fs = getFS(env as Env | undefined); - const settings = (config.settings || {}) as Partial; - const mappingSettings = (rule?.settings || {}) as Mapping; + const fs = getFS(env); + const settings: Partial = config.settings || {}; + const mappingSettings: Mapping = rule?.settings || {}; + let calls = 0; + const sent = (fn: () => unknown) => { + calls++; + return send(fn); + }; // 1. Identify -- rule-level override wins over destination-level. const identifyMapping = mappingSettings.identify ?? settings.identify; @@ -74,19 +157,14 @@ export const destinationFullStory: Destination = { collector, }); if (isObject(resolved)) { - const { uid, properties } = resolved as { - uid?: unknown; - properties?: unknown; - }; + const { uid, properties } = resolved; if (isString(uid) && uid) { const identityOpts: { uid: string; properties?: Record; } = { uid }; - if (isObject(properties)) { - identityOpts.properties = properties as Record; - } - fs.setIdentity(identityOpts); + if (isObject(properties)) identityOpts.properties = properties; + await sent(() => fs.setIdentity(identityOpts)); } } } @@ -96,48 +174,48 @@ export const destinationFullStory: Destination = { const resolved = await getMappingValue(event, mappingSettings.set, { collector, }); - if (isObject(resolved)) { + if (isObject(resolved) && Object.keys(resolved).length) { const propertyType = mappingSettings.setType || 'user'; - fs.setProperties({ - type: propertyType, - properties: resolved as Record, - }); + await sent(() => + fs.setProperties({ type: propertyType, properties: resolved }), + ); } } // 3. Default trackEvent -- unless rule.silent is set if (rule?.silent !== true) { const eventName = isString(rule?.name) ? rule.name : event.name; - const properties = isObject(data) - ? (data as Record) - : {}; - fs.trackEvent({ name: eventName, properties }); + const properties = isObject(data) ? data : {}; + await sent(() => fs.trackEvent({ name: eventName, properties })); } - }, - - on(type, context) { - if (type !== 'consent' || !context.data) return; - const fs = getFS(context.env as Env | undefined); - const consent = context.data as WalkerOS.Consent; - const settings = (context.config?.settings || {}) as Partial; - const consentMap = settings.consent; - if (!consentMap) return; - - for (const [walkerKey, fsAction] of Object.entries(consentMap)) { - if (!(walkerKey in consent)) continue; - const granted = consent[walkerKey] === true; + // A silent rule whose own settings resolved nothing sent nothing. + const ruleSettings = + mappingSettings.identify !== undefined || + mappingSettings.set !== undefined; + if (!calls && ruleSettings) + throw new StepError( + `FullStory: the silent rule of "${event.name}" resolved no identity or properties`, + { code: 'missing-input' }, + ); + }, - if (fsAction === 'capture') { - if (granted) { - fs.start(); - } else { - fs.shutdown(); - } - } else if (fsAction === 'consent') { - fs.setIdentity({ consent: granted }); - } - } + async on(type, { config, data, env, collector }) { + if (type !== 'consent' || !isObject(data)) return; + const calls = consentCalls( + getFS(env), + config.settings, + config.consent, + data, + collector.consent, + ); + + for (const fn of calls) + await call( + fn, + 'consent-not-applied', + 'FullStory did not take the consent update', + ); }, }; diff --git a/packages/web/destinations/fullstory/src/schemas/settings.ts b/packages/web/destinations/fullstory/src/schemas/settings.ts index 30106370d..72e4334b5 100644 --- a/packages/web/destinations/fullstory/src/schemas/settings.ts +++ b/packages/web/destinations/fullstory/src/schemas/settings.ts @@ -1,55 +1,57 @@ import { z } from '@walkeros/core/dev'; -export const SettingsSchema = z.object({ - orgId: z - .string() - .min(1) - .describe( - 'Your FullStory organization ID (e.g. "o-XXXXXX-na1"). Find it in FullStory under Settings > General.', - ), - host: z - .string() - .describe('Recording server host for proxy support.') - .optional(), - script: z.string().describe('Custom script CDN host domain.').optional(), - cookieDomain: z - .string() - .describe('Override cookie domain for cross-subdomain tracking.') - .optional(), - debug: z - .boolean() - .describe('Enable browser console debug logging.') - .optional(), - devMode: z - .boolean() - .describe('Disable recording entirely for development environments.') - .optional(), - startCaptureManually: z - .boolean() - .describe( - 'Delay capture until FullStory("start") is called. The SDK script still loads at init. With settings.consent mapping a key to "capture", recording starts when that consent is granted.', - ) - .optional(), - namespace: z - .string() - .describe('Global FS identifier override (default: "FS").') - .optional(), - recordCrossDomainIFrames: z - .boolean() - .describe('Enable cross-domain iframe recording.') - .optional(), - identify: z - .unknown() - .describe( - 'Destination-level identity mapping. Resolves to { uid, properties? } for FullStory setIdentity. Fires on every push.', - ) - .optional(), - consent: z - .record(z.string(), z.enum(['capture', 'consent'])) - .describe( - 'Translation table from walkerOS consent keys to FullStory consent actions. "capture" controls start/shutdown (recording on/off). "consent" controls setIdentity({ consent }) flag. Example: { "analytics": "capture" }.', - ) - .optional(), -}); +export const SettingsSchema = z + .object({ + orgId: z + .string() + .min(1) + .describe( + 'Your FullStory organization ID (e.g. "o-XXXXXX-na1"). Find it in FullStory under Settings > General.', + ), + host: z + .string() + .describe('Recording server host for proxy support.') + .optional(), + script: z.string().describe('Custom script CDN host domain.').optional(), + cookieDomain: z + .string() + .describe('Override cookie domain for cross-subdomain tracking.') + .optional(), + debug: z + .boolean() + .describe('Enable browser console debug logging.') + .optional(), + devMode: z + .boolean() + .describe('Disable recording entirely for development environments.') + .optional(), + startCaptureManually: z + .boolean() + .describe( + 'Delay capture until FullStory("start") is called. The SDK script still loads at init. With settings.consent mapping a key to "capture", recording starts when that consent is granted.', + ) + .optional(), + namespace: z + .string() + .describe('Global FS identifier override (default: "FS").') + .optional(), + recordCrossDomainIFrames: z + .boolean() + .describe('Enable cross-domain iframe recording.') + .optional(), + identify: z + .unknown() + .describe( + 'Destination-level identity mapping. Resolves to { uid, properties? } for FullStory setIdentity. Fires on every push.', + ) + .optional(), + consent: z + .record(z.string(), z.enum(['capture', 'consent'])) + .describe( + 'Translation table from walkerOS consent keys to FullStory consent actions. "capture" controls start/shutdown (recording on/off). "consent" controls setIdentity({ consent }) flag. Example: { "analytics": "capture" }.', + ) + .optional(), + }) + .strict(); export type Settings = z.infer; diff --git a/packages/web/destinations/gtag/src/__tests__/conformance.test.ts b/packages/web/destinations/gtag/src/__tests__/conformance.test.ts new file mode 100644 index 000000000..d3a8af611 --- /dev/null +++ b/packages/web/destinations/gtag/src/__tests__/conformance.test.ts @@ -0,0 +1,21 @@ +import path from 'path'; +import { conformance } from '@walkeros/collector/dev'; +import { destinationGtag } from '..'; +import { examples, schemas } from '../dev'; + +describe('gtag conformance', () => { + beforeEach(() => { + jest.useRealTimers(); + }); + + it('conforms', async () => { + await conformance('destination', destinationGtag, { + profile: 'vendor-global', + examples: examples.step, + schema: schemas.settings, + settings: { ga4: { measurementId: 'G-XXXXXX-1' } }, + env: { push: examples.env.push, simulation: examples.env.simulation }, + root: path.join(__dirname, '..', '..'), + }); + }); +}); diff --git a/packages/web/destinations/gtag/src/__tests__/droppedParams.test.ts b/packages/web/destinations/gtag/src/__tests__/droppedParams.test.ts new file mode 100644 index 000000000..47ead41fc --- /dev/null +++ b/packages/web/destinations/gtag/src/__tests__/droppedParams.test.ts @@ -0,0 +1,44 @@ +import type { Logger } from '@walkeros/core'; +import { startFlow } from '@walkeros/collector'; +import { clone, getEvent, Level } from '@walkeros/core'; +import { destinationGtag } from '..'; +import { examples } from '../dev'; + +describe('GA4 parameter names the mapping cannot carry', () => { + it('sends the event without the parameter and writes one ERROR line per cause', async () => { + const levels: Logger.Level[] = []; + const gtag = jest.fn(); + const env = clone(examples.env.push); + env.window.gtag = gtag; + const { elb } = await startFlow({ + // The default level: only ERROR lines reach the handler. + logger: { handler: (level) => levels.push(level) }, + destinations: { + gtag: { + code: { ...destinationGtag, env }, + config: { + settings: { ga4: { measurementId: 'G-DROPPED' } }, + mapping: { + product: { + view: { + data: { map: { 'a-b': 'data.id', 'a.b': 'data.name' } }, + }, + }, + }, + }, + }, + }, + }); + + const first = await elb(getEvent('product view')); + const second = await elb(getEvent('product view')); + + expect(first.ok).toBe(true); + expect(second.ok).toBe(true); + const sent = gtag.mock.calls.filter(([command]) => command === 'event'); + expect(sent).toHaveLength(2); + expect(sent[0][2]).toEqual({ a_b: 'ers', send_to: 'G-DROPPED' }); + // The second occurrence within the window is counted, not logged. + expect(levels.filter((level) => level === Level.ERROR)).toHaveLength(1); + }); +}); diff --git a/packages/web/destinations/gtag/src/__tests__/ga4.test.ts b/packages/web/destinations/gtag/src/__tests__/ga4.test.ts index aa7089ca4..785bdf2df 100644 --- a/packages/web/destinations/gtag/src/__tests__/ga4.test.ts +++ b/packages/web/destinations/gtag/src/__tests__/ga4.test.ts @@ -1,11 +1,11 @@ import { initGA4, pushGA4Event } from '../ga4'; -import { resetLoadedScripts } from '../shared/gtag'; import { examples } from '../dev'; import { clone, createMockLogger, getEvent } from '@walkeros/core'; import type { GA4Settings } from '../types'; describe('GA4 Implementation', () => { const mockGtag = jest.fn(); + const dropped = jest.fn(); const mockEnv = clone(examples.env.push); mockEnv.window.gtag = mockGtag; @@ -97,34 +97,38 @@ describe('GA4 Implementation', () => { }); describe('gtag.js loading', () => { - // Build an env whose document.createElement captures the injected script - // so the resolved `script.src` can be asserted. + beforeEach(() => { + // The fallback follows the script's error event on real timers. + jest.useRealTimers(); + }); + + // The scripts loadVendor added in this test (once per src and page). const setupCapture = () => { - resetLoadedScripts(); - type FakeScript = { - src: string; - onerror?: () => void; - setAttribute: (name: string, value: string) => void; - removeAttribute: (name: string) => void; - }; - const created: FakeScript[] = []; + const scripts = () => + Array.from( + document.querySelectorAll('script[data-walkeros-vendor]'), + (script) => script.getAttribute('src') ?? '', + ); + const before = new Set(scripts()); + const added = () => scripts().filter((src) => !before.has(src)); const env = clone(examples.env.push); env.window.gtag = mockGtag; - env.document.createElement = () => { - const script: FakeScript = { - src: '', - setAttribute: () => {}, - removeAttribute: () => {}, - }; - created.push(script); - return script; + const getSrc = (): string | undefined => { + const list = added(); + return list[list.length - 1]; }; return { env, - getSrc: () => created[created.length - 1]?.src, - countCreated: () => created.length, - // Simulate the browser failing the most recently injected script. - failScript: () => created[created.length - 1]?.onerror?.(), + getSrc, + countCreated: () => added().length, + // Simulate the browser failing the most recently injected script, + // then let the fallback follow. + failScript: async () => { + document + .querySelector(`script[src="${getSrc()}"]`) + ?.dispatchEvent(new Event('error')); + await new Promise((resolve) => setTimeout(resolve, 0)); + }, }; }; @@ -219,33 +223,33 @@ describe('GA4 Implementation', () => { expect(getSrc()).toBe('https://example.com/metrics/tag_serving_path/'); }); - it('falls back to googletagmanager.com when scriptSrc fails to load', () => { + it('falls back to googletagmanager.com when scriptSrc fails to load', async () => { const { env, getSrc, failScript } = setupCapture(); const settings: GA4Settings = { - measurementId: 'G-FIRSTPARTY', + measurementId: 'G-FALLBACK', scriptSrc: 'https://example.com/metrics/wrong_path/', }; initGA4(settings, true, env, createMockLogger()); expect(getSrc()).toBe('https://example.com/metrics/wrong_path/'); - failScript(); + await failScript(); expect(getSrc()).toBe( - 'https://www.googletagmanager.com/gtag/js?id=G-FIRSTPARTY', + 'https://www.googletagmanager.com/gtag/js?id=G-FALLBACK', ); }); - it('does not fall back twice when the fallback itself fails', () => { + it('does not fall back twice when the fallback itself fails', async () => { const { env, failScript, countCreated } = setupCapture(); const settings: GA4Settings = { - measurementId: 'G-FIRSTPARTY', - scriptSrc: 'https://example.com/metrics/wrong_path/', + measurementId: 'G-TWICE', + scriptSrc: 'https://example.com/metrics/other_wrong_path/', }; initGA4(settings, true, env, createMockLogger()); - failScript(); // first-party fails -> fallback injected - failScript(); // fallback fails -> must not loop + await failScript(); // first-party fails -> fallback injected + await failScript(); // fallback fails -> must not loop expect(countCreated()).toBe(2); }); @@ -263,7 +267,7 @@ describe('GA4 Implementation', () => { const logger = createThrowingLogger(); expect(() => - pushGA4Event(mockEvent, settings, {}, mockEnv, logger), + pushGA4Event(mockEvent, settings, {}, mockEnv, logger, dropped), ).toThrow('Config settings ga4.measurementId missing'); }); @@ -276,6 +280,7 @@ describe('GA4 Implementation', () => { { value: 123.45 }, mockEnv, createMockLogger(), + dropped, ); expect(mockGtag).toHaveBeenCalledWith( @@ -297,6 +302,7 @@ describe('GA4 Implementation', () => { { price: 99.99, currency: 'USD' }, mockEnv, createMockLogger(), + dropped, ); expect(mockGtag).toHaveBeenCalledWith( @@ -316,7 +322,14 @@ describe('GA4 Implementation', () => { debug: true, }; - pushGA4Event(mockEvent, settings, {}, mockEnv, createMockLogger()); + pushGA4Event( + mockEvent, + settings, + {}, + mockEnv, + createMockLogger(), + dropped, + ); expect(mockGtag).toHaveBeenCalledWith( 'event', @@ -334,7 +347,14 @@ describe('GA4 Implementation', () => { snakeCase: false, }; - pushGA4Event(mockEvent, settings, {}, mockEnv, createMockLogger()); + pushGA4Event( + mockEvent, + settings, + {}, + mockEnv, + createMockLogger(), + dropped, + ); expect(mockGtag).toHaveBeenCalledWith( 'event', @@ -358,6 +378,7 @@ describe('GA4 Implementation', () => { { 'data_creative-type': 'Sale' }, mockEnv, createMockLogger(), + dropped, ); expect(paramsOf()).toMatchObject({ data_creative_type: 'Sale' }); @@ -374,6 +395,7 @@ describe('GA4 Implementation', () => { { source_platform: 'web', data_id: 'x' }, mockEnv, createMockLogger(), + dropped, ); expect(paramsOf()).not.toHaveProperty('source_platform'); @@ -389,6 +411,7 @@ describe('GA4 Implementation', () => { { user_id: 'us3r' }, mockEnv, createMockLogger(), + dropped, ); expect(paramsOf()).not.toHaveProperty('user_id'); @@ -401,6 +424,7 @@ describe('GA4 Implementation', () => { { 'source-platform': 'web' }, mockEnv, createMockLogger(), + dropped, ); expect(paramsOf()).not.toHaveProperty('source_platform'); @@ -416,6 +440,7 @@ describe('GA4 Implementation', () => { { currency: 'EUR', value: 12.5, transaction_id: 'T1' }, mockEnv, createMockLogger(), + dropped, ); expect(paramsOf()).toMatchObject({ @@ -425,16 +450,24 @@ describe('GA4 Implementation', () => { }); }); - it('should drop an empty param name, and warn', () => { + it('should drop an empty param name, report it and send the event', () => { const logger = createMockLogger(); - pushGA4Event(mockEvent, settings, { '': 'x' }, mockEnv, logger); + pushGA4Event( + mockEvent, + settings, + { '': 'x', a: 1 }, + mockEnv, + logger, + dropped, + ); expect(paramsOf()).not.toHaveProperty(''); - expect(logger.warn).toHaveBeenCalled(); + expect(paramsOf()).toHaveProperty('a', 1); + expect(dropped).toHaveBeenCalledWith(expect.any(String), ''); }); - it('should warn when two param names normalize to one', () => { + it('should keep the first of two names that normalize to one and report the second', () => { const logger = createMockLogger(); pushGA4Event( @@ -443,17 +476,28 @@ describe('GA4 Implementation', () => { { 'a-b': 1, 'a.b': 2 }, mockEnv, logger, + dropped, ); - expect(logger.warn).toHaveBeenCalled(); + expect(paramsOf()).toHaveProperty('a_b', 1); + expect(dropped).toHaveBeenCalledTimes(1); + expect(dropped).toHaveBeenCalledWith(expect.any(String), 'a.b'); }); it('should not warn when every param name is already valid', () => { const logger = createMockLogger(); - pushGA4Event(mockEvent, settings, { currency: 'EUR' }, mockEnv, logger); + pushGA4Event( + mockEvent, + settings, + { currency: 'EUR' }, + mockEnv, + logger, + dropped, + ); expect(logger.warn).not.toHaveBeenCalled(); + expect(dropped).not.toHaveBeenCalled(); }); }); }); diff --git a/packages/web/destinations/gtag/src/__tests__/gtm.test.ts b/packages/web/destinations/gtag/src/__tests__/gtm.test.ts index a8e4f4469..6fcabeb72 100644 --- a/packages/web/destinations/gtag/src/__tests__/gtm.test.ts +++ b/packages/web/destinations/gtag/src/__tests__/gtm.test.ts @@ -86,33 +86,17 @@ describe('GTM Implementation', () => { }); it('should push the init event and load the container when loadScript is true', () => { - const script = { - src: '', - setAttribute: () => {}, - removeAttribute: () => {}, - }; - const appended: unknown[] = []; - const env: Env = { - window: { dataLayer: mockDataLayer }, - document: { - createElement: () => script, - head: { - appendChild: (node) => { - appended.push(node); - }, - }, - }, - }; + const env: Env = { window: { dataLayer: mockDataLayer } }; - initGTM({ containerId: 'GTM-XXXXXXX' }, true, env); + initGTM({ containerId: 'GTM-LOADSCRIPT' }, true, env); expect(mockDataLayer).toEqual([ { 'gtm.start': expect.any(Number), event: 'gtm.js' }, ]); - expect(appended).toHaveLength(1); - expect(script.src).toBe( - 'https://www.googletagmanager.com/gtm.js?id=GTM-XXXXXXX', + const script = document.querySelector( + 'script[src="https://www.googletagmanager.com/gtm.js?id=GTM-LOADSCRIPT"]', ); + expect(script?.hasAttribute('data-walkeros-vendor')).toBe(true); }); }); @@ -160,7 +144,6 @@ describe('GTM Implementation', () => { window: { dataLayer: customDataLayer, }, - document: mockEnv.document, }; pushGTMEvent(mockEvent, settings, {}, {}, customEnv); diff --git a/packages/web/destinations/gtag/src/__tests__/index.test.ts b/packages/web/destinations/gtag/src/__tests__/index.test.ts index 35faeca18..72af4bad5 100644 --- a/packages/web/destinations/gtag/src/__tests__/index.test.ts +++ b/packages/web/destinations/gtag/src/__tests__/index.test.ts @@ -11,19 +11,20 @@ import { } from '@walkeros/core'; // Mock all tool implementations +// The init helpers and pushAdsEvent answer the results of their calls. jest.mock('../ga4', () => ({ - initGA4: jest.fn(), + initGA4: jest.fn(() => []), pushGA4Event: jest.fn(), })); jest.mock('../ads', () => ({ - initAds: jest.fn(), - pushAdsEvent: jest.fn(), + initAds: jest.fn(() => []), + pushAdsEvent: jest.fn(() => []), resolveUserData: jest.fn(), })); jest.mock('../gtm', () => ({ - initGTM: jest.fn(), + initGTM: jest.fn(() => []), pushGTMEvent: jest.fn(), })); @@ -70,10 +71,10 @@ describe('Unified Gtag Destination', () => { }); describe('init', () => { - it('should throw when no tools are configured', () => { + it('should throw when no tools are configured', async () => { const config = { settings: {} }; - expect(() => + await expect( destinationGtag.init!({ id: contextId, config, @@ -81,16 +82,20 @@ describe('Unified Gtag Destination', () => { collector: mockCollector, logger: mockLogger, }), - ).toThrow('Config settings missing'); + ).rejects.toMatchObject({ + code: 'config', + message: + 'Config settings ga4.measurementId, ads.conversionId or gtm.containerId missing', + }); }); - it('should initialize GA4 only', () => { + it('should initialize GA4 only', async () => { const settings: Settings = { ga4: { measurementId: 'G-XXXXXXXXXX' }, }; const config = { settings, loadScript: true }; - const result = destinationGtag.init!({ + const result = await destinationGtag.init!({ id: contextId, config, env: mockEnv, @@ -105,18 +110,19 @@ describe('Unified Gtag Destination', () => { mockEnv, mockLogger, true, + undefined, ); expect(initAds).not.toHaveBeenCalled(); expect(initGTM).not.toHaveBeenCalled(); }); - it('should initialize Google Ads only', () => { + it('should initialize Google Ads only', async () => { const settings: Settings = { ads: { conversionId: 'AW-XXXXXXXXX' }, }; const config = { settings, loadScript: true }; - const result = destinationGtag.init!({ + const result = await destinationGtag.init!({ id: contextId, config, env: mockEnv, @@ -131,18 +137,19 @@ describe('Unified Gtag Destination', () => { mockEnv, mockLogger, true, + undefined, ); expect(initGA4).not.toHaveBeenCalled(); expect(initGTM).not.toHaveBeenCalled(); }); - it('should initialize GTM only', () => { + it('should initialize GTM only', async () => { const settings: Settings = { gtm: { containerId: 'GTM-XXXXXXX' }, }; const config = { settings, loadScript: true }; - const result = destinationGtag.init!({ + const result = await destinationGtag.init!({ id: contextId, config, env: mockEnv, @@ -156,12 +163,13 @@ describe('Unified Gtag Destination', () => { true, mockEnv, mockLogger, + undefined, ); expect(initGA4).not.toHaveBeenCalled(); expect(initAds).not.toHaveBeenCalled(); }); - it('should initialize all tools when configured', () => { + it('should initialize all tools when configured', async () => { const settings: Settings = { ga4: { measurementId: 'G-XXXXXXXXXX' }, ads: { conversionId: 'AW-XXXXXXXXX' }, @@ -169,7 +177,7 @@ describe('Unified Gtag Destination', () => { }; const config = { settings, loadScript: true }; - const result = destinationGtag.init!({ + const result = await destinationGtag.init!({ id: contextId, config, env: mockEnv, @@ -184,6 +192,7 @@ describe('Unified Gtag Destination', () => { mockEnv, mockLogger, true, + undefined, ); expect(initAds).toHaveBeenCalledWith( settings.ads, @@ -191,22 +200,24 @@ describe('Unified Gtag Destination', () => { mockEnv, mockLogger, true, + undefined, ); expect(initGTM).toHaveBeenCalledWith( settings.gtm, true, mockEnv, mockLogger, + undefined, ); }); - it('should pass loadScript parameter correctly', () => { + it('should pass loadScript parameter correctly', async () => { const settings: Settings = { ga4: { measurementId: 'G-XXXXXXXXXX' }, }; const config = { settings, loadScript: false }; - destinationGtag.init!({ + await destinationGtag.init!({ id: contextId, config, env: mockEnv, @@ -220,6 +231,7 @@ describe('Unified Gtag Destination', () => { mockEnv, mockLogger, true, + undefined, ); }); }); @@ -256,6 +268,7 @@ describe('Unified Gtag Destination', () => { mockData, mockEnv, mockLogger, + expect.any(Function), ); }); @@ -512,6 +525,7 @@ describe('Unified Gtag Destination', () => { mockData, mockEnv, mockLogger, + expect.any(Function), ); expect(pushAdsEvent).toHaveBeenCalledWith( mockEvent, @@ -578,6 +592,7 @@ describe('Unified Gtag Destination', () => { mockData, mockEnv, mockLogger, + expect.any(Function), ); }); }); @@ -952,20 +967,23 @@ describe('Unified Gtag Destination', () => { orderEnv.window.gtag = gtag; }); - it('emits consent default (denied) before tool init when config.consent is set', () => { + it('emits consent default (denied) before tool init when config.consent is set', async () => { const calls: string[] = []; gtag.mockImplementation((...args: unknown[]) => { if (args[0] === 'consent' && args[1] === 'default') calls.push('default'); }); - (initGA4 as jest.Mock).mockImplementation(() => calls.push('initGA4')); + (initGA4 as jest.Mock).mockImplementation(() => { + calls.push('initGA4'); + return []; + }); const config = { consent: { marketing: true }, settings: { ga4: { measurementId: 'G-X' } }, }; - destinationGtag.init!({ + await destinationGtag.init!({ id: contextId, config, env: orderEnv, @@ -982,10 +1000,10 @@ describe('Unified Gtag Destination', () => { }); }); - it('does NOT emit a default when there is no consent signal', () => { + it('does NOT emit a default when there is no consent signal', async () => { const config = { settings: { ga4: { measurementId: 'G-X' } } }; - destinationGtag.init!({ + await destinationGtag.init!({ id: contextId, config, env: orderEnv, @@ -1000,12 +1018,12 @@ describe('Unified Gtag Destination', () => { ); }); - it('adds wait_for_update in advanced mode', () => { + it('adds wait_for_update in advanced mode', async () => { const config = { settings: { como_advanced: true, ga4: { measurementId: 'G-X' } }, }; - destinationGtag.init!({ + await destinationGtag.init!({ id: contextId, config, env: orderEnv, @@ -1020,12 +1038,12 @@ describe('Unified Gtag Destination', () => { ); }); - it('does NOT emit a default when como_advanced is false and no config.consent', () => { + it('does NOT emit a default when como_advanced is false and no config.consent', async () => { const config = { settings: { como_advanced: false, ga4: { measurementId: 'G-X' } }, }; - destinationGtag.init!({ + await destinationGtag.init!({ id: contextId, config, env: orderEnv, @@ -1040,7 +1058,7 @@ describe('Unified Gtag Destination', () => { ); }); - it('emits the custom como mapping as the denied default', () => { + it('emits the custom como mapping as the denied default', async () => { const config = { consent: { marketing: true }, settings: { @@ -1049,7 +1067,7 @@ describe('Unified Gtag Destination', () => { }, }; - destinationGtag.init!({ + await destinationGtag.init!({ id: contextId, config, env: orderEnv, diff --git a/packages/web/destinations/gtag/src/__tests__/reservedParams.test.ts b/packages/web/destinations/gtag/src/__tests__/reservedParams.test.ts index d166311db..c8349a880 100644 --- a/packages/web/destinations/gtag/src/__tests__/reservedParams.test.ts +++ b/packages/web/destinations/gtag/src/__tests__/reservedParams.test.ts @@ -74,6 +74,7 @@ const _sourceDriftGuard: [UnstrippedSourceCollision] extends [never] : false = true; describe('GA4 reserved parameter guard', () => { + const dropped = jest.fn(); const mockGtag = jest.fn(); const mockEnv = clone(examples.env.push); mockEnv.window.gtag = mockGtag; @@ -117,7 +118,14 @@ describe('GA4 reserved parameter guard', () => { const event = fullEvent(); const included = flattenIncludeSections(event, ['all']); - pushGA4Event(event, settings, included, mockEnv, createMockLogger()); + pushGA4Event( + event, + settings, + included, + mockEnv, + createMockLogger(), + dropped, + ); const emitted = Object.keys(mockGtag.mock.calls[0][2]); const collisions = emitted.filter((name) => @@ -133,7 +141,14 @@ describe('GA4 reserved parameter guard', () => { const event = fullEvent(); const included = flattenIncludeSections(event, ['all']); - pushGA4Event(event, settings, included, mockEnv, createMockLogger()); + pushGA4Event( + event, + settings, + included, + mockEnv, + createMockLogger(), + dropped, + ); expect(mockGtag.mock.calls[0][2]).toMatchObject({ source_type: 'browser', diff --git a/packages/web/destinations/gtag/src/__tests__/shared.test.ts b/packages/web/destinations/gtag/src/__tests__/shared.test.ts index 5b3786bdd..6e30ade30 100644 --- a/packages/web/destinations/gtag/src/__tests__/shared.test.ts +++ b/packages/web/destinations/gtag/src/__tests__/shared.test.ts @@ -1,4 +1,4 @@ -import { addScript, initializeGtag, resetLoadedScripts } from '../shared/gtag'; +import { checkGtag, initializeGtag, loadGtag } from '../shared/gtag'; import { normalizeEventName, normalizeParamName, @@ -14,56 +14,54 @@ const makeGtagWindow = ( init: { dataLayer?: unknown[]; gtag?: Gtag.Gtag } = {}, ): WindowWithDataLayer => widen(init); -// Setup DOM mocks -const mockScript = { src: '' }; -const mockCreateElement = jest.fn(() => mockScript); -const mockAppendChild = jest.fn(); +/** The src of every script `loadVendor` added. */ +const vendorScripts = () => + Array.from( + document.querySelectorAll('script[data-walkeros-vendor]'), + (script) => script.getAttribute('src'), + ); -// Mock document methods -Object.defineProperty(document, 'createElement', { - value: mockCreateElement, - writable: true, -}); +const failScript = (src: string) => + document + .querySelector(`script[src="${src}"]`) + ?.dispatchEvent(new Event('error')); -Object.defineProperty(document, 'head', { - value: { appendChild: mockAppendChild }, - writable: true, -}); +const settled = () => new Promise((resolve) => setTimeout(resolve, 0)); describe('Shared Utilities', () => { beforeEach(() => { - jest.clearAllMocks(); - mockCreateElement.mockReturnValue(mockScript); - resetLoadedScripts(); - mockScript.src = ''; // Reset script src + jest.useRealTimers(); }); - describe('addScript', () => { - it('should create and append script with correct src', () => { - addScript('G-XXXXXXXXXX'); + describe('loadGtag', () => { + it('loads gtag.js for the id through loadVendor, once', () => { + loadGtag('G-ONE'); + loadGtag('G-ONE'); - expect(mockCreateElement).toHaveBeenCalledWith('script'); - expect(mockScript.src).toBe( - 'https://www.googletagmanager.com/gtag/js?id=G-XXXXXXXXXX', - ); - expect(mockAppendChild).toHaveBeenCalledWith(mockScript); + expect(vendorScripts()).toEqual([ + 'https://www.googletagmanager.com/gtag/js?id=G-ONE', + ]); }); // A first-party serving path maps to the id inside the server container, // so the id must not be appended. - it('should use a custom src verbatim, without appending the id', () => { - addScript('G-XXXXXXXXXX', 'https://custom.domain.com/tag_serving_path/'); - - expect(mockScript.src).toBe( - 'https://custom.domain.com/tag_serving_path/', - ); - }); + it('uses a custom src verbatim and falls back to googletagmanager.com when it fails', async () => { + const src = 'https://custom.domain.com/tag_serving_path/'; + loadGtag('G-TWO', src); + expect(vendorScripts()).toContain(src); - it('should not load the same script twice', () => { - addScript('G-XXXXXXXXXX'); - addScript('G-XXXXXXXXXX'); + failScript(src); + await settled(); - expect(mockAppendChild).toHaveBeenCalledTimes(1); + expect(vendorScripts()).toContain( + 'https://www.googletagmanager.com/gtag/js?id=G-TWO', + ); + // The fallback decides once the first-party script failed. + expect(() => checkGtag('G-TWO', src)).not.toThrow(); + failScript('https://www.googletagmanager.com/gtag/js?id=G-TWO'); + expect(() => checkGtag('G-TWO', src)).toThrow( + expect.objectContaining({ code: 'vendor-not-loaded' }), + ); }); }); diff --git a/packages/web/destinations/gtag/src/__tests__/stepExamples.test.ts b/packages/web/destinations/gtag/src/__tests__/stepExamples.test.ts index 6df0ce537..ba2b76fe0 100644 --- a/packages/web/destinations/gtag/src/__tests__/stepExamples.test.ts +++ b/packages/web/destinations/gtag/src/__tests__/stepExamples.test.ts @@ -8,7 +8,6 @@ import { SettingsSchema } from '../schemas'; import { expectSimulationResolves } from '@walkeros/core/dev'; const INIT_DATE_MS = 1700000000000; -import { resetLoadedScripts } from '../shared/gtag'; type CallRecord = [string, ...unknown[]]; @@ -58,7 +57,6 @@ function makeTestEnv(): { describe('gtag web destination -- step examples', () => { beforeEach(() => { resetConsentState(); - resetLoadedScripts(); jest.useFakeTimers(); jest.setSystemTime(new Date(INIT_DATE_MS)); }); @@ -123,8 +121,9 @@ describe('gtag web destination -- step examples', () => { expect(result.error?.issues).toBeUndefined(); }); + // Non-public examples (silent, consent revoke) run in the conformance test. const stepEntries = Object.entries(examples.step).filter( - ([name]) => !initNames.has(name), + ([name, example]) => !initNames.has(name) && example.public !== false, ); it.each(stepEntries)('%s', async (name, example) => { diff --git a/packages/web/destinations/gtag/src/ads/config.ts b/packages/web/destinations/gtag/src/ads/config.ts index f4abbf1ec..2e2c91fc4 100644 --- a/packages/web/destinations/gtag/src/ads/config.ts +++ b/packages/web/destinations/gtag/src/ads/config.ts @@ -1,37 +1,47 @@ import type { Logger } from '@walkeros/core'; import type { AdsSettings, Env } from '../types'; -import { addScript, initializeGtag } from '../shared/gtag'; +import { StepError } from '@walkeros/core'; +import { initializeGtag, loadGtag } from '../shared/gtag'; import { getEnv } from '@walkeros/web-core'; export function initAds( settings: AdsSettings, loadScript: boolean | undefined, env: Env | undefined, - logger: Logger.Instance, + _logger: Logger.Instance, gtagExternal = false, -): void { - const { window, document } = getEnv(env); + // The destination's timeout, the deadline of the injected script. + timeout?: number, +): unknown[] { + const { window } = getEnv(env); const { conversionId } = settings; - if (!conversionId) logger.throw('Config settings ads.conversionId missing'); + if (!conversionId) + throw new StepError('Config settings ads.conversionId missing', { + code: 'config', + }); // Default currency value if (!settings.currency) settings.currency = 'EUR'; - if (loadScript) addScript(conversionId, undefined, document); + if (loadScript) loadGtag(conversionId, undefined, timeout); // Initialize gtag infrastructure initializeGtag(window); const gtag = window.gtag!; + const calls: unknown[] = []; // See ga4/config.ts: only signal initialisation if walkerOS did it. - if (!gtagExternal) gtag('js', new Date()); + if (!gtagExternal) calls.push(gtag('js', new Date())); // gtag init call if (settings.enhancedConversions) { - gtag('config', conversionId, { allow_enhanced_conversions: true }); + calls.push( + gtag('config', conversionId, { allow_enhanced_conversions: true }), + ); } else { - gtag('config', conversionId); + calls.push(gtag('config', conversionId)); } + return calls; } diff --git a/packages/web/destinations/gtag/src/ads/push.ts b/packages/web/destinations/gtag/src/ads/push.ts index 57b5dd05a..761ef1057 100644 --- a/packages/web/destinations/gtag/src/ads/push.ts +++ b/packages/web/destinations/gtag/src/ads/push.ts @@ -1,7 +1,7 @@ import type { WalkerOS, Logger } from '@walkeros/core'; import type { AdsSettings, AdsMapping } from '../types'; import type { DestinationWeb } from '@walkeros/web-core'; -import { isObject } from '@walkeros/core'; +import { isObject, StepError } from '@walkeros/core'; import { getEnv } from '@walkeros/web-core'; export function pushAdsEvent( @@ -11,15 +11,16 @@ export function pushAdsEvent( data: unknown, mappingName: string | undefined, env: DestinationWeb.Env | undefined, - logger: Logger.Instance, + _logger: Logger.Instance, userData?: Record, -): void { +): unknown[] { const { conversionId, currency } = settings; const eventData = isObject(data) ? data : {}; // Use label from mapping settings, fallback to mappingName for backward compatibility const conversionLabel = mapping.label || mappingName; - if (!conversionLabel) logger.throw('Config mapping ads.label missing'); + if (!conversionLabel) + throw new StepError('Config mapping ads.label missing', { code: 'config' }); const params: Gtag.CustomParams = { send_to: `${conversionId}/${conversionLabel}`, @@ -31,9 +32,9 @@ export function pushAdsEvent( const gtag = window.gtag as Gtag.Gtag; // Set user data for enhanced conversions (must be before conversion event) - if (userData) { - gtag('set', 'user_data', userData); - } + const calls: unknown[] = []; + if (userData) calls.push(gtag('set', 'user_data', userData)); - gtag('event', 'conversion', params); + calls.push(gtag('event', 'conversion', params)); + return calls; } diff --git a/packages/web/destinations/gtag/src/examples/env.ts b/packages/web/destinations/gtag/src/examples/env.ts index 83bf5deae..053b1256e 100644 --- a/packages/web/destinations/gtag/src/examples/env.ts +++ b/packages/web/destinations/gtag/src/examples/env.ts @@ -15,14 +15,6 @@ export const init: Env | undefined = { // gtag absent: initializeGtag() installs it during init. dataLayer: [], }, - document: { - createElement: () => ({ - src: '', - setAttribute: () => {}, - removeAttribute: () => {}, - }), - head: { appendChild: () => {} }, - }, }; export const push: Env = { @@ -32,14 +24,6 @@ export const push: Env = { }), dataLayer: [], }, - document: { - createElement: () => ({ - src: '', - setAttribute: () => {}, - removeAttribute: () => {}, - }), - head: { appendChild: () => {} }, - }, }; export const simulation = [ diff --git a/packages/web/destinations/gtag/src/examples/step.ts b/packages/web/destinations/gtag/src/examples/step.ts index feb0a0d56..01e8ef64c 100644 --- a/packages/web/destinations/gtag/src/examples/step.ts +++ b/packages/web/destinations/gtag/src/examples/step.ts @@ -500,3 +500,38 @@ export const multipleToolsSimultaneous: Flow.StepExample = { ], ], }; + +/** A silent rule sends none of the default gtag or dataLayer calls. */ +export const silentRule: Flow.StepExample = { + public: false, + description: + 'A rule with silent: true sends no GA4 event, no Ads conversion and no GTM push.', + in: getEvent('page view', { timestamp: 1700000500 }), + mapping: { silent: true }, + out: [], +}; + +/** + * A revocation updates Consent Mode: with config.consent { marketing: true }, + * granted first, revoking marketing denies the ad parameters the default + * mapping assigns to it. + */ +export const consentRevoke: Flow.StepExample = { + public: false, + description: + 'Revoking marketing updates gtag Consent Mode v2 with the ad parameters denied.', + command: 'consent', + in: { marketing: false }, + out: [ + [ + 'window.gtag', + 'consent', + 'update', + { + ad_storage: 'denied', + ad_user_data: 'denied', + ad_personalization: 'denied', + }, + ], + ], +}; diff --git a/packages/web/destinations/gtag/src/ga4/config.ts b/packages/web/destinations/gtag/src/ga4/config.ts index 343fe6617..3289d64ed 100644 --- a/packages/web/destinations/gtag/src/ga4/config.ts +++ b/packages/web/destinations/gtag/src/ga4/config.ts @@ -1,18 +1,21 @@ import type { WalkerOS, Logger } from '@walkeros/core'; import type { GA4Settings, Env } from '../types'; -import { addScript, initializeGtag } from '../shared/gtag'; +import { StepError } from '@walkeros/core'; +import { initializeGtag, loadGtag } from '../shared/gtag'; import { getEnv } from '@walkeros/web-core'; export function initGA4( settings: GA4Settings, loadScript: boolean | undefined, env: Env | undefined, - logger: Logger.Instance, + _logger: Logger.Instance, // True when gtag was already initialised by something else, e.g. a tag // manager. Observed once before walkerOS touches the window. gtagExternal = false, -): void { - const { window, document } = getEnv(env); + // The destination's timeout, the deadline of the injected script. + timeout?: number, +): unknown[] { + const { window } = getEnv(env); const { measurementId, transport_url, @@ -22,7 +25,10 @@ export function initGA4( init, } = settings; - if (!measurementId) logger.throw('Config settings ga4.measurementId missing'); + if (!measurementId) + throw new StepError('Config settings ga4.measurementId missing', { + code: 'config', + }); // Ensure the gtag stub exists either way, so events can queue. initializeGtag(window); @@ -31,12 +37,12 @@ export function initGA4( // configures this measurement ID itself. walkerOS then only sends events, // routed by `send_to`. Loading the script or issuing a second, later // `config` would override the identity the container already established. - if (init === false) return; + if (init === false) return []; // Load the gtag script. `server_container_url` routes measurement data and // says nothing about script delivery, so first-party serving needs its own // `scriptSrc` pointing at the container's configured tag serving path. - if (loadScript) addScript(measurementId, scriptSrc, document); + if (loadScript) loadGtag(measurementId, scriptSrc, timeout); const gtagSettings: WalkerOS.AnyObject = {}; @@ -54,8 +60,10 @@ export function initGA4( // `js` announces that gtag initialised here. If an external tag manager // already bootstrapped it, a second one carries a false, later timestamp. - if (!gtagExternal) gtag('js', new Date()); + const calls: unknown[] = []; + if (!gtagExternal) calls.push(gtag('js', new Date())); // gtag init call - gtag('config', measurementId, gtagSettings); + calls.push(gtag('config', measurementId, gtagSettings)); + return calls; } diff --git a/packages/web/destinations/gtag/src/ga4/push.ts b/packages/web/destinations/gtag/src/ga4/push.ts index 7047cfb66..0569bd511 100644 --- a/packages/web/destinations/gtag/src/ga4/push.ts +++ b/packages/web/destinations/gtag/src/ga4/push.ts @@ -1,6 +1,12 @@ import type { WalkerOS, Logger } from '@walkeros/core'; import type { GA4Settings, Parameters, Env } from '../types'; -import { isObject } from '@walkeros/core'; +import { isObject, StepError } from '@walkeros/core'; + +/** + * Reports a parameter the mapping could not name: the parameter is dropped + * and the event still goes out. + */ +export type ReportDroppedParam = (message: string, mappingKey: string) => void; import { normalizeEventName, normalizeParamName } from '../shared/mapping'; import { getEnv } from '@walkeros/web-core'; @@ -38,11 +44,14 @@ export function pushGA4Event( data: WalkerOS.AnyObject, env: Env | undefined, logger: Logger.Instance, -): void { + reportDropped: ReportDroppedParam, +): unknown { const { window } = getEnv(env); if (!settings.measurementId) - logger.throw('Config settings ga4.measurementId missing'); + throw new StepError('Config settings ga4.measurementId missing', { + code: 'config', + }); // Coerce every param name into GA4's charset before it goes out. Reserved // matching runs on the normalized name, so an illegal spelling cannot @@ -52,8 +61,10 @@ export function pushGA4Event( for (const [rawName, value] of Object.entries(data)) { const name = normalizeParamName(rawName); + // A name GA4 cannot carry, or two names one param: the mapping is + // wrong. The parameter is dropped and reported, the event still goes. if (!name) { - logger.warn('GA4 param dropped: empty name'); + reportDropped('GA4 param dropped: its name is empty', rawName); continue; } @@ -63,10 +74,13 @@ export function pushGA4Event( continue; } - if (name in eventParams) - logger.warn( - `GA4 param "${name}" overwritten: "${rawName}" collides after normalization`, + if (name in eventParams) { + reportDropped( + 'GA4 param dropped: its name collides with an earlier one after normalization', + rawName, ); + continue; + } eventParams[name] = value; } @@ -85,5 +99,5 @@ export function pushGA4Event( if (settings.debug) eventParams.debug_mode = true; const gtag = window.gtag!; - gtag('event', eventName, eventParams); + return gtag('event', eventName, eventParams); } diff --git a/packages/web/destinations/gtag/src/gtm/config.ts b/packages/web/destinations/gtag/src/gtm/config.ts index 4d6752870..01273b1ce 100644 --- a/packages/web/destinations/gtag/src/gtm/config.ts +++ b/packages/web/destinations/gtag/src/gtm/config.ts @@ -1,19 +1,36 @@ import type { Logger } from '@walkeros/core'; import type { GTMSettings, Env } from '../types'; import { isArray } from '@walkeros/core'; -import { getEnv } from '@walkeros/web-core'; +import { getEnv, loadVendor } from '@walkeros/web-core'; export const defaultDataLayer = 'dataLayer'; const defaultDomain = 'https://www.googletagmanager.com/gtm.js?id='; +/** + * The GTM container script `loadVendor` injects (once per src and page). The + * container replays its dataLayer, so it is only watched for a failure. + */ +export function gtmVendor(settings: GTMSettings, timeout?: number) { + const { containerId = '', dataLayer, domain } = settings; + const dataLayerName = dataLayer || defaultDataLayer; + const dl = dataLayerName != defaultDataLayer ? '&l=' + dataLayerName : ''; + return loadVendor( + (domain || defaultDomain) + containerId + dl, + () => getEnv().window[dataLayerName], + { timeout }, + ); +} + export function initGTM( settings: GTMSettings, loadScript?: boolean, env?: Env, _logger?: Logger.Instance, -): void { - const { window, document } = getEnv(env); - const { containerId, dataLayer, domain } = settings; + // The destination's timeout, the deadline of the injected script. + timeout?: number, +): unknown[] { + const { window } = getEnv(env); + const { containerId, dataLayer } = settings; const dataLayerName = dataLayer || defaultDataLayer; // Initialize the dataLayer (default or custom name). Custom names resolve @@ -22,27 +39,17 @@ export function initGTM( const dataLayerArray: unknown[] = isArray(existing) ? existing : []; window[dataLayerName] = dataLayerArray; + if (!loadScript || !containerId) return []; + // Load the gtm script and container. The `gtm.js` bootstrap object is part of // that snippet and belongs to whoever loads the container: a container loaded // elsewhere pushes its own, and a second one re-fires its Page View triggers. - if (loadScript && containerId) { + const calls = [ dataLayerArray.push({ 'gtm.start': new Date().getTime(), event: 'gtm.js', - }); - - addScript(containerId, domain || defaultDomain, dataLayerName, document); - } -} - -function addScript( - containerId: string, - src: string, - dataLayerName: string, - document: Document = globalThis.document, -) { - const dl = dataLayerName != defaultDataLayer ? '&l=' + dataLayerName : ''; - const script = document.createElement('script'); - script.src = src + containerId + dl; - document.head.appendChild(script); + }), + ]; + gtmVendor(settings, timeout); + return calls; } diff --git a/packages/web/destinations/gtag/src/gtm/push.ts b/packages/web/destinations/gtag/src/gtm/push.ts index b33c5cd38..0cbf407ec 100644 --- a/packages/web/destinations/gtag/src/gtm/push.ts +++ b/packages/web/destinations/gtag/src/gtm/push.ts @@ -11,7 +11,7 @@ export function pushGTMEvent( data: unknown, env?: Env, _logger?: Logger.Instance, -): void { +): unknown { const { window } = getEnv(env); const obj = { event: event.name }; // Use the name mapping by default @@ -20,7 +20,7 @@ export function pushGTMEvent( const dataLayerArray: unknown[] = isArray(existing) ? existing : []; window[dataLayerName] = dataLayerArray; - dataLayerArray.push({ + return dataLayerArray.push({ ...obj, ...(isObject(data) ? data : event), }); diff --git a/packages/web/destinations/gtag/src/index.ts b/packages/web/destinations/gtag/src/index.ts index 65b3eb564..4f3ef4609 100644 --- a/packages/web/destinations/gtag/src/index.ts +++ b/packages/web/destinations/gtag/src/index.ts @@ -3,14 +3,16 @@ import type { Settings, Destination, Env, Config } from './types'; import { initGA4, pushGA4Event } from './ga4'; import { initAds, pushAdsEvent, resolveUserData } from './ads'; import { initGTM, pushGTMEvent } from './gtm'; +import { gtmVendor } from './gtm/config'; import { getData } from './shared/mapping'; import { buildDefaultConsent, hasConsentSignal, resolveConsentMapping, } from './shared/consent'; -import { initializeGtag } from './shared/gtag'; -import { isObject } from '@walkeros/core'; +import { checkGtag, initializeGtag, settle } from './shared/gtag'; +import { hasTool, TOOL_PATHS } from './shared/tools'; +import { isObject, reportCause, stepId, StepError } from '@walkeros/core'; import { getEnv } from '@walkeros/web-core'; // Types @@ -28,20 +30,34 @@ export function resetConsentState(): void { // per Google's required ordering. Gated on the destination's OWN config: a // consent signal (config.consent set, or como_advanced) and como !== false. // Idempotent via defaultConsentSet (consent state is window-global). -function setupConsentDefault(config: Config, env: Env | undefined): void { +function setupConsentDefault(config: Config, env: Env | undefined): unknown[] { const settings = config.settings || {}; - if (defaultConsentSet) return; - if (!hasConsentSignal(config.consent, settings)) return; + if (defaultConsentSet) return []; + if (!hasConsentSignal(config.consent, settings)) return []; const defaultConsent = buildDefaultConsent(settings); - if (!defaultConsent) return; // como:false or no params + if (!defaultConsent) return []; // como:false or no params const { window } = getEnv(env); const gtag = initializeGtag(window); // ensure stub exists before any config - if (!gtag) return; + if (!gtag) return []; - gtag('consent', 'default', defaultConsent); defaultConsentSet = true; + return [gtag('consent', 'default', defaultConsent)]; +} + +/** + * Throws `vendor-not-loaded` once a script the destination loaded failed: + * gtag replays its stub, so events before that go out as usual. + */ +function checkScripts(config: Config): void { + const { settings = {}, loadScript, timeout } = config; + const { ga4, ads, gtm } = settings; + if (!loadScript) return; + if (ga4?.measurementId && ga4.init !== false) + checkGtag(ga4.measurementId, ga4.scriptSrc, timeout); + if (ads?.conversionId) checkGtag(ads.conversionId, undefined, timeout); + if (gtm?.containerId) gtmVendor(gtm, timeout).check(); } export const destinationGtag: Destination = { @@ -53,43 +69,46 @@ export const destinationGtag: Destination = { config: { settings: {} }, - init({ config, env, logger }) { - const { settings = {}, loadScript } = config; + async init({ config, env, logger }) { + const { settings = {}, loadScript, timeout } = config; const { ga4, ads, gtm } = settings; // Fail if no tools are configured - if (!ga4?.measurementId && !ads?.conversionId && !gtm?.containerId) { - logger.throw( - 'Config settings missing. Set ga4.measurementId, ads.conversionId, or gtm.containerId', - ); - } + if (!hasTool(settings)) + throw new StepError(`Config settings ${TOOL_PATHS} missing`, { + code: 'config', + }); // Observe gtag's origin BEFORE anything (including the consent default) // installs a stub, so a later `js` is only sent if walkerOS bootstrapped. const gtagExternal = !!getEnv(env).window.gtag; - // Emit the denied consent default before any tool's config call. - setupConsentDefault(config, env); - - // Initialize GA4 if configured - if (ga4?.measurementId) { - initGA4(ga4, loadScript, env, logger, gtagExternal); - } - - // Initialize Google Ads if configured - if (ads?.conversionId) { - initAds(ads, loadScript, env, logger, gtagExternal); - } - - // Initialize GTM if configured - if (gtm?.containerId) { - initGTM(gtm, loadScript, env, logger); - } + await settle( + () => [ + // Emit the denied consent default before any tool's config call. + ...setupConsentDefault(config, env), + // Initialize GA4 if configured + ...(ga4?.measurementId + ? initGA4(ga4, loadScript, env, logger, gtagExternal, timeout) + : []), + // Initialize Google Ads if configured + ...(ads?.conversionId + ? initAds(ads, loadScript, env, logger, gtagExternal, timeout) + : []), + // Initialize GTM if configured + ...(gtm?.containerId + ? initGTM(gtm, loadScript, env, logger, timeout) + : []), + ], + 'vendor-not-ready', + 'Google tag did not take its setup calls', + ); return config; }, - async push(event, { config, rule = {}, data, env, collector, logger }) { + async push(event, { config, rule = {}, data, env, collector, logger, id }) { + checkScripts(config); const { settings = {} } = config; const { ga4, ads, gtm } = settings; const eventMapping = rule.settings || {}; @@ -121,39 +140,58 @@ export const destinationGtag: Destination = { collector, ); - // Push to GA4 if configured - if (ga4?.measurementId) { - pushGA4Event(event, ga4, ga4Data, env, logger); - } + // A silent rule sends none of the default calls below. + const send = !rule.silent; // Resolve enhanced conversions user data if configured let adsUserData: Record | undefined; - if (ads?.conversionId && ads?.enhancedConversions) { + if (send && ads?.conversionId && ads?.enhancedConversions) { adsUserData = await resolveUserData(event, ads, collector); } - // @TODO: Fix condition - should check for rule.settings?.ads?.label || rule.name - // Currently requires rule.name even when label is provided via settings.ads.label - if (ads?.conversionId && rule.name) { - pushAdsEvent( - event, - ads, - eventMapping.ads, - adsData, - rule.name, - env, - logger, - adsUserData, - ); - } - - // Push to GTM if configured - if (gtm?.containerId) { - pushGTMEvent(event, gtm, eventMapping.gtm, gtmData, env, logger); - } + await settle( + () => [ + // Push to GA4 if configured + ...(send && ga4?.measurementId + ? [ + pushGA4Event(event, ga4, ga4Data, env, logger, (message, key) => + // One line per cause through the collector's cause log; the + // context names the mapping key, never its value. + reportCause( + collector, + logger, + { step: stepId('destination', id), code: 'mapping' }, + message, + { event: event.name, mappingKey: key }, + ), + ), + ] + : []), + // @TODO: Fix condition - should check for rule.settings?.ads?.label || rule.name + // Currently requires rule.name even when label is provided via settings.ads.label + ...(send && ads?.conversionId && rule.name + ? pushAdsEvent( + event, + ads, + eventMapping.ads, + adsData, + rule.name, + env, + logger, + adsUserData, + ) + : []), + // Push to GTM if configured + ...(send && gtm?.containerId + ? [pushGTMEvent(event, gtm, eventMapping.gtm, gtmData, env, logger)] + : []), + ], + 'transport', + 'Google tag call failed', + ); }, - on(type, context) { + async on(type, context) { // Only handle consent events if (type !== 'consent' || !isObject(context.data)) return; @@ -175,11 +213,10 @@ export const destinationGtag: Destination = { // Safety net for setups that fire consent without init having seen a signal: // establish the denied default once before the first update. - if (!defaultConsentSet) { - const defaultConsent = buildDefaultConsent(settings); - if (defaultConsent) gtag('consent', 'default', defaultConsent); - defaultConsentSet = true; - } + const defaultConsent = defaultConsentSet + ? undefined + : buildDefaultConsent(settings); + defaultConsentSet = true; // Build gtag consent object for update const gtagConsent: Record = {}; @@ -197,11 +234,17 @@ export const destinationGtag: Destination = { }); }); - // Only proceed if we have consent parameters to update - if (Object.keys(gtagConsent).length === 0) return; - - // Always use update after the initial default - gtag('consent', 'update', gtagConsent); + await settle( + () => [ + ...(defaultConsent ? [gtag('consent', 'default', defaultConsent)] : []), + // Always use update after the initial default, when a parameter changes + ...(Object.keys(gtagConsent).length + ? [gtag('consent', 'update', gtagConsent)] + : []), + ], + 'consent-not-applied', + 'Google tag did not take the consent update', + ); }, }; diff --git a/packages/web/destinations/gtag/src/schemas/settings.ts b/packages/web/destinations/gtag/src/schemas/settings.ts index dc09a8569..07b2dc4ff 100644 --- a/packages/web/destinations/gtag/src/schemas/settings.ts +++ b/packages/web/destinations/gtag/src/schemas/settings.ts @@ -5,6 +5,7 @@ import { AdsSettingsSchema, GTMSettingsSchema, } from './primitives'; +import { hasTool, TOOL_PATHS, TOOLS } from '../shared/tools'; export const SettingsSchema = z .object({ @@ -27,18 +28,16 @@ export const SettingsSchema = z 'Google Tag Manager specific configuration settings (like { containerId: GTM-XXXXXXX })', ).optional(), }) - .refine( - (data) => { - return ( - data.ga4?.measurementId || - data.ads?.conversionId || - data.gtm?.containerId - ); - }, - { - message: - 'At least one Google tool must be configured (ga4.measurementId, ads.conversionId, or gtm.containerId)', - }, - ); + .strict() + .refine(hasTool, { + message: `At least one Google tool must be configured (${TOOL_PATHS})`, + }) + // The same rule in JSON Schema: each alternative requires one tool's id. + .meta({ + anyOf: TOOLS.map(({ tool, id }) => ({ + required: [tool], + properties: { [tool]: { required: [id] } }, + })), + }); export type Settings = z.infer; diff --git a/packages/web/destinations/gtag/src/shared/gtag.ts b/packages/web/destinations/gtag/src/shared/gtag.ts index 926197c32..6a25fb891 100644 --- a/packages/web/destinations/gtag/src/shared/gtag.ts +++ b/packages/web/destinations/gtag/src/shared/gtag.ts @@ -1,44 +1,60 @@ // Gtag script loading and initialization utilities import type { WindowWithDataLayer } from '../types'; - -const loadedScripts = new Set(); - -// For testing: allow resetting loaded scripts -export function resetLoadedScripts(): void { - loadedScripts.clear(); -} +import { StepError } from '@walkeros/core'; +import { getEnv, loadVendor } from '@walkeros/web-core'; export const gtagScriptSrc = 'https://www.googletagmanager.com/gtag/js?id='; +/** gtag replays its dataLayer stub, so loadVendor only watches the script. */ +const gtagGlobal = () => getEnv().window.gtag; + /** - * Injects the gtag.js script. + * Loads gtag.js through `loadVendor` (once per src and page). * * `src` loads first-party from a tagging server and is used verbatim: the * serving path maps to the id inside the container, so the id is not appended. * If it fails to load, the script falls back to googletagmanager.com so a * misconfigured container degrades to working measurement instead of none. */ -export function addScript( - id: string, - src?: string, - document: Document = globalThis.document, -): void { - // Prevent loading the same script multiple times - if (loadedScripts.has(id)) return; - loadedScripts.add(id); - - const inject = (url: string, onError?: () => void) => { - const script = document.createElement('script'); - script.src = url; - if (onError) script.onerror = onError; - document.head.appendChild(script); - }; - - if (src) { - inject(src, () => inject(gtagScriptSrc + id)); - } else { - inject(gtagScriptSrc + id); +export function loadGtag(id: string, src?: string, timeout?: number): void { + const primary = loadVendor(src ?? gtagScriptSrc + id, gtagGlobal, { + timeout, + }); + if (src) + primary.ready().catch((error: unknown) => { + if (error instanceof StepError && error.code === 'vendor-not-loaded') + loadVendor(gtagScriptSrc + id, gtagGlobal, { timeout }); + }); +} + +/** + * Throws `vendor-not-loaded` once gtag.js for `id` failed to load, its + * googletagmanager.com fallback too where a first-party `src` was set. + */ +export function checkGtag(id: string, src?: string, timeout?: number): void { + try { + loadVendor(src ?? gtagScriptSrc + id, gtagGlobal, { timeout }).check(); + } catch (error) { + if (!src) throw error; + loadVendor(gtagScriptSrc + id, gtagGlobal, { timeout }).check(); + } +} + +/** + * Waits for what the gtag or dataLayer calls of one step returned (the stub + * returns nothing); a throw or a rejection becomes a StepError of `code`. + */ +export async function settle( + calls: () => unknown[], + code: string, + message: string, +): Promise { + try { + await Promise.all(calls()); + } catch (error) { + if (error instanceof StepError) throw error; + throw new StepError(message, { code, cause: error }); } } diff --git a/packages/web/destinations/gtag/src/shared/tools.ts b/packages/web/destinations/gtag/src/shared/tools.ts new file mode 100644 index 000000000..9acc988a3 --- /dev/null +++ b/packages/web/destinations/gtag/src/shared/tools.ts @@ -0,0 +1,34 @@ +import { isObject } from '@walkeros/core'; + +/** A Google tool and the setting that identifies it. */ +export interface Tool { + tool: 'ga4' | 'ads' | 'gtm'; + id: string; +} + +/** + * The tools of which the settings configure at least one. The one + * definition of that rule: init checks it, the settings schema refines on it + * and states it as JSON Schema alternatives for validate and the kit. + */ +export const TOOLS: readonly Tool[] = [ + { tool: 'ga4', id: 'measurementId' }, + { tool: 'ads', id: 'conversionId' }, + { tool: 'gtm', id: 'containerId' }, +]; + +/** The tools' setting paths in words: `ga4.measurementId, ... or gtm.containerId`. */ +export const TOOL_PATHS = TOOLS.map(({ tool, id }) => `${tool}.${id}`) + .join(', ') + .replace(/, ([^,]*)$/, ' or $1'); + +/** Whether the settings configure at least one tool. */ +export function hasTool(settings: unknown): boolean { + return ( + isObject(settings) && + TOOLS.some(({ tool, id }) => { + const value = settings[tool]; + return isObject(value) && Boolean(value[id]); + }) + ); +} diff --git a/packages/web/destinations/gtag/src/types/index.ts b/packages/web/destinations/gtag/src/types/index.ts index f21edcc1a..55bc9457d 100644 --- a/packages/web/destinations/gtag/src/types/index.ts +++ b/packages/web/destinations/gtag/src/types/index.ts @@ -25,17 +25,6 @@ export interface Env extends DestinationWeb.Env { gtag?: Gtag.Gtag; dataLayer: unknown[]; }; - document: { - createElement: (tagName: string) => { - src: string; - async?: boolean; - setAttribute: (name: string, value: string) => void; - removeAttribute: (name: string) => void; - }; - head: { - appendChild: (node: unknown) => void; - }; - }; } // Consent mode configuration diff --git a/packages/web/destinations/heap/src/__tests__/conformance.test.ts b/packages/web/destinations/heap/src/__tests__/conformance.test.ts new file mode 100644 index 000000000..79f8680fc --- /dev/null +++ b/packages/web/destinations/heap/src/__tests__/conformance.test.ts @@ -0,0 +1,21 @@ +import path from 'path'; +import { conformance } from '@walkeros/collector/dev'; +import { destinationHeap } from '..'; +import { examples, schemas } from '../dev'; + +describe('heap conformance', () => { + beforeEach(() => { + jest.useRealTimers(); + }); + + it('conforms', async () => { + await conformance('destination', destinationHeap, { + profile: 'vendor-global', + examples: examples.step, + schema: schemas.settings, + settings: { appId: 'test-app-id' }, + env: { push: examples.env.push, simulation: examples.env.simulation }, + root: path.join(__dirname, '..', '..'), + }); + }); +}); diff --git a/packages/web/destinations/heap/src/__tests__/index.test.ts b/packages/web/destinations/heap/src/__tests__/index.test.ts new file mode 100644 index 000000000..b943f0341 --- /dev/null +++ b/packages/web/destinations/heap/src/__tests__/index.test.ts @@ -0,0 +1,73 @@ +import { startFlow } from '@walkeros/collector'; +import { getEvent } from '@walkeros/core'; +import { destinationHeap } from '..'; +import type { HeapSDK } from '../types'; + +const SRC = 'https://cdn.heapanalytics.com/js/heap-APPID.js'; + +/** The script `loadVendor` added, once the destination initialized. */ +async function injected(src: string): Promise { + for (let tick = 0; tick < 50; tick++) { + const script = document.querySelector(`script[src="${src}"]`); + if (script) return script; + await new Promise((resolve) => setTimeout(resolve, 1)); + } + return null; +} + +/** The classic SDK's object as it replaces window.heap on load (no start/stopTracking). */ +function classicHeap(tracked: string[]): HeapSDK { + const noop = () => undefined; + return { + appid: 'APPID', + track: (name) => { + tracked.push(name); + }, + identify: noop, + resetIdentity: noop, + addUserProperties: noop, + addEventProperties: noop, + clearEventProperties: noop, + }; +} + +describe('heap destination', () => { + beforeEach(() => { + jest.useRealTimers(); + delete window.heap; + }); + + it('sends the events pushed before the classic SDK loaded once it has, and takes consent commands', async () => { + const tracked: string[] = []; + const { elb, collector } = await startFlow({ + destinations: { + heap: { + code: destinationHeap, + config: { + loadScript: true, + consent: { analytics: true }, + settings: { appId: 'APPID' }, + }, + }, + }, + }); + + // The grant starts the destination, whose consent call waits for the SDK. + const granted = elb('walker consent', { analytics: true }); + const script = await injected(SRC); + expect(script?.hasAttribute('data-walkeros-vendor')).toBe(true); + const first = elb(getEvent('page view')); + // The SDK read the app id from window.heap and replaces it on load. + expect(window.heap?.appid).toBe('APPID'); + window.heap = classicHeap(tracked); + script?.dispatchEvent(new Event('load')); + + await granted; + expect((await first).ok).toBe(true); + await elb('walker consent', { analytics: true }); + expect((await elb(getEvent('product view'))).ok).toBe(true); + + expect(tracked).toEqual(['page view', 'product view']); + expect(collector.status.destinations.heap?.failed ?? 0).toBe(0); + }); +}); diff --git a/packages/web/destinations/heap/src/__tests__/stepExamples.test.ts b/packages/web/destinations/heap/src/__tests__/stepExamples.test.ts index c13a06b3e..184bbd03b 100644 --- a/packages/web/destinations/heap/src/__tests__/stepExamples.test.ts +++ b/packages/web/destinations/heap/src/__tests__/stepExamples.test.ts @@ -17,7 +17,7 @@ function spyEnv(env: Env): { env: Env; collected: () => CallRecord[] } { const record = (name: string) => (...args: unknown[]) => { - calls.push([`heap.${name}`, ...args]); + calls.push([`window.heap.${name}`, ...args]); }; const heap: HeapSDK = { @@ -42,58 +42,65 @@ function spyEnv(env: Env): { env: Env; collected: () => CallRecord[] } { } describe('heap destination — step examples', () => { - it.each<[string, HeapStepExample]>(Object.entries(examples.step))( - '%s', - async (name, example) => { - const env = clone(examples.env.push) as Env; - const { env: spiedEnv, collected } = spyEnv(env); + // Non-public examples (silent) run in the conformance test. + const published = Object.entries(examples.step).filter( + ([, example]) => example.public !== false, + ); + + it.each<[string, HeapStepExample]>(published)('%s', async (name, example) => { + const env = clone(examples.env.push) as Env; + const { env: spiedEnv, collected } = spyEnv(env); - const dest = jest.requireActual('../').default; - const { elb } = await startFlow(); + const dest = jest.requireActual('../').default; + const { elb } = await startFlow(); - const baseSettings: Partial & { appId: string } = { - appId: 'test-app-id', - ...(example.settings || {}), - }; + // Effects before this index belong to the example's `before` grant. + let before = 0; + const baseSettings: Partial & { appId: string } = { + appId: 'test-app-id', + ...(example.settings || {}), + }; - if (example.command === 'consent') { - // Consent examples: declare the consent key on the destination so - // on() knows what to check, then fire walker consent. - await elb('walker destination', { - code: { ...dest, env: spiedEnv }, - config: { - consent: { analytics: true }, - settings: baseSettings, - }, - }); - // Grant first when the example declares it, so the gated destination is - // loaded before the consent under test (it never loads under denial). - // Both the grant and the consent-under-test effects are asserted. - if (example.before) await elb('walker consent', example.before); - await elb('walker consent', example.in as WalkerOS.Consent); - } else { - const event = example.in as WalkerOS.Event; - const mapping = example.mapping as WalkerOSMapping.Rule | undefined; - const mappingConfig = mapping - ? { [event.entity]: { [event.action]: mapping } } - : undefined; + if (example.command === 'consent') { + // Consent examples: declare the consent key on the destination so + // on() knows what to check, then fire walker consent. + await elb('walker destination', { + code: { ...dest, env: spiedEnv }, + config: { + consent: { analytics: true }, + settings: baseSettings, + }, + }); + // Grant first when the example declares it, so the gated destination is + // loaded before the consent under test (it never loads under denial). + // Both the grant and the consent-under-test effects are asserted. + if (example.before) await elb('walker consent', example.before); + before = collected().length; + await elb('walker consent', example.in as WalkerOS.Consent); + } else { + const event = example.in as WalkerOS.Event; + const mapping = example.mapping as WalkerOSMapping.Rule | undefined; + const mappingConfig = mapping + ? { [event.entity]: { [event.action]: mapping } } + : undefined; - await elb('walker destination', { - code: { ...dest, env: spiedEnv }, - config: { - settings: baseSettings, - mapping: mappingConfig, - }, - }); - await elb(event); - } + await elb('walker destination', { + code: { ...dest, env: spiedEnv }, + config: { + settings: baseSettings, + mapping: mappingConfig, + }, + }); + await elb(event); + } - const expected = (example.out ?? []) as ReadonlyArray; - // Filter init-time load() call. - const actual = collected().filter(([path]) => path !== 'heap.load'); - expect(actual).toEqual(expected); - }, - ); + const expected = (example.out ?? []) as ReadonlyArray; + // Filter init-time load() call. + const actual = collected() + .slice(before) + .filter(([path]) => path !== 'window.heap.load'); + expect(actual).toEqual(expected); + }); }); it('declares simulation paths that resolve', () => diff --git a/packages/web/destinations/heap/src/examples/step.ts b/packages/web/destinations/heap/src/examples/step.ts index 103b6f4e4..5a0fe16b3 100644 --- a/packages/web/destinations/heap/src/examples/step.ts +++ b/packages/web/destinations/heap/src/examples/step.ts @@ -22,7 +22,7 @@ export const defaultEventForwarding: HeapStepExample = { description: 'A walker event becomes a Heap track call with the event name and empty properties.', in: getEvent('product view', { timestamp: 1700000100 }), - out: [['heap.track', 'product view', {}]], + out: [['window.heap.track', 'product view', {}]], }; /** @@ -46,7 +46,7 @@ export const destinationLevelInclude: HeapStepExample = { }, out: [ [ - 'heap.track', + 'window.heap.track', 'purchase', { order_id: '0rd3r1d', total: 555, currency: 'EUR' }, ], @@ -66,8 +66,8 @@ export const destinationLevelIdentify: HeapStepExample = { identify: 'user.id', }, out: [ - ['heap.identify', 'us3r'], - ['heap.track', 'page view', {}], + ['window.heap.identify', 'us3r'], + ['window.heap.track', 'page view', {}], ], }; @@ -101,8 +101,8 @@ export const userLoginIdentify: HeapStepExample = { }, }, out: [ - ['heap.identify', 'user@example.com'], - ['heap.addUserProperties', { plan: 'premium', company: 'Acme' }], + ['window.heap.identify', 'user@example.com'], + ['window.heap.addUserProperties', { plan: 'premium', company: 'Acme' }], ], }; @@ -120,7 +120,7 @@ export const userLogoutReset: HeapStepExample = { reset: true, }, }, - out: [['heap.resetIdentity']], + out: [['window.heap.resetIdentity']], }; /** @@ -151,10 +151,10 @@ export const eventWithUserProperties: HeapStepExample = { }, out: [ [ - 'heap.addUserProperties', + 'window.heap.addUserProperties', { last_order_value: 555, last_order_currency: 'EUR' }, ], - ['heap.track', 'order complete', {}], + ['window.heap.track', 'order complete', {}], ], }; @@ -180,23 +180,23 @@ export const globalEventProperties: HeapStepExample = { }, }, }, - out: [['heap.addEventProperties', { page_category: 'docs' }]], + out: [['window.heap.addEventProperties', { page_category: 'docs' }]], }; /** - * Consent granted then revoked: the destination loads under the grant - * (heap.startTracking), and the later revoke calls heap.stopTracking via the - * on('consent') handler. A gated destination never loads under denial, so the + * Consent granted then revoked: the destination loads under the grant, and + * the revoke calls heap.stopTracking via the on('consent') handler; `out` is + * the revoke's call. A gated destination never loads under denial, so the * stop is a real revocation of an already-granted destination. */ export const consentRevokeStopTracking: HeapStepExample = { title: 'Consent revoked', description: - 'With consent: { analytics: true } in the destination config, the destination calls heap.startTracking once analytics is granted and heap.stopTracking when it is revoked. The classic Heap SDK that loadScript loads has neither method.', + 'With consent: { analytics: true } in the destination config, a revocation of analytics calls heap.stopTracking once Heap has loaded. The classic Heap SDK that loadScript loads has no such method, so there only the pushes stop.', command: 'consent', before: { analytics: true }, in: { analytics: false }, - out: [['heap.startTracking'], ['heap.stopTracking']], + out: [['window.heap.stopTracking']], }; /** @@ -209,5 +209,14 @@ export const consentGrantStartTracking: HeapStepExample = { 'With consent: { analytics: true } in the destination config, a walker consent grant for analytics calls heap.startTracking.', command: 'consent', in: { analytics: true }, - out: [['heap.startTracking']], + out: [['window.heap.startTracking']], +}; + +/** A silent rule without settings of its own calls nothing. */ +export const silentRule: HeapStepExample = { + public: false, + description: 'A rule with silent: true and no settings calls nothing.', + in: getEvent('page view', { timestamp: 1700000107 }), + mapping: { silent: true }, + out: [], }; diff --git a/packages/web/destinations/heap/src/index.ts b/packages/web/destinations/heap/src/index.ts index 4a93e3d6b..852d2dc71 100644 --- a/packages/web/destinations/heap/src/index.ts +++ b/packages/web/destinations/heap/src/index.ts @@ -1,9 +1,13 @@ -import type { DestinationWeb } from '@walkeros/web-core'; -import { getMappingValue, isObject, isString } from '@walkeros/core'; -import { getEnv } from '@walkeros/web-core'; +import { + getGrantedConsent, + getMappingValue, + isObject, + isString, + StepError, +} from '@walkeros/core'; +import { getEnv, loadVendor } from '@walkeros/web-core'; import type { Destination, - Env, HeapConfig, HeapPropertyValue, HeapSDK, @@ -14,15 +18,53 @@ import type { // Types export export * as DestinationHeap from './types'; +/** The classic Heap SDK `loadScript` loads. */ +const scriptSrc = (appId: string) => + `https://cdn.heapanalytics.com/js/heap-${appId}.js`; + +function isHeapSDK(value: unknown): value is HeapSDK { + return ( + isObject(value) && + [ + 'track', + 'identify', + 'resetIdentity', + 'addUserProperties', + 'addEventProperties', + 'clearEventProperties', + ].every((key) => typeof value[key] === 'function') + ); +} + +/** + * The Heap SDK: the caller-provided mock (tests) via env.window.heap, + * otherwise window.heap once it is the SDK (the page's snippet or the + * loaded classic SDK), never the loader object init leaves there. + */ +function getHeap(env: unknown): HeapSDK | undefined { + const mock = + isObject(env) && isObject(env.window) ? env.window.heap : undefined; + if (isHeapSDK(mock)) return mock; + const { heap } = getEnv().window; + return isHeapSDK(heap) ? heap : undefined; +} + /** - * Resolve the Heap SDK: prefer the caller-provided mock (tests) via - * env.window.heap, otherwise fall back to the global window.heap the - * init function established (real snippet command queue). + * The classic SDK replays no object stub, so a push waits until the script + * `loadVendor` injected (`loadScript`) has loaded; without it the page loads + * Heap and each push finds it or fails `vendor-not-ready`. */ -function getHeap(env: Env | undefined): HeapSDK | undefined { - if (env?.window?.heap) return env.window.heap; - const { window } = getEnv(env as DestinationWeb.Env | undefined); - return (window as Window).heap; +function vendor( + appId: string | undefined, + loadScript: boolean | undefined, + env: unknown, + timeout?: number, +) { + return loadVendor( + loadScript && appId ? scriptSrc(appId) : undefined, + () => getHeap(env), + { timeout }, + ); } /** @@ -44,6 +86,22 @@ function toHeapProperties(value: unknown): Record { return out; } +/** Runs one Heap call; a throw or a rejection becomes a StepError. */ +async function call( + fn: () => unknown, + code: string, + message: string, +): Promise { + try { + await fn(); + } catch (error) { + if (error instanceof StepError) throw error; + throw new StepError(message, { code, cause: error }); + } +} + +const send = (fn: () => unknown) => call(fn, 'transport', 'Heap call failed'); + export const destinationHeap: Destination = { type: 'heap', @@ -51,89 +109,35 @@ export const destinationHeap: Destination = { init({ config, env }) { const settings = config.settings; - if (!settings?.appId) return false; - - // In tests a mock is provided via env.window.heap and there's nothing - // for us to bootstrap. In production we install the Heap snippet's - // command queue and call heap.load(). - if (env?.window?.heap) return config; - - const { window, document } = getEnv(env as DestinationWeb.Env | undefined); - const w = window as Window; - const doc = document as Document; - - const heapConfig: HeapConfig = { - disableTextCapture: settings.disableTextCapture ?? true, - disablePageviewAutocapture: settings.disablePageviewAutocapture ?? true, - ...(settings.disableSessionReplay !== undefined && { - disableSessionReplay: settings.disableSessionReplay, - }), - ...(settings.secureCookie !== undefined && { - secureCookie: settings.secureCookie, - }), - ...(settings.ingestServer !== undefined && { - ingestServer: settings.ingestServer, - }), - ...settings.heapConfig, - }; - - // Install the Heap snippet's command queue if not already present. - if (!w.heap) { - const heap: HeapSDK = { - load: (appId: string, cfg?: HeapConfig) => { - heap.appid = appId; - heap.config = cfg ?? {}; - }, - track: (() => undefined) as HeapSDK['track'], - identify: (() => undefined) as HeapSDK['identify'], - resetIdentity: () => undefined, - addUserProperties: (() => undefined) as HeapSDK['addUserProperties'], - addEventProperties: (() => undefined) as HeapSDK['addEventProperties'], - clearEventProperties: () => undefined, - startTracking: () => undefined, - stopTracking: () => undefined, - q: [], + if (!settings?.appId) + throw new StepError('Heap needs settings.appId', { code: 'config' }); + + if (config.loadScript) { + // The classic SDK reads appid and config from window.heap on load. + const heapConfig: HeapConfig = { + disableTextCapture: settings.disableTextCapture ?? true, + ...(settings.secureCookie !== undefined && { + secureCookie: settings.secureCookie, + }), + ...settings.heapConfig, }; - - // Queue all calls until the real snippet boots and replaces these. - const methods: Array = [ - 'addEventProperties', - 'addUserProperties', - 'clearEventProperties', - 'identify', - 'resetIdentity', - 'track', - 'startTracking', - 'stopTracking', - ]; - for (const method of methods) { - const queue = (...args: unknown[]) => { - heap.q = heap.q || []; - heap.q.push([method, ...args]); - }; - (heap as unknown as Record void>)[ - method as string - ] = queue; - } - - w.heap = heap; + const { window } = getEnv(); + if (!window.heap) + window.heap = { appid: settings.appId, config: heapConfig }; } - // Load real snippet script unless disabled. - if (config.loadScript) addScript(settings.appId, doc); - - const heap = w.heap as HeapSDK; - if (!heap.appid) heap.load(settings.appId, heapConfig); - - return config; + vendor(settings.appId, config.loadScript, env, config.timeout); }, async push(event, { config, data, env, rule, collector }) { - const heap = getHeap(env as Env | undefined); - if (!heap) return; - - const settings = (config.settings || {}) as Partial; - const mappingSettings = (rule?.settings || {}) as Mapping; + const settings: Partial = config.settings || {}; + const heap = await vendor( + settings.appId, + config.loadScript, + env, + config.timeout, + ).ready(); + const mappingSettings: Mapping = rule?.settings || {}; // 1. Reset identity first - subsequent identify/userProperties // operate on the fresh anonymous user. @@ -142,7 +146,7 @@ export const destinationHeap: Destination = { typeof mappingSettings.reset === 'boolean' ? mappingSettings.reset : await getMappingValue(event, mappingSettings.reset, { collector }); - if (resolved) heap.resetIdentity(); + if (resolved) await send(() => heap.resetIdentity()); } // 2. Identify - rule-level wins over destination-level. @@ -151,9 +155,8 @@ export const destinationHeap: Destination = { const resolved = await getMappingValue(event, identifyMapping, { collector, }); - if (isString(resolved) && resolved.length > 0) { - heap.identify(resolved); - } + if (isString(resolved) && resolved.length > 0) + await send(() => heap.identify(resolved)); } // 3. User properties - rule-level wins over destination-level. @@ -163,9 +166,8 @@ export const destinationHeap: Destination = { const resolved = await getMappingValue(event, userPropsMapping, { collector, }); - if (isObject(resolved)) { - heap.addUserProperties(toHeapProperties(resolved)); - } + if (isObject(resolved)) + await send(() => heap.addUserProperties(toHeapProperties(resolved))); } // 4. Persistent event properties. @@ -175,9 +177,8 @@ export const destinationHeap: Destination = { mappingSettings.eventProperties, { collector }, ); - if (isObject(resolved)) { - heap.addEventProperties(toHeapProperties(resolved)); - } + if (isObject(resolved)) + await send(() => heap.addEventProperties(toHeapProperties(resolved))); } // 5. Clear persistent event properties. @@ -188,47 +189,42 @@ export const destinationHeap: Destination = { : await getMappingValue(event, mappingSettings.clearEventProperties, { collector, }); - if (resolved) heap.clearEventProperties(); + if (resolved) await send(() => heap.clearEventProperties()); } // 6. Default track - unless rule.silent is set. if (rule?.silent !== true) { const eventName = isString(rule?.name) ? rule.name : event.name; const properties = toHeapProperties(data); - heap.track(eventName, properties); + await send(() => heap.track(eventName, properties)); } }, - on(type, context) { - if (type !== 'consent' || !context.data) return; - - const heap = getHeap(context.env as Env | undefined); + // The destination's required consent decides, as it gates the pushes. The + // call waits for the loaded SDK; the classic SDK has no tracking switch, so + // a revocation there stops only the pushes. + async on(type, { config, data, env, collector }) { + const required = config.consent; + if (type !== 'consent' || !required || !isObject(data)) return; + if (!Object.keys(required).some((key) => key in data)) return; + const granted = getGrantedConsent(required, collector.consent) !== false; + + const settings = isObject(config.settings) ? config.settings : {}; + const appId = isString(settings.appId) ? settings.appId : undefined; + const heap = await vendor(appId, config.loadScript, env, config.timeout) + .ready() + .catch(() => undefined); + // Without a loaded Heap nothing collects, so there is nothing to tell. if (!heap) return; - const required = context.config?.consent; - if (!required || Object.keys(required).length === 0) return; - - const consent = context.data as Record; - const allGranted = Object.keys(required).every( - (key) => consent[key] === true, + const toggle = granted ? heap.startTracking : heap.stopTracking; + if (!toggle) return; + await call( + () => toggle.call(heap), + 'consent-not-applied', + 'Heap did not take the consent update', ); - - if (allGranted) heap.startTracking(); - else heap.stopTracking(); }, }; -function addScript(appId: string, doc: Document) { - const script = doc.createElement('script'); - script.type = 'text/javascript'; - script.async = true; - script.src = `https://cdn.heapanalytics.com/js/heap-${appId}.js`; - const firstScript = doc.getElementsByTagName('script')[0]; - if (firstScript && firstScript.parentNode) { - firstScript.parentNode.insertBefore(script, firstScript); - } else { - doc.head.appendChild(script); - } -} - export default destinationHeap; diff --git a/packages/web/destinations/heap/src/schemas/settings.ts b/packages/web/destinations/heap/src/schemas/settings.ts index 83530dc85..2afa2a849 100644 --- a/packages/web/destinations/heap/src/schemas/settings.ts +++ b/packages/web/destinations/heap/src/schemas/settings.ts @@ -1,54 +1,37 @@ import { z } from '@walkeros/core/dev'; -export const SettingsSchema = z.object({ - appId: z - .string() - .min(1) - .describe( - 'Heap App ID. Find it in your Heap project under Settings > App ID.', - ), - disableTextCapture: z - .boolean() - .describe('Disable Heap auto text capture. Default: true.') - .optional(), - disablePageviewAutocapture: z - .boolean() - .describe( - 'Disable Heap automatic pageview tracking. Default: true (walkerOS sources handle pageviews). Passed to heap.load(); the classic Heap SDK that loadScript loads ignores it.', - ) - .optional(), - disableSessionReplay: z - .boolean() - .describe( - 'Disable Heap session replay. Passed to heap.load(); the classic Heap SDK that loadScript loads ignores it.', - ) - .optional(), - secureCookie: z.boolean().describe('SSL-only cookies.').optional(), - ingestServer: z - .string() - .url() - .describe( - 'Custom server endpoint for proxying Heap data. Passed to heap.load(); the classic Heap SDK that loadScript loads ignores it.', - ) - .optional(), - heapConfig: z - .record(z.string(), z.unknown()) - .describe( - 'Additional heap.load() options, merged over the options above. Example: { "trackingServer": "https://heap.example.com" }.', - ) - .optional(), - identify: z - .unknown() - .describe( - 'Destination-level identity mapping. Resolves to a string for heap.identify(). Example: { "key": "user.id" }.', - ) - .optional(), - userProperties: z - .unknown() - .describe( - 'Destination-level user properties mapping. Resolves to object for heap.addUserProperties(). Example: { "map": { "plan": "data.plan" } }.', - ) - .optional(), -}); +export const SettingsSchema = z + .object({ + appId: z + .string() + .min(1) + .describe( + 'Heap App ID. Find it in your Heap project under Settings > App ID.', + ), + disableTextCapture: z + .boolean() + .describe('Disable Heap auto text capture. Default: true.') + .optional(), + secureCookie: z.boolean().describe('SSL-only cookies.').optional(), + heapConfig: z + .record(z.string(), z.unknown()) + .describe( + 'Additional heap.load() options, merged over the options above. Example: { "trackingServer": "https://heap.example.com" }.', + ) + .optional(), + identify: z + .unknown() + .describe( + 'Destination-level identity mapping. Resolves to a string for heap.identify(). Example: { "key": "user.id" }.', + ) + .optional(), + userProperties: z + .unknown() + .describe( + 'Destination-level user properties mapping. Resolves to object for heap.addUserProperties(). Example: { "map": { "plan": "data.plan" } }.', + ) + .optional(), + }) + .strict(); export type Settings = z.infer; diff --git a/packages/web/destinations/heap/src/types/index.ts b/packages/web/destinations/heap/src/types/index.ts index 2a1eaeb88..5c4f64343 100644 --- a/packages/web/destinations/heap/src/types/index.ts +++ b/packages/web/destinations/heap/src/types/index.ts @@ -10,12 +10,9 @@ export type HeapPropertyValue = string | number | boolean; /** Options accepted by heap.load() as the second argument. */ export interface HeapConfig { disableTextCapture?: boolean; - disablePageviewAutocapture?: boolean; - disableSessionReplay?: boolean; secureCookie?: boolean; compressCookies?: boolean; clearEventPropertiesOnNewUser?: boolean; - ingestServer?: string; trackingServer?: string; eventPropertiesStorageMedium?: string; metadataStorageMedium?: string; @@ -27,7 +24,7 @@ export interface HeapConfig { * Mirrors the snippet-provided command queue so tests can mock each method. */ export interface HeapSDK { - load: (appId: string, config?: HeapConfig) => void; + load?: (appId: string, config?: HeapConfig) => void; track: ( event: string, properties?: Record, @@ -37,11 +34,21 @@ export interface HeapSDK { addUserProperties: (properties: Record) => void; addEventProperties: (properties: Record) => void; clearEventProperties: () => void; - startTracking: () => void; - stopTracking: () => void; + /** Not in the classic SDK that loadScript loads. */ + startTracking?: () => void; + /** Not in the classic SDK that loadScript loads. */ + stopTracking?: () => void; appid?: string; config?: HeapConfig; - q?: unknown[][]; +} + +/** + * What `window.heap` holds before the classic SDK loads: the app id and + * config it reads, no methods (it replays no object stub). + */ +export interface HeapLoader { + appid: string; + config: HeapConfig; } /** @@ -55,14 +62,8 @@ export interface Settings { appId: string; /** Disable Heap auto text capture. Default: true. */ disableTextCapture?: boolean; - /** Disable Heap automatic pageview tracking. Default: true (walkerOS handles pageviews). */ - disablePageviewAutocapture?: boolean; - /** Disable Heap session replay. */ - disableSessionReplay?: boolean; /** SSL-only cookies. */ secureCookie?: boolean; - /** Custom server endpoint (first-party proxy). */ - ingestServer?: string; /** Destination-level identity mapping. Resolves to a string for heap.identify(). */ identify?: WalkerOSMapping.Value; /** Destination-level user properties mapping. Resolves to object for heap.addUserProperties(). */ @@ -103,7 +104,7 @@ export interface Env extends DestinationWeb.Env { declare global { interface Window { - heap?: HeapSDK; + heap?: HeapSDK | HeapLoader; } } diff --git a/packages/web/destinations/matomo/src/__tests__/conformance.test.ts b/packages/web/destinations/matomo/src/__tests__/conformance.test.ts new file mode 100644 index 000000000..da4f7bfef --- /dev/null +++ b/packages/web/destinations/matomo/src/__tests__/conformance.test.ts @@ -0,0 +1,21 @@ +import path from 'path'; +import { conformance } from '@walkeros/collector/dev'; +import { destinationMatomo } from '..'; +import { examples, schemas } from '../dev'; + +describe('matomo conformance', () => { + beforeEach(() => { + jest.useRealTimers(); + }); + + it('conforms', async () => { + await conformance('destination', destinationMatomo, { + profile: 'vendor-global', + examples: examples.step, + schema: schemas.settings, + settings: { siteId: '1', url: 'https://analytics.example.com/' }, + env: { push: examples.env.push, simulation: examples.env.simulation }, + root: path.join(__dirname, '..', '..'), + }); + }); +}); diff --git a/packages/web/destinations/matomo/src/__tests__/stepExamples.test.ts b/packages/web/destinations/matomo/src/__tests__/stepExamples.test.ts index 22d52984e..7f24bacaf 100644 --- a/packages/web/destinations/matomo/src/__tests__/stepExamples.test.ts +++ b/packages/web/destinations/matomo/src/__tests__/stepExamples.test.ts @@ -66,8 +66,10 @@ function makeMockPaq(): { } describe('matomo web destination -- step examples', () => { + // Consent and failure examples run in the conformance test. const stepEntries = Object.entries(examples.step).filter( - ([name]) => name !== 'init', + ([name, example]) => + name !== 'init' && !example.command && !example.outcome, ); // Matomo records a cart or an order only after an addEcommerceItem call per diff --git a/packages/web/destinations/matomo/src/examples/env.ts b/packages/web/destinations/matomo/src/examples/env.ts index fe2400019..1a0e17170 100644 --- a/packages/web/destinations/matomo/src/examples/env.ts +++ b/packages/web/destinations/matomo/src/examples/env.ts @@ -2,8 +2,8 @@ import type { Env } from '../types'; /** * Example environment configurations for Matomo destination. - * Mocks the _paq command queue and DOM elements for testing - * without requiring real browser or network. + * Mocks the _paq command queue for testing without requiring real browser + * or network. */ export const init: Env | undefined = { @@ -11,15 +11,6 @@ export const init: Env | undefined = { // _paq absent: init() is expected to install the queue. location: { href: 'https://www.example.com/page' }, }, - document: { - createElement: () => ({ - type: '', - src: '', - async: false, - defer: false, - }), - head: { appendChild: () => {} }, - }, }; export const push: Env = { @@ -27,15 +18,6 @@ export const push: Env = { _paq: [], location: { href: 'https://www.example.com/page' }, }, - document: { - createElement: () => ({ - type: '', - src: '', - async: false, - defer: false, - }), - head: { appendChild: () => {} }, - }, }; /** diff --git a/packages/web/destinations/matomo/src/examples/step.ts b/packages/web/destinations/matomo/src/examples/step.ts index f51786f5c..e491f72fe 100644 --- a/packages/web/destinations/matomo/src/examples/step.ts +++ b/packages/web/destinations/matomo/src/examples/step.ts @@ -360,3 +360,39 @@ export const customDimensions: StepExample = { ['_paq.push', ['setCustomDimension', 1, 'docs']], ], }; + +/** A silent rule without a goal sends nothing. */ +export const silentRule: StepExample = { + public: false, + description: + 'A rule with silent: true and no goal sends no command, its dimensions included.', + in: getEvent('page view', { timestamp: 1700000300 }), + mapping: { silent: true }, + out: [], +}; + +/** + * An event no rule maps: Matomo has no command for it, so the push fails + * instead of counting as delivered. + */ +export const unmappedEvent: StepExample = { + public: false, + description: + 'An event without a rule name, a tracking flag or a goal, and not a page view, has no Matomo command: the push fails with missing-input.', + in: getEvent('product view', { timestamp: 1700000301 }), + out: [], + outcome: { status: 'failed', code: 'missing-input', retryable: false }, +}; + +/** + * A revocation of the consent the destination requires stops Matomo: + * config.consent { marketing: true }, granted first, then revoked. + */ +export const consentRevoke: StepExample = { + public: false, + description: + 'For a destination with config.consent { marketing: true }, revoking marketing pushes forgetConsentGiven, which stops every tracking request of the page, link tracking included.', + command: 'consent', + in: { marketing: false }, + out: [['window._paq.push', ['forgetConsentGiven']]], +}; diff --git a/packages/web/destinations/matomo/src/index.test.ts b/packages/web/destinations/matomo/src/index.test.ts index 67f290b8a..604a4f762 100644 --- a/packages/web/destinations/matomo/src/index.test.ts +++ b/packages/web/destinations/matomo/src/index.test.ts @@ -1,4 +1,4 @@ -import type { Logger, WalkerOS } from '@walkeros/core'; +import type { Elb, Logger, WalkerOS } from '@walkeros/core'; import type { DestinationMatomo } from '.'; import { startFlow } from '@walkeros/collector'; import { createEvent, createMockLogger, getEvent, Level } from '@walkeros/core'; @@ -16,23 +16,12 @@ interface Harness { logs: LogEntry[]; } -interface ScriptElement { - type: string; - src: string; - async?: boolean; - defer?: boolean; -} - const siteId = '1'; const url = 'https://analytics.example.com/'; function createEnv(paq?: unknown[]): DestinationMatomo.Env { return { window: paq ? { _paq: paq } : {}, - document: { - createElement: () => ({ type: '', src: '', async: false, defer: false }), - head: { appendChild: () => {} }, - }, }; } @@ -65,24 +54,27 @@ async function setup( return { elb, paq, logs }; } +/** The src of every script `loadVendor` added. */ +function vendorScripts(): string[] { + return Array.from( + document.querySelectorAll('script[data-walkeros-vendor]'), + (script) => script.getAttribute('src') ?? '', + ); +} + +/** The reason the one destination of a push failed with. */ +function failure(result: Elb.PushResult) { + return Object.values(result.failed ?? {})[0]?.reason; +} + /** Runs init directly on a fresh collector and returns what it pushed and loaded. */ async function runInit(config: DestinationMatomo.Config) { const { init } = destinationMatomo; if (!init) throw new Error('init missing'); const paq: unknown[] = []; - const scripts: ScriptElement[] = []; - const env: DestinationMatomo.Env = { - window: { _paq: paq }, - document: { - createElement: () => { - const script: ScriptElement = { type: '', src: '' }; - scripts.push(script); - return script; - }, - head: { appendChild: () => {} }, - }, - }; + const env: DestinationMatomo.Env = { window: { _paq: paq } }; + const before = vendorScripts(); const { collector } = await startFlow(); await init({ @@ -93,7 +85,11 @@ async function runInit(config: DestinationMatomo.Config) { collector, }); - return { paq, scripts }; + // loadVendor adds a script once per src and page. + return { + paq, + scripts: vendorScripts().filter((src) => !before.includes(src)), + }; } function warnings(logs: LogEntry[], text: string): LogEntry[] { @@ -138,24 +134,21 @@ describe('Destination Matomo', () => { expect(paq).toEqual([['trackPageView', '/docs/']]); }); - test('unmapped event pushes nothing and warns once', async () => { + test('unmapped event pushes nothing and fails with missing-input', async () => { const { elb, paq, logs } = await setup(); - await elb('foo bar'); - await elb('foo bar'); + const result = await elb('foo bar'); - const skipped = 'Event "foo bar" skipped'; expect(paq).toEqual([]); - expect(warnings(logs, skipped)).toHaveLength(1); - expect( - logs.filter( - (log) => log.level === Level.DEBUG && log.message.includes(skipped), - ), - ).toHaveLength(1); + expect(failure(result)).toMatchObject({ + code: 'missing-input', + retryable: false, + }); + expect(warnings(logs, 'foo bar')).toEqual([]); }); - test('implicit rule without a goal is skipped', async () => { - const { elb, paq, logs } = await setup({ + test('implicit rule without a goal fails with missing-input', async () => { + const { elb, paq } = await setup({ mapping: { product: { add: { settings: { customDimensions: { '2': 'data.id' } } }, @@ -163,10 +156,22 @@ describe('Destination Matomo', () => { }, }); - await elb(getEvent('product add')); + const result = await elb(getEvent('product add')); expect(paq).toEqual([]); - expect(warnings(logs, 'Event "product add" skipped')).toHaveLength(1); + expect(failure(result)).toMatchObject({ code: 'missing-input' }); + }); + + test('a silent rule sends only its goal', async () => { + const { elb, paq } = await setup({ + mapping: { + page: { view: { silent: true, settings: { goalId: '1' } } }, + }, + }); + + await elb(pageView); + + expect(paq).toEqual([['trackGoal', '1', undefined]]); }); test('implicit rule with a goal sends only the goal', async () => { @@ -207,38 +212,57 @@ describe('Destination Matomo', () => { expect(paq).toEqual([['trackGoal', goalId, undefined]]); }); - test('an invalid goalId sends the main hit, no goal and warns once', async () => { - const { elb, paq, logs } = await setup({ - mapping: { - promotion: { - visible: { - name: 'trackEvent', - data: { set: [{ value: 'promotion' }, { value: 'visible' }] }, - settings: { goalId: 'goal_1' }, + test('an invalid goalId sends the hit without the goal and one ERROR line per cause', async () => { + const paq: unknown[] = []; + const logs: LogEntry[] = []; + // The default level: only ERROR lines reach the handler. + const { elb } = await startFlow({ + logger: { handler: (level, message) => logs.push({ level, message }) }, + destinations: { + matomo: { + code: { ...destinationMatomo, env: createEnv(paq) }, + config: { + settings: { siteId, url, enableLinkTracking: false }, + mapping: { + promotion: { + visible: { + name: 'trackEvent', + data: { + set: [{ value: 'promotion' }, { value: 'visible' }], + }, + settings: { goalId: 'goal_1' }, + }, + }, + }, }, }, }, }); - await elb(getEvent('promotion visible')); - await elb(getEvent('promotion visible')); + const first = await elb(getEvent('promotion visible')); + const second = await elb(getEvent('promotion visible')); + expect(first.ok).toBe(true); + expect(second.ok).toBe(true); expect(paq).toEqual([ ['trackEvent', 'promotion', 'visible'], ['trackEvent', 'promotion', 'visible'], ]); - expect(warnings(logs, 'goalId')).toHaveLength(1); + // The second occurrence within the window is counted, not logged. + const errors = logs.filter((log) => log.level === Level.ERROR); + expect(errors).toHaveLength(1); + expect(errors[0].message).toContain('goalId'); }); - test('an implicit rule with only an invalid goal is skipped as unmapped', async () => { - const { elb, paq, logs } = await setup({ + test('an implicit rule with only an invalid goal is unmapped', async () => { + const { elb, paq } = await setup({ mapping: { product: { add: { settings: { goalId: 'goal_1' } } } }, }); - await elb(getEvent('product add')); + const result = await elb(getEvent('product add')); expect(paq).toEqual([]); - expect(warnings(logs, 'Event "product add" skipped')).toHaveLength(1); + expect(failure(result)).toMatchObject({ code: 'missing-input' }); }); test('explicit rule name passes through', async () => { @@ -283,13 +307,41 @@ describe('Destination Matomo', () => { expect(paq).toEqual([...group('/a/'), ...group('/b/')]); }); - test('missing _paq warns and pushes nothing', async () => { - const { elb, paq, logs } = await setup({ init: true }, { queue: false }); + test('missing _paq fails with vendor-not-ready and pushes nothing', async () => { + const { elb, paq } = await setup({ init: true }, { queue: false }); - await elb(pageView); + const result = await elb(pageView); expect(paq).toEqual([]); - expect(warnings(logs, '_paq')).toHaveLength(1); + expect(failure(result)).toMatchObject({ + code: 'vendor-not-ready', + retryable: true, + }); + }); + + test('a consent revocation stops Matomo for a consent-gated destination, a grant resumes it', async () => { + const { elb, paq } = await setup({ consent: { marketing: true } }); + + await elb('walker consent', { marketing: true }); + await elb(pageView); + await elb('walker consent', { marketing: false }); + + expect(paq).toEqual([ + ['setConsentGiven'], + ['trackPageView', title], + ['forgetConsentGiven'], + ]); + }); + + test('with disableCookies a consent grant keeps cookies off', async () => { + const { elb, paq } = await setup({ + consent: { marketing: true }, + settings: { disableCookies: true }, + }); + + await elb('walker consent', { marketing: true }); + + expect(paq).toEqual([['disableCookies'], ['setConsentGiven', false]]); }); }); @@ -422,9 +474,7 @@ describe('Destination Matomo', () => { settings: { siteId, url }, }); - expect(scripts.map((script) => script.src)).toEqual([ - 'https://analytics.example.com/matomo.js', - ]); + expect(scripts).toEqual(['https://analytics.example.com/matomo.js']); expect(paq).toEqual([ ['setTrackerUrl', 'https://analytics.example.com/matomo.php'], ['setSiteId', siteId], @@ -435,9 +485,9 @@ describe('Destination Matomo', () => { test.each([ ['siteId', { url }], ['url', { siteId }], - ])('throws for missing %s with loadScript', async (_name, settings) => { + ])('throws for missing %s with loadScript', async (name, settings) => { await expect(runInit({ loadScript: true, settings })).rejects.toThrow( - 'missing', + `needs settings ${name}`, ); }); @@ -455,14 +505,11 @@ describe('Destination Matomo', () => { 'https://analytics.example.com', 'https://analytics.example.com//', ])('normalizes %s to one trailing slash', async (baseUrl) => { - const { paq, scripts } = await runInit({ + const { paq } = await runInit({ loadScript: true, settings: { siteId, url: baseUrl, enableLinkTracking: false }, }); - expect(scripts.map((script) => script.src)).toEqual([ - 'https://analytics.example.com/matomo.js', - ]); expect(paq).toEqual([ ['setTrackerUrl', 'https://analytics.example.com/matomo.php'], ['setSiteId', siteId], diff --git a/packages/web/destinations/matomo/src/index.ts b/packages/web/destinations/matomo/src/index.ts index a781d999a..07b537777 100644 --- a/packages/web/destinations/matomo/src/index.ts +++ b/packages/web/destinations/matomo/src/index.ts @@ -1,86 +1,130 @@ import type { Mapping, Destination, Env } from './types'; -import { getMappingValue, isArray } from '@walkeros/core'; -import { getEnv } from '@walkeros/web-core'; +import { + getGrantedConsent, + getMappingValue, + isArray, + isObject, + reportCause, + stepId, + StepError, +} from '@walkeros/core'; +import { getEnv, loadVendor } from '@walkeros/web-core'; import { resolveDimensionMap, sequence } from './dimensions'; -import { logSkip } from './skip'; // Types export * as DestinationMatomo from './types'; +/** + * Pushes commands onto the queue in one synchronous block, so another + * event's commands never interleave, then waits for what each push returns. + */ +async function pushAll( + queue: unknown[], + commands: unknown[][], + code: string, + message: string, +): Promise { + try { + await Promise.all(commands.map((command) => queue.push(command))); + } catch (error) { + throw new StepError(message, { code, cause: error }); + } +} + +/** + * matomo.js, injected by `loadVendor` with `loadScript`. `_paq` replays its + * own queue, so commands are pushed synchronously and only a failed script + * is reported. + */ +function vendor(url: string | undefined, env?: Env, timeout?: number) { + return loadVendor( + url ? `${normalizeUrl(url)}matomo.js` : undefined, + () => getEnv(env).window._paq, + { timeout }, + ); +} + export const destinationMatomo: Destination = { type: 'matomo', config: {}, - init({ config, env, logger }) { + async init({ config, env }) { const { window } = getEnv(env); const { settings = {}, loadScript } = config; const { siteId, url } = settings; - - // Set up the Matomo command queue - window._paq = window._paq || []; - const queue = window._paq; - const paq = (command: unknown[]) => { - queue.push(command); - }; + const commands: unknown[][] = []; if (loadScript) { // Required only to load and address the tracker - if (!siteId) return logger.throw('Config settings siteId missing'); - if (!url) return logger.throw('Config settings url missing'); + if (!siteId || !url) + throw new StepError( + `Matomo with loadScript needs settings ${[ + ...(siteId ? [] : ['siteId']), + ...(url ? [] : ['url']), + ].join(' and ')}`, + { code: 'config' }, + ); const baseUrl = normalizeUrl(url); - - // Load the Matomo tracking script - addScript(baseUrl, env); - - // Configure tracker URL and site ID - paq(['setTrackerUrl', baseUrl + 'matomo.php']); - paq(['setSiteId', siteId]); + commands.push(['setTrackerUrl', baseUrl + 'matomo.php']); + commands.push(['setSiteId', siteId]); } // Cookie-free tracking - if (settings.disableCookies) paq(['disableCookies']); + if (settings.disableCookies) commands.push(['disableCookies']); // Enable link tracking by default - if (settings.enableLinkTracking !== false) paq(['enableLinkTracking']); + if (settings.enableLinkTracking !== false) + commands.push(['enableLinkTracking']); // Heart beat timer for accurate time-on-page if (settings.enableHeartBeatTimer) - paq(['enableHeartBeatTimer', settings.enableHeartBeatTimer]); + commands.push(['enableHeartBeatTimer', settings.enableHeartBeatTimer]); + + // The Matomo command queue + window._paq = window._paq || []; + if (loadScript) vendor(url, env, config.timeout); + await pushAll( + window._paq, + commands, + 'vendor-not-ready', + 'Matomo did not take its setup commands', + ); }, async push(event, { config, rule = {}, data, env, collector, logger, id }) { - const { window } = getEnv(env); - const queue = window._paq; - if (!queue) { - logger.warn('Matomo _paq queue missing, init() not run'); - return; - } - const paq = (command: unknown[]) => { - queue.push(command); - }; - const settings = config.settings || {}; + if (config.loadScript) vendor(settings.url, env, config.timeout).check(); + const queue = getEnv(env).window._paq; + if (!queue) + throw new StepError('Matomo _paq queue missing: init did not run', { + code: 'vendor-not-ready', + }); + const eventMapping: Mapping = rule.settings || {}; const { goalId } = eventMapping; const parameters = isArray(data) ? data : [data]; - // Matomo drops a conversion whose goal id is not a positive integer - const goal = isGoalId(goalId) ? goalId : undefined; + // Matomo drops a conversion whose goal id is not a positive integer: the + // hit goes without it, and the cause log reports it once per cause. + const goal = goalId === undefined || isGoalId(goalId) ? goalId : undefined; if (goalId !== undefined && goal === undefined) - logSkip( + reportCause( + collector, logger, - `${id}|${event.name}|invalid goalId`, - `Goal of event "${event.name}" skipped: goalId ${JSON.stringify(goalId)} is not a positive integer`, - { event: event.name, reason: 'invalid goalId' }, + { step: stepId('destination', id), code: 'config' }, + 'Matomo goal skipped: goalId is not a positive integer', + { event: event.name, mappingKey: 'goalId' }, ); // A tracking flag picks the method, an explicit rule name (already // applied to event.name) passes through, and a page view defaults to - // trackPageView. Anything else sends only its goal, or is unmapped. + // trackPageView. A silent rule sends none of them, only its goal. let command: unknown[] | undefined; - if (eventMapping.siteSearch) { + if (rule.silent) { + command = undefined; + } else if (eventMapping.siteSearch) { command = ['trackSiteSearch', ...parameters]; } else if (eventMapping.contentImpression) { command = ['trackContentImpression', ...parameters]; @@ -98,13 +142,10 @@ export const destinationMatomo: Destination = { ] : ['trackPageView', ...parameters]; } else if (goal === undefined) { - logSkip( - logger, - `${id}|${event.name}|unmapped`, - `Event "${event.name}" skipped: no rule name, tracking flag or goal, and not a page view`, - { event: event.name, reason: 'unmapped' }, + throw new StepError( + `Matomo has no command for "${event.name}": map it with a rule name, a tracking flag or a goal, or ignore it`, + { code: 'missing-input' }, ); - return; } // Resolve everything first, so this event's commands are pushed in one @@ -123,13 +164,48 @@ export const destinationMatomo: Destination = { ), ); - before.forEach(paq); - if (command) paq(command); - - // Goal tracking alongside event - if (goal !== undefined) paq(['trackGoal', goal, goalValue]); + // A silent rule without a goal sends nothing, dimensions included. + if (command || goal !== undefined) + await pushAll( + queue, + [ + ...before, + ...(command ? [command] : []), + // Goal tracking alongside event + ...(goal !== undefined ? [['trackGoal', goal, goalValue]] : []), + ...after, + ], + 'transport', + 'Matomo did not take the event commands', + ); + }, - after.forEach(paq); + // The destination's required consent decides, as it gates the pushes: a + // revocation stops every tracking request (link tracking included) for the + // page, a grant resumes them. With disableCookies the grant keeps cookies + // off. + async on(type, { config, data, env, collector }) { + const required = config.consent; + if (type !== 'consent' || !required || !isObject(data)) return; + if (!Object.keys(required).some((key) => key in data)) return; + const queue = getEnv(env).window._paq; + if (!queue) return; + + const granted = getGrantedConsent(required, collector.consent) !== false; + const disableCookies = + isObject(config.settings) && config.settings.disableCookies === true; + const command = !granted + ? ['forgetConsentGiven'] + : disableCookies + ? ['setConsentGiven', false] + : ['setConsentGiven']; + + await pushAll( + queue, + [command], + 'consent-not-applied', + 'Matomo did not take the consent update', + ); }, }; @@ -147,14 +223,4 @@ function normalizeUrl(url: string): string { return `${url.replace(/\/+$/, '')}/`; } -function addScript(url: string, env?: Env) { - const { document } = getEnv(env); - const script = document.createElement('script'); - script.type = 'text/javascript'; - script.src = url + 'matomo.js'; - script.async = true; - script.defer = true; - document.head.appendChild(script); -} - export default destinationMatomo; diff --git a/packages/web/destinations/matomo/src/schemas/settings.ts b/packages/web/destinations/matomo/src/schemas/settings.ts index 7796c4104..98b9b8104 100644 --- a/packages/web/destinations/matomo/src/schemas/settings.ts +++ b/packages/web/destinations/matomo/src/schemas/settings.ts @@ -7,40 +7,42 @@ export const CustomDimensionsSchema = z.record( ValueSchema, ); -export const SettingsSchema = z.object({ - siteId: z - .string() - .min(1) - .describe( - 'Matomo Site ID (like 1, 2, etc.). Required when loadScript is true', - ) - .optional(), - url: z - .string() - .url() - .describe( - 'Base URL of your Matomo instance, normalized to one trailing slash (like https://analytics.example.com/ or https://yourname.matomo.cloud/). Required when loadScript is true', - ) - .optional(), - disableCookies: z - .boolean() - .default(false) - .describe('Disable all tracking cookies for cookie-free analytics'), - enableLinkTracking: z - .boolean() - .default(true) - .describe('Enable automatic outlink and download tracking'), - enableHeartBeatTimer: z - .number() - .int() - .positive() - .optional() - .describe( - 'Enable heart beat timer with interval in seconds for accurate time-on-page', - ), - customDimensions: CustomDimensionsSchema.describe( - 'Custom dimensions applied to every hit, keyed by bare dimension id (like { "1": "data.userType" }). Each value is resolved per event. Rule-level customDimensions win per key.', - ).optional(), -}); +export const SettingsSchema = z + .object({ + siteId: z + .string() + .min(1) + .describe( + 'Matomo Site ID (like 1, 2, etc.). Required when loadScript is true', + ) + .optional(), + url: z + .string() + .url() + .describe( + 'Base URL of your Matomo instance, normalized to one trailing slash (like https://analytics.example.com/ or https://yourname.matomo.cloud/). Required when loadScript is true', + ) + .optional(), + disableCookies: z + .boolean() + .default(false) + .describe('Disable all tracking cookies for cookie-free analytics'), + enableLinkTracking: z + .boolean() + .default(true) + .describe('Enable automatic outlink and download tracking'), + enableHeartBeatTimer: z + .number() + .int() + .positive() + .optional() + .describe( + 'Enable heart beat timer with interval in seconds for accurate time-on-page', + ), + customDimensions: CustomDimensionsSchema.describe( + 'Custom dimensions applied to every hit, keyed by bare dimension id (like { "1": "data.userType" }). Each value is resolved per event. Rule-level customDimensions win per key.', + ).optional(), + }) + .strict(); export type Settings = z.infer; diff --git a/packages/web/destinations/matomo/src/skip.test.ts b/packages/web/destinations/matomo/src/skip.test.ts deleted file mode 100644 index 605efb06d..000000000 --- a/packages/web/destinations/matomo/src/skip.test.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { createMockLogger } from '@walkeros/core'; -import { logSkip, SKIP_KEY_CAP } from './skip'; - -describe('logSkip', () => { - test('warns once per key, then logs at debug', () => { - const logger = createMockLogger(); - - logSkip(logger, 'dest|foo bar|unmapped', 'skipped', { reason: 'unmapped' }); - logSkip(logger, 'dest|foo bar|unmapped', 'skipped', { reason: 'unmapped' }); - - expect(logger.warn).toHaveBeenCalledTimes(1); - expect(logger.warn).toHaveBeenCalledWith('skipped', { reason: 'unmapped' }); - expect(logger.debug).toHaveBeenCalledTimes(1); - }); - - test('beyond the cap every new key logs at debug', () => { - const logger = createMockLogger(); - for (let i = 0; i < SKIP_KEY_CAP; i++) - logSkip(logger, `dest|event ${i}|unmapped`, 'skipped', {}); - logger.warn.mockClear(); - logger.debug.mockClear(); - - logSkip(logger, 'dest|one more|unmapped', 'skipped', {}); - - expect(logger.warn).not.toHaveBeenCalled(); - expect(logger.debug).toHaveBeenCalledTimes(1); - }); -}); diff --git a/packages/web/destinations/matomo/src/skip.ts b/packages/web/destinations/matomo/src/skip.ts deleted file mode 100644 index 4508f2c59..000000000 --- a/packages/web/destinations/matomo/src/skip.ts +++ /dev/null @@ -1,23 +0,0 @@ -import type { Logger } from '@walkeros/core'; - -/** Most distinct skip keys that get a warning; beyond it every skip logs at debug. */ -export const SKIP_KEY_CAP = 1000; - -// Module-level and bounded, so unbounded event names cannot grow memory. -const warned = new Set(); - -/** Warns once per key, then logs the same skip at debug. */ -export function logSkip( - logger: Logger.Instance, - key: string, - message: string, - meta: Record, -): void { - if (!warned.has(key) && warned.size < SKIP_KEY_CAP) { - warned.add(key); - logger.warn(message, meta); - return; - } - - logger.debug(message, meta); -} diff --git a/packages/web/destinations/matomo/src/types/index.ts b/packages/web/destinations/matomo/src/types/index.ts index 4c7ed9000..0a45f5734 100644 --- a/packages/web/destinations/matomo/src/types/index.ts +++ b/packages/web/destinations/matomo/src/types/index.ts @@ -51,17 +51,6 @@ export interface Env extends DestinationWeb.Env { _paq?: Array; location?: { href?: string }; }; - document: { - createElement: (tagName: string) => { - type: string; - src: string; - async?: boolean; - defer?: boolean; - }; - head: { - appendChild: (node: unknown) => void; - }; - }; } export type Types = CoreDestination.Types; diff --git a/packages/web/destinations/meta/src/__tests__/conformance.test.ts b/packages/web/destinations/meta/src/__tests__/conformance.test.ts new file mode 100644 index 000000000..83245fa43 --- /dev/null +++ b/packages/web/destinations/meta/src/__tests__/conformance.test.ts @@ -0,0 +1,21 @@ +import path from 'path'; +import { conformance } from '@walkeros/collector/dev'; +import { destinationMeta } from '..'; +import { examples, schemas } from '../dev'; + +describe('meta conformance', () => { + beforeEach(() => { + jest.useRealTimers(); + }); + + it('conforms', async () => { + await conformance('destination', destinationMeta, { + profile: 'vendor-global', + examples: examples.step, + schema: schemas.settings, + settings: { pixelId: '1234567890' }, + env: { push: examples.env.push, simulation: examples.env.simulation }, + root: path.join(__dirname, '..', '..'), + }); + }); +}); diff --git a/packages/web/destinations/meta/src/__tests__/stepExamples.test.ts b/packages/web/destinations/meta/src/__tests__/stepExamples.test.ts index d471c91d6..919572ff4 100644 --- a/packages/web/destinations/meta/src/__tests__/stepExamples.test.ts +++ b/packages/web/destinations/meta/src/__tests__/stepExamples.test.ts @@ -16,8 +16,9 @@ const initOut = (examples.step.init.out ?? []) as ReadonlyArray; const noopLogger = createLogger(); describe('meta web destination -- step examples', () => { + // Non-public examples (silent, consent) run in the conformance test. const stepEntries = Object.entries(examples.step).filter( - ([name]) => name !== 'init', + ([name, example]) => name !== 'init' && example.public !== false, ); it('init', async () => { diff --git a/packages/web/destinations/meta/src/examples/env.ts b/packages/web/destinations/meta/src/examples/env.ts index 5bc4afaa0..2a475e47e 100644 --- a/packages/web/destinations/meta/src/examples/env.ts +++ b/packages/web/destinations/meta/src/examples/env.ts @@ -13,15 +13,6 @@ const noop = () => {}; export const init: Env | undefined = { // Environment before initialization (fbq/_fbq absent until setup() runs) window: {}, - document: { - createElement: () => ({ - src: '', - async: false, - setAttribute: () => {}, - removeAttribute: () => {}, - }), - head: { appendChild: () => {} }, - }, }; export const push: Env = { @@ -43,15 +34,6 @@ export const push: Env = { version: '2.0', }), }, - document: { - createElement: () => ({ - src: '', - async: false, - setAttribute: () => {}, - removeAttribute: () => {}, - }), - head: { appendChild: () => {} }, - }, }; /** @@ -61,6 +43,3 @@ export const push: Env = { export const simulation = [ 'call:window.fbq', // Track fbq function calls ]; - -// Future: error scenarios (v2) -// export const error: Env = { ... }; diff --git a/packages/web/destinations/meta/src/examples/step.ts b/packages/web/destinations/meta/src/examples/step.ts index 43aa6ecc1..934c7f9fa 100644 --- a/packages/web/destinations/meta/src/examples/step.ts +++ b/packages/web/destinations/meta/src/examples/step.ts @@ -258,3 +258,28 @@ export const customEventWithTrackCustom: Flow.StepExample = { ], ], }; + +/** A silent rule sends no fbq event. */ +export const silentRule: Flow.StepExample = { + public: false, + description: 'A rule with silent: true sends no fbq event.', + in: getEvent('page view', { + timestamp: 1700000006, + id: '1700000006-gr0up-1', + }), + mapping: { silent: true }, + out: [], +}; + +/** + * A revocation of the consent the destination requires pauses the pixel's + * own fires: config.consent { marketing: true }, granted first, then revoked. + */ +export const consentRevoke: Flow.StepExample = { + public: false, + description: + 'For a destination with config.consent { marketing: true }, revoking marketing calls fbq consent revoke, so the pixel stops its own fires too.', + command: 'consent', + in: { marketing: false }, + out: [['window.fbq', 'consent', 'revoke']], +}; diff --git a/packages/web/destinations/meta/src/index.test.ts b/packages/web/destinations/meta/src/index.test.ts index 85183fecd..4ac9c68f7 100644 --- a/packages/web/destinations/meta/src/index.test.ts +++ b/packages/web/destinations/meta/src/index.test.ts @@ -60,21 +60,9 @@ describe('Destination Meta Pixel', () => { }); test('init with load script', async () => { - const scriptEnv = clone(examples.env.push); - const createElementSpy = jest.fn(() => ({ - src: '', - async: false, - setAttribute: jest.fn(), - removeAttribute: jest.fn(), - })); - const appendChildSpy = jest.fn(); - - scriptEnv.document.createElement = createElementSpy; - scriptEnv.document.head.appendChild = appendChildSpy; - const destinationWithEnv = { ...destination, - env: scriptEnv, + env: clone(examples.env.push), }; elb('walker destination', { code: destinationWithEnv, @@ -86,8 +74,33 @@ describe('Destination Meta Pixel', () => { await elb(event); - expect(createElementSpy).toHaveBeenCalledWith('script'); - expect(appendChildSpy).toHaveBeenCalled(); + const script = document.querySelector( + 'script[src="https://connect.facebook.net/en_US/fbevents.js"]', + ); + expect(script?.hasAttribute('data-walkeros-vendor')).toBe(true); + }); + + test('consent revocation reaches the pixel of a gated destination', async () => { + const calls: unknown[][] = []; + const env = clone(examples.env.push); + env.window.fbq = (...args: unknown[]) => { + calls.push(args); + }; + elb('walker destination', { + code: { ...destination, env }, + config: { consent: { marketing: true }, settings: { pixelId } }, + }); + + await elb('walker consent', { marketing: true }); + await elb(event); + await elb('walker consent', { marketing: false }); + + expect(calls).toEqual([ + ['init', pixelId], + ['consent', 'grant'], + ['track', event.name, {}, { eventID: event.id }], + ['consent', 'revoke'], + ]); }); test('push', async () => { diff --git a/packages/web/destinations/meta/src/index.ts b/packages/web/destinations/meta/src/index.ts index 4ff5aff28..b45872d20 100644 --- a/packages/web/destinations/meta/src/index.ts +++ b/packages/web/destinations/meta/src/index.ts @@ -1,37 +1,74 @@ -import type { Destination, Env } from './types'; -import { addScript, setup } from './setup'; -import { isObject } from '@walkeros/core'; -import { getEnv } from '@walkeros/web-core'; +import type { Config, Destination, Env } from './types'; +import { SCRIPT_SRC, setup } from './setup'; +import { getGrantedConsent, isObject, StepError } from '@walkeros/core'; +import { getEnv, loadVendor } from '@walkeros/web-core'; // Types export * as DestinationMeta from './types'; +/** + * The script `loadVendor` injects with `loadScript`. fbq replays its own stub, + * so it is called synchronously and only a failed script is reported. + */ +function vendor(config: Config, env?: Env) { + return loadVendor( + config.loadScript ? SCRIPT_SRC : undefined, + () => getEnv(env).window.fbq, + { timeout: config.timeout }, + ); +} + +/** The fbq stub or library, or a StepError when it is not there. */ +function pixel(env?: Env): facebook.Pixel.Event { + const { fbq } = getEnv(env).window; + if (!fbq) + throw new StepError('Meta Pixel not ready: window.fbq is missing', { + code: 'vendor-not-ready', + }); + return fbq; +} + +/** Runs one fbq call; a throw or a rejection becomes a StepError of `code`. */ +async function call( + fn: () => unknown, + code: string, + message: string, +): Promise { + try { + await fn(); + } catch (error) { + throw new StepError(message, { code, cause: error }); + } +} + export const destinationMeta: Destination = { type: 'meta-pixel', config: {}, - init({ config, env, logger }) { - const { settings, loadScript } = config; - const { pixelId } = settings || {}; - - // Load Meta Pixel script if required (fbevents.js) - if (loadScript) addScript(env); - - // Required pixel id - if (!pixelId) logger.throw('Config settings pixelId missing'); + async init({ config, env }) { + const { pixelId } = config.settings || {}; + if (!pixelId) + throw new StepError('Meta Pixel needs settings.pixelId', { + code: 'config', + }); - // fbq function setup + // The fbq stub first, so calls queue until fbevents.js replays them setup(env); + vendor(config, env); - const { window } = getEnv(env); - window.fbq!('init', pixelId!); + const fbq = pixel(env); + await call( + () => fbq('init', pixelId), + 'vendor-not-ready', + 'Meta Pixel init call failed', + ); }, - push(event, { config, rule = {}, data, env }) { + async push(event, { config, rule = {}, data, env }) { + vendor(config, env).check(); + const fbq = pixel(env); const { track, trackCustom } = rule.settings || {}; - const { window } = getEnv(env); - const fbq = window.fbq!; // page view if (event.name === 'page view' && !rule.settings) { @@ -41,11 +78,35 @@ export const destinationMeta: Destination = { const eventName = track || trackCustom || event.name; - fbq( - trackCustom ? 'trackCustom' : 'track', - String(eventName), - isObject(data) ? data : {}, - { eventID: event.id }, + if (!rule.silent) + await call( + () => + fbq( + trackCustom ? 'trackCustom' : 'track', + String(eventName), + isObject(data) ? data : {}, + { eventID: event.id }, + ), + 'transport', + 'Meta Pixel call failed', + ); + }, + + // The destination's required consent decides, as it gates the pushes: a + // revocation pauses the pixel's own fires, a grant resumes them. + async on(type, { config, data, env, collector }) { + const required = config.consent; + if (type !== 'consent' || !required || !isObject(data)) return; + if (!Object.keys(required).some((key) => key in data)) return; + + const granted = getGrantedConsent(required, collector.consent) !== false; + const { fbq } = getEnv(env).window; + if (typeof fbq !== 'function') return; + + await call( + () => fbq('consent', granted ? 'grant' : 'revoke'), + 'consent-not-applied', + 'Meta Pixel did not take the consent update', ); }, }; diff --git a/packages/web/destinations/meta/src/schemas/settings.ts b/packages/web/destinations/meta/src/schemas/settings.ts index 63b3c84f3..06b67222a 100644 --- a/packages/web/destinations/meta/src/schemas/settings.ts +++ b/packages/web/destinations/meta/src/schemas/settings.ts @@ -1,10 +1,12 @@ import { z } from '@walkeros/core/dev'; import { PixelId } from './primitives'; -export const SettingsSchema = z.object({ - pixelId: PixelId.describe( - 'Your Meta Pixel ID from Facebook Business Manager (like 1234567890)', - ), -}); +export const SettingsSchema = z + .object({ + pixelId: PixelId.describe( + 'Your Meta Pixel ID from Facebook Business Manager (like 1234567890)', + ), + }) + .strict(); export type Settings = z.infer; diff --git a/packages/web/destinations/meta/src/setup.ts b/packages/web/destinations/meta/src/setup.ts index a6e6961b7..587c2a467 100644 --- a/packages/web/destinations/meta/src/setup.ts +++ b/packages/web/destinations/meta/src/setup.ts @@ -1,16 +1,8 @@ import type { Env } from './types'; import { getEnv } from '@walkeros/web-core'; -export function addScript( - env?: Env, - src = 'https://connect.facebook.net/en_US/fbevents.js', -) { - const { document } = getEnv(env); - const script = document.createElement('script'); - script.src = src; - script.async = true; - document.head.appendChild(script); -} +/** The Meta Pixel library, loaded through `loadVendor`. */ +export const SCRIPT_SRC = 'https://connect.facebook.net/en_US/fbevents.js'; /** * The fbevents.js bootstrap stub: a callable that queues calls until the real diff --git a/packages/web/destinations/meta/src/types/index.ts b/packages/web/destinations/meta/src/types/index.ts index b8c65e94a..72d0de556 100644 --- a/packages/web/destinations/meta/src/types/index.ts +++ b/packages/web/destinations/meta/src/types/index.ts @@ -28,15 +28,6 @@ export interface Env extends DestinationWeb.Env { fbq?: facebook.Pixel.Event; _fbq?: facebook.Pixel.Event; }; - document: { - createElement: (tagName: string) => { - src: string; - async: boolean; - setAttribute: (name: string, value: string) => void; - removeAttribute: (name: string) => void; - }; - head: { appendChild: (node: unknown) => void }; - }; } export type Types = CoreDestination.Types; diff --git a/packages/web/destinations/piano/src/__tests__/conformance.test.ts b/packages/web/destinations/piano/src/__tests__/conformance.test.ts new file mode 100644 index 000000000..f34e482d4 --- /dev/null +++ b/packages/web/destinations/piano/src/__tests__/conformance.test.ts @@ -0,0 +1,21 @@ +import path from 'path'; +import { conformance } from '@walkeros/collector/dev'; +import destinationPiano from '..'; +import { examples, schemas } from '../dev'; + +describe('piano conformance', () => { + beforeEach(() => { + jest.useRealTimers(); + }); + + it('conforms', async () => { + await conformance('destination', destinationPiano, { + profile: 'vendor-global', + examples: examples.step, + schema: schemas.settings, + settings: { site: 123456789, collectDomain: 'https://example.pa-cd.com' }, + env: { push: examples.env.push, simulation: examples.env.simulation }, + root: path.join(__dirname, '..', '..'), + }); + }); +}); diff --git a/packages/web/destinations/piano/src/__tests__/script.test.ts b/packages/web/destinations/piano/src/__tests__/script.test.ts new file mode 100644 index 000000000..d26cc2512 --- /dev/null +++ b/packages/web/destinations/piano/src/__tests__/script.test.ts @@ -0,0 +1,53 @@ +import { startFlow } from '@walkeros/collector'; +import { getEvent } from '@walkeros/core'; +import destinationPiano from '..'; +import type { Env } from '../types'; + +const SCRIPT_SRC = 'https://tag.aticdn.net/piano-analytics.js'; + +/** The script `loadVendor` added, once the first push initialized the destination. */ +async function injected(src: string): Promise { + for (let tick = 0; tick < 50; tick++) { + const script = document.querySelector(`script[src="${src}"]`); + if (script) return script; + await new Promise((resolve) => setTimeout(resolve, 1)); + } + return null; +} + +describe('piano script', () => { + beforeEach(() => { + jest.useRealTimers(); + }); + + test('a script that failed to load fails the pushes with vendor-not-loaded', async () => { + const sendEvent = jest.fn(); + const env: Env = { + window: { + pa: { setConfigurations: jest.fn(), sendEvent, sendEvents: jest.fn() }, + }, + }; + const { elb } = await startFlow({ + destinations: { + piano: { + code: { ...destinationPiano, env }, + config: { + loadScript: true, + timeout: 1000, + settings: { site: 1, collectDomain: 'https://example.pa-cd.com' }, + }, + }, + }, + }); + + const pushed = elb(getEvent()); + (await injected(SCRIPT_SRC))?.dispatchEvent(new Event('error')); + const result = await pushed; + + expect(result.ok).toBe(false); + expect(result.failed?.piano?.reason).toMatchObject({ + code: 'vendor-not-loaded', + }); + expect(sendEvent).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/web/destinations/piano/src/__tests__/stepExamples.test.ts b/packages/web/destinations/piano/src/__tests__/stepExamples.test.ts index c271cf465..e7ed2fc85 100644 --- a/packages/web/destinations/piano/src/__tests__/stepExamples.test.ts +++ b/packages/web/destinations/piano/src/__tests__/stepExamples.test.ts @@ -49,8 +49,9 @@ const initOut: Effect[] = [...(initExample.out ?? [])].map((effect) => [ ]); describe('piano destination — step examples', () => { + // Non-public examples (silent, consent) run in the conformance test. const stepEntries = Object.entries(examples.step).filter( - ([name]) => name !== 'init', + ([name, example]) => name !== 'init' && example.public !== false, ); it.each(stepEntries)('%s', async (name, example) => { diff --git a/packages/web/destinations/piano/src/examples/step.ts b/packages/web/destinations/piano/src/examples/step.ts index 3203b0db0..0dd4a8c0f 100644 --- a/packages/web/destinations/piano/src/examples/step.ts +++ b/packages/web/destinations/piano/src/examples/step.ts @@ -100,3 +100,25 @@ export const customEvent: Flow.StepExample = { ], ], }; + +/** A silent rule sends no Piano event. */ +export const silentRule: Flow.StepExample = { + public: false, + description: 'A rule with silent: true sends no Piano event.', + in: getEvent('page view', { timestamp: 1700000203 }), + mapping: { silent: true }, + out: [], +}; + +/** + * Piano sends only on sendEvent, which the collector stops for a revoked + * consent; the destination has no consent call to make. + */ +export const consentRevoke: Flow.StepExample = { + public: false, + description: + 'Piano Analytics sends only when the destination calls sendEvent, which walkerOS stops for a consent-gated destination after a revocation, so no consent call is needed.', + command: 'consent', + in: { analytics: false }, + out: [], +}; diff --git a/packages/web/destinations/piano/src/index.test.ts b/packages/web/destinations/piano/src/index.test.ts index c9237631c..08ba4aa1d 100644 --- a/packages/web/destinations/piano/src/index.test.ts +++ b/packages/web/destinations/piano/src/index.test.ts @@ -1,24 +1,38 @@ import type { Config, Env } from './types'; -import type { Destination } from '@walkeros/core'; import { startFlow } from '@walkeros/collector'; import { getEvent } from '@walkeros/core'; import destinationPiano from '.'; const SCRIPT_SRC = 'https://tag.aticdn.net/piano-analytics.js'; +/** The script `loadVendor` added, once the first push initialized the destination. */ +async function injected(src: string): Promise { + for (let tick = 0; tick < 50; tick++) { + const script = document.querySelector(`script[src="${src}"]`); + if (script) return script; + await new Promise((resolve) => setTimeout(resolve, 1)); + } + return null; +} + describe('destination piano', () => { const site = 123456789; const collectDomain = 'https://example.pa-cd.com'; beforeEach(() => { jest.restoreAllMocks(); + // The script deadline and the push timeout run on real timers. + jest.useRealTimers(); }); async function register(env: Env, config: Config = {}) { - const { elb } = await startFlow(); - await elb('walker destination', { - code: { ...destinationPiano, env }, - config: { settings: { site, collectDomain }, ...config }, + const { elb } = await startFlow({ + destinations: { + piano: { + code: { ...destinationPiano, env }, + config: { settings: { site, collectDomain }, ...config }, + }, + }, }); return elb; } @@ -59,97 +73,38 @@ describe('destination piano', () => { }); }); - test('tolerates a missing SDK', async () => { + test('reports a missing SDK as vendor-not-ready', async () => { const env: Env = { window: {} }; const elb = await register(env); - await expect(elb(getEvent())).resolves.toBeDefined(); - }); + const result = await elb(getEvent()); - test('loadScript injects the Piano script', async () => { - const sendEvent = jest.fn(); - const env: Env = { - window: { - pa: { setConfigurations: jest.fn(), sendEvent, sendEvents: jest.fn() }, - }, - }; - - const script = document.createElement('script'); - const createElement = jest - .spyOn(document, 'createElement') - .mockReturnValue(script); - const appendChild = jest - .spyOn(document.head, 'appendChild') - .mockImplementation((node) => node); - - const elb = await register(env, { loadScript: true }); - await elb(getEvent()); - - expect(createElement).toHaveBeenCalledWith('script'); - expect(script.src).toBe(SCRIPT_SRC); - expect(appendChild).toHaveBeenCalledWith(script); + expect(result.ok).toBe(false); + expect(result.failed?.piano?.reason).toMatchObject({ + code: 'vendor-not-ready', + retryable: true, + }); }); - test('loadScript defers configuration until the script loads', async () => { + test('loadScript waits for the script, then configures and sends', async () => { const setConfigurations = jest.fn(); + const sendEvent = jest.fn(); const env: Env = { window: { - pa: { setConfigurations, sendEvent: jest.fn(), sendEvents: jest.fn() }, + pa: { setConfigurations, sendEvent, sendEvents: jest.fn() }, }, }; - const script = document.createElement('script'); - jest.spyOn(document, 'createElement').mockReturnValue(script); - jest.spyOn(document.head, 'appendChild').mockImplementation((node) => node); - const elb = await register(env, { loadScript: true }); - await elb(getEvent()); + const pushed = elb(getEvent()); - // The SDK loads asynchronously, so nothing is configured yet. - expect(setConfigurations).not.toHaveBeenCalled(); + const script = await injected(SCRIPT_SRC); + expect(script?.hasAttribute('data-walkeros-vendor')).toBe(true); + expect(sendEvent).not.toHaveBeenCalled(); - // Once the script fires onload, configuration runs. - script.onload?.(new Event('load')); + script?.dispatchEvent(new Event('load')); + expect((await pushed).ok).toBe(true); expect(setConfigurations).toHaveBeenCalledWith({ site, collectDomain }); - }); - - test('warns when the Piano script fails to load', () => { - const setConfigurations = jest.fn(); - const env: Env = { - window: { - pa: { setConfigurations, sendEvent: jest.fn(), sendEvents: jest.fn() }, - }, - }; - - const script = document.createElement('script'); - jest.spyOn(document, 'createElement').mockReturnValue(script); - jest.spyOn(document.head, 'appendChild').mockImplementation((node) => node); - - const warn = jest.fn(); - const logger = { - error: jest.fn(), - warn, - info: jest.fn(), - debug: jest.fn(), - throw: jest.fn(), - } as unknown as Destination.Context['logger']; - - destinationPiano.init?.({ - id: 'piano', - config: { loadScript: true, settings: { site, collectDomain } }, - env, - logger, - collector: {} as Destination.Context['collector'], - }); - - // A blocked or failed script never fires onload, so nothing is configured. - expect(setConfigurations).not.toHaveBeenCalled(); - - // onerror surfaces a single warning instead of failing silently. - script.onerror?.(new Event('error')); - expect(warn).toHaveBeenCalledWith( - 'Piano Analytics script failed to load, destination not configured', - ); - expect(setConfigurations).not.toHaveBeenCalled(); + expect(sendEvent).toHaveBeenCalledTimes(1); }); }); diff --git a/packages/web/destinations/piano/src/index.ts b/packages/web/destinations/piano/src/index.ts index d4735ae45..8c3e06de0 100644 --- a/packages/web/destinations/piano/src/index.ts +++ b/packages/web/destinations/piano/src/index.ts @@ -1,52 +1,69 @@ -import type { Env, Destination, PianoAnalytics } from './types'; -import { isDefined, isObject } from '@walkeros/core'; +import type { Config, Env, Destination, PianoAnalytics } from './types'; +import { isObject, StepError } from '@walkeros/core'; +import { loadVendor } from '@walkeros/web-core'; // Types export * as DestinationPiano from './types'; const SCRIPT_SRC = 'https://tag.aticdn.net/piano-analytics.js'; +/** + * The Piano SDK. It has no stub that replays calls, so a push waits for the + * script `loadVendor` injected (`loadScript`), or finds `pa` on the page. + */ +function vendor(config: Config, env?: Env) { + return loadVendor( + config.loadScript ? SCRIPT_SRC : undefined, + () => resolvePa(env), + { timeout: config.timeout }, + ); +} + +// SDK instances already configured, so setConfigurations runs once each. +const configured = new WeakSet(); + export const destinationPiano: Destination = { type: 'piano', config: {}, - init({ config, env, logger }) { - const settings = config.settings || {}; + init({ config, env }) { + const { site, collectDomain } = config.settings || {}; + const missing = [ + site === undefined ? 'site' : undefined, + !collectDomain ? 'collectDomain' : undefined, + ].filter((key) => key !== undefined); + if (missing.length) + throw new StepError( + `Piano Analytics needs settings ${missing.join(' and ')}`, + { code: 'config' }, + ); + + vendor(config, env); + }, - const configure = () => { - const pa = resolvePa(env); - if (pa && isDefined(settings.site) && isDefined(settings.collectDomain)) { - pa.setConfigurations({ + async push(event, { config, data, env, rule }) { + const pa = await vendor(config, env).ready(); + const settings = config.settings; + + try { + if (settings && !configured.has(pa)) { + await pa.setConfigurations({ site: settings.site, collectDomain: settings.collectDomain, ...(isObject(settings.options) ? settings.options : {}), }); + configured.add(pa); } - }; - // When we inject the SDK it loads asynchronously, so `window.pa` is not - // available in the same tick. Defer configuration until `onload` fires, - // and warn if the script never loads (e.g. blocked or network failure) so - // the destination does not silently send unconfigured events. - if (config.loadScript) { - addScript(SCRIPT_SRC, configure, () => - logger.warn( - 'Piano Analytics script failed to load, destination not configured', - ), - ); - } else { - configure(); + if (!rule?.silent) + await pa.sendEvent(event.name, isObject(data) ? data : {}); + } catch (error) { + throw new StepError('Piano Analytics call failed', { + code: 'transport', + cause: error, + }); } - - return config; - }, - - push(event, { data, env }) { - const pa = resolvePa(env); - if (!pa) return; - - pa.sendEvent(event.name, isObject(data) ? data : {}); }, }; @@ -54,23 +71,10 @@ export const destinationPiano: Destination = { * Resolve the Piano `pa` SDK without casts: prefer the injected env (tests, * simulation), fall back to the browser global declared on `Window`. */ -function resolvePa(env: Env): PianoAnalytics | undefined { +function resolvePa(env?: Env): PianoAnalytics | undefined { return ( - env.window?.pa ?? (typeof window !== 'undefined' ? window.pa : undefined) + env?.window?.pa ?? (typeof window !== 'undefined' ? window.pa : undefined) ); } -function addScript( - src = SCRIPT_SRC, - onReady?: () => void, - onError?: () => void, -) { - if (typeof document === 'undefined') return; - const script = document.createElement('script'); - script.src = src; - if (onReady) script.onload = () => onReady(); - if (onError) script.onerror = () => onError(); - document.head.appendChild(script); -} - export default destinationPiano; diff --git a/packages/web/destinations/piano/src/schemas/settings.ts b/packages/web/destinations/piano/src/schemas/settings.ts index 7600f01cf..e7d218a6b 100644 --- a/packages/web/destinations/piano/src/schemas/settings.ts +++ b/packages/web/destinations/piano/src/schemas/settings.ts @@ -1,18 +1,20 @@ import { z } from '@walkeros/core/dev'; -export const SettingsSchema = z.object({ - site: z - .number() - .describe( - 'Piano Analytics site id (numeric), from your collection settings', - ), - collectDomain: z - .string() - .describe('Collection domain endpoint, like https://xxxxxxx.pa-cd.com'), - options: z - .record(z.string(), z.unknown()) - .describe('Additional Piano setConfigurations options merged on init') - .optional(), -}); +export const SettingsSchema = z + .object({ + site: z + .number() + .describe( + 'Piano Analytics site id (numeric), from your collection settings', + ), + collectDomain: z + .string() + .describe('Collection domain endpoint, like https://xxxxxxx.pa-cd.com'), + options: z + .record(z.string(), z.unknown()) + .describe('Additional Piano setConfigurations options merged on init') + .optional(), + }) + .strict(); export type Settings = z.infer; diff --git a/packages/web/destinations/piwikpro/src/__tests__/conformance.test.ts b/packages/web/destinations/piwikpro/src/__tests__/conformance.test.ts new file mode 100644 index 000000000..391d4959f --- /dev/null +++ b/packages/web/destinations/piwikpro/src/__tests__/conformance.test.ts @@ -0,0 +1,24 @@ +import path from 'path'; +import { conformance } from '@walkeros/collector/dev'; +import { destinationPiwikPro } from '..'; +import { examples, schemas } from '../dev'; + +describe('piwikpro conformance', () => { + beforeEach(() => { + jest.useRealTimers(); + }); + + it('conforms', async () => { + await conformance('destination', destinationPiwikPro, { + profile: 'vendor-global', + examples: examples.step, + schema: schemas.settings, + settings: { + appId: 'XXX-XXX-XXX-XXX-XXX', + url: 'https://your_account_name.piwik.pro/', + }, + env: { push: examples.env.push, simulation: examples.env.simulation }, + root: path.join(__dirname, '..', '..'), + }); + }); +}); diff --git a/packages/web/destinations/piwikpro/src/__tests__/skip.test.ts b/packages/web/destinations/piwikpro/src/__tests__/skip.test.ts deleted file mode 100644 index 0ebc76738..000000000 --- a/packages/web/destinations/piwikpro/src/__tests__/skip.test.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { createMockLogger } from '@walkeros/core'; -import { logSkip, SKIP_KEY_CAP } from '../skip'; - -describe('logSkip', () => { - test('warns once per key, then logs at debug', () => { - const logger = createMockLogger(); - - logSkip(logger, 'dest|foo bar|unmapped', 'skipped', { reason: 'unmapped' }); - logSkip(logger, 'dest|foo bar|unmapped', 'skipped', { reason: 'unmapped' }); - - expect(logger.warn).toHaveBeenCalledTimes(1); - expect(logger.warn).toHaveBeenCalledWith('skipped', { reason: 'unmapped' }); - expect(logger.debug).toHaveBeenCalledTimes(1); - }); - - test('beyond the cap every new key logs at debug', () => { - const logger = createMockLogger(); - for (let i = 0; i < SKIP_KEY_CAP; i++) - logSkip(logger, `dest|event ${i}|unmapped`, 'skipped', {}); - logger.warn.mockClear(); - logger.debug.mockClear(); - - logSkip(logger, 'dest|one more|unmapped', 'skipped', {}); - - expect(logger.warn).not.toHaveBeenCalled(); - expect(logger.debug).toHaveBeenCalledTimes(1); - }); -}); diff --git a/packages/web/destinations/piwikpro/src/__tests__/stepExamples.test.ts b/packages/web/destinations/piwikpro/src/__tests__/stepExamples.test.ts index aec6a4ad2..92ec3e83b 100644 --- a/packages/web/destinations/piwikpro/src/__tests__/stepExamples.test.ts +++ b/packages/web/destinations/piwikpro/src/__tests__/stepExamples.test.ts @@ -73,8 +73,10 @@ function makeMockPaq(): { } describe('piwikpro web destination -- step examples', () => { + // Consent and failure examples run in the conformance test. const stepEntries = Object.entries(examples.step).filter( - ([name]) => name !== 'init', + ([name, example]) => + name !== 'init' && !example.command && !example.outcome, ); it('pass no currency argument, the tracker has none (W6)', () => { diff --git a/packages/web/destinations/piwikpro/src/examples/env.ts b/packages/web/destinations/piwikpro/src/examples/env.ts index 169558254..a3ec72ea6 100644 --- a/packages/web/destinations/piwikpro/src/examples/env.ts +++ b/packages/web/destinations/piwikpro/src/examples/env.ts @@ -11,30 +11,12 @@ export const init: Env | undefined = { window: { // _paq absent: init() is expected to install the queue. }, - document: { - createElement: () => ({ - type: '', - src: '', - async: false, - defer: false, - }), - head: { appendChild: () => {} }, - }, }; export const push: Env = { window: { _paq: [], }, - document: { - createElement: () => ({ - type: '', - src: '', - async: false, - defer: false, - }), - head: { appendChild: () => {} }, - }, }; /** diff --git a/packages/web/destinations/piwikpro/src/examples/step.ts b/packages/web/destinations/piwikpro/src/examples/step.ts index 77f32ad28..df3e253d3 100644 --- a/packages/web/destinations/piwikpro/src/examples/step.ts +++ b/packages/web/destinations/piwikpro/src/examples/step.ts @@ -480,3 +480,40 @@ export const customDimensionsPageView: Flow.StepExample = { ['deleteCustomDimension', 1], ], }; + +/** A silent rule without a goal sends nothing. */ +export const silentRule: Flow.StepExample = { + public: false, + description: + 'A rule with silent: true and no goal sends no command, no identification change either.', + in: getEvent('page view', { timestamp: 1700000400 }), + mapping: { silent: true }, + out: [], +}; + +/** + * An event no rule maps: Piwik PRO has no method for it, so the push fails + * instead of counting as delivered. + */ +export const unmappedEvent: Flow.StepExample = { + public: false, + description: + 'An event without a rule name or a goal, and not a page view, has no Piwik PRO method: the push fails with missing-input.', + in: getEvent('product view', { timestamp: 1700000401 }), + out: [], + outcome: { status: 'failed', code: 'missing-input', retryable: false }, +}; + +/** + * A revocation of the consent the destination requires turns off the link + * tracking it enabled: config.consent { marketing: true }, granted first, + * then revoked. + */ +export const consentRevoke: Flow.StepExample = { + public: false, + description: + 'For a destination with config.consent { marketing: true }, revoking marketing pushes disableLinkTracking, so the tracker stops sending link clicks on its own.', + command: 'consent', + in: { marketing: false }, + out: [['window._paq.push', ['disableLinkTracking']]], +}; diff --git a/packages/web/destinations/piwikpro/src/index.test.ts b/packages/web/destinations/piwikpro/src/index.test.ts index d2a990f29..b39b3c29d 100644 --- a/packages/web/destinations/piwikpro/src/index.test.ts +++ b/packages/web/destinations/piwikpro/src/index.test.ts @@ -1,4 +1,4 @@ -import type { Logger, WalkerOS } from '@walkeros/core'; +import type { Elb, Logger, WalkerOS } from '@walkeros/core'; import type { DestinationPiwikPro } from '.'; import { startFlow } from '@walkeros/collector'; import { @@ -23,13 +23,6 @@ interface Harness { logs: LogEntry[]; } -interface ScriptElement { - type: string; - src: string; - async?: boolean; - defer?: boolean; -} - const appId = 'XXX-XXX-XXX-XXX-XXX'; const url = 'https://your_account_name.piwik.pro/'; @@ -44,10 +37,6 @@ const toIdentified = [['enableCookies'], ['deanonymizeUser']]; function createEnv(paq?: unknown[]): DestinationPiwikPro.Env { return { window: paq ? { _paq: paq } : {}, - document: { - createElement: () => ({ type: '', src: '', async: false, defer: false }), - head: { appendChild: () => {} }, - }, }; } @@ -89,18 +78,8 @@ async function runInit( if (!init) throw new Error('init missing'); const paq: unknown[] = []; - const scripts: ScriptElement[] = []; - const env: DestinationPiwikPro.Env = { - window: { _paq: paq }, - document: { - createElement: () => { - const script: ScriptElement = { type: '', src: '' }; - scripts.push(script); - return script; - }, - head: { appendChild: () => {} }, - }, - }; + const env: DestinationPiwikPro.Env = { window: { _paq: paq } }; + const before = vendorScripts(); const { collector, elb } = await startFlow(); if (consent) await elb('walker consent', consent); @@ -112,7 +91,24 @@ async function runInit( collector, }); - return { paq, scripts }; + // loadVendor adds a script once per src and page. + return { + paq, + scripts: vendorScripts().filter((src) => !before.includes(src)), + }; +} + +/** The src of every script `loadVendor` added. */ +function vendorScripts(): string[] { + return Array.from( + document.querySelectorAll('script[data-walkeros-vendor]'), + (script) => script.getAttribute('src') ?? '', + ); +} + +/** The reason the one destination of a push failed with. */ +function failure(result: Elb.PushResult) { + return Object.values(result.failed ?? {})[0]?.reason; } function warnings(logs: LogEntry[], text: string): LogEntry[] { @@ -157,20 +153,17 @@ describe('Destination PiwikPro', () => { expect(paq).toEqual([['trackPageView', '/docs/']]); }); - test('unmapped event pushes nothing and warns once (W2)', async () => { + test('unmapped event pushes nothing and fails with missing-input (W2)', async () => { const { elb, paq, logs } = await setup(); - await elb('foo bar'); - await elb('foo bar'); + const result = await elb('foo bar'); - const skipped = 'Event "foo bar" skipped'; expect(paq).toEqual([]); - expect(warnings(logs, skipped)).toHaveLength(1); - expect( - logs.filter( - (log) => log.level === Level.DEBUG && log.message.includes(skipped), - ), - ).toHaveLength(1); + expect(failure(result)).toMatchObject({ + code: 'missing-input', + retryable: false, + }); + expect(warnings(logs, 'foo bar')).toEqual([]); }); test('an implicit rule with a goal sends only the goal', async () => { @@ -199,7 +192,7 @@ describe('Destination PiwikPro', () => { }); test('an implicit rule without a goal is still unmapped', async () => { - const { elb, paq, logs } = await setup({ + const { elb, paq } = await setup({ mapping: { order: { complete: { settings: { customDimensions: { '1': 'data.id' } } }, @@ -207,10 +200,10 @@ describe('Destination PiwikPro', () => { }, }); - await elb(getEvent('order complete')); + const result = await elb(getEvent('order complete')); expect(paq).toEqual([]); - expect(warnings(logs, 'Event "order complete" skipped')).toHaveLength(1); + expect(failure(result)).toMatchObject({ code: 'missing-input' }); }); test('explicit rule name passes through as a state setter', async () => { @@ -293,25 +286,48 @@ describe('Destination PiwikPro', () => { expect(paq).toEqual([...group('/a/'), ...group('/b/')]); }); - test('silent rule without a goal sends nothing and warns once', async () => { + test('silent rule without a goal sends nothing', async () => { const { elb, paq, logs } = await setup({ mapping: { page: { view: { silent: true } } }, }); - await elb(pageView); - await elb(pageView); + expect((await elb(pageView)).ok).toBe(true); expect(paq).toEqual([]); - expect(warnings(logs, 'Event "page view" skipped')).toHaveLength(1); + expect(warnings(logs, 'page view')).toEqual([]); }); - test('missing _paq warns and pushes nothing (W13)', async () => { - const { elb, paq, logs } = await setup({ init: true }, { queue: false }); + test('missing _paq fails with vendor-not-ready and pushes nothing (W13)', async () => { + const { elb, paq } = await setup({ init: true }, { queue: false }); - await elb(pageView); + const result = await elb(pageView); expect(paq).toEqual([]); - expect(warnings(logs, '_paq')).toHaveLength(1); + expect(failure(result)).toMatchObject({ + code: 'vendor-not-ready', + retryable: true, + }); + }); + + test('a consent revocation turns off link tracking for a consent-gated destination, the next hit turns it on', async () => { + const { elb, paq } = await setup({ + consent: { marketing: true }, + settings: { linkTracking: true }, + }); + + await elb('walker consent', { marketing: true }); + await elb(pageView); + await elb('walker consent', { marketing: false }); + await elb('walker consent', { marketing: true }); + await elb(pageView); + + expect(paq).toEqual([ + ['trackPageView', title], + ['enableLinkTracking'], + ['disableLinkTracking'], + ['trackPageView', title], + ['enableLinkTracking'], + ]); }); }); @@ -514,9 +530,9 @@ describe('Destination PiwikPro', () => { test.each([ ['appId', { url }], ['url', { appId }], - ])('throws for missing %s with loadScript', async (_name, settings) => { + ])('throws for missing %s with loadScript', async (name, settings) => { await expect(runInit({ loadScript: true, settings })).rejects.toThrow( - 'missing', + `needs settings ${name}`, ); }); @@ -536,9 +552,7 @@ describe('Destination PiwikPro', () => { settings: { appId, url: 'https://acc.piwik.pro' }, }); - expect(scripts.map((script) => script.src)).toEqual([ - 'https://acc.piwik.pro/ppms.js', - ]); + expect(scripts).toEqual(['https://acc.piwik.pro/ppms.js']); expect(paq).toEqual([ ['setTrackerUrl', 'https://acc.piwik.pro/ppms.php'], ['setSiteId', appId], diff --git a/packages/web/destinations/piwikpro/src/index.ts b/packages/web/destinations/piwikpro/src/index.ts index 9f1b179ed..ea8571a82 100644 --- a/packages/web/destinations/piwikpro/src/index.ts +++ b/packages/web/destinations/piwikpro/src/index.ts @@ -1,6 +1,12 @@ import type { Mapping, Destination, Env } from './types'; -import { getMappingValue, isArray } from '@walkeros/core'; -import { getEnv } from '@walkeros/web-core'; +import { + getGrantedConsent, + getMappingValue, + isArray, + isObject, + StepError, +} from '@walkeros/core'; +import { getEnv, loadVendor } from '@walkeros/web-core'; import type { DimensionValues } from './dimensions'; import type { MethodName } from './methods'; import { @@ -10,7 +16,6 @@ import { } from './dimensions'; import { isIdentified, transition } from './identified'; import { DIMENSIONS_ARG, isMethodName } from './methods'; -import { logSkip } from './skip'; // Types export * as DestinationPiwikPro from './types'; @@ -22,58 +27,98 @@ export * as DestinationPiwikPro from './types'; // wrapped env. const linkTracked = new WeakSet(); +/** + * Pushes commands onto the queue in one synchronous block, so another + * event's commands never interleave, then waits for what each push returns. + */ +async function pushAll( + queue: unknown[], + commands: unknown[][], + code: string, + message: string, +): Promise { + try { + await Promise.all(commands.map((command) => queue.push(command))); + } catch (error) { + throw new StepError(message, { code, cause: error }); + } +} + +/** + * ppms.js, injected by `loadVendor` with `loadScript`. `_paq` replays its own + * queue, so commands are pushed synchronously and only a failed script is + * reported. + */ +function vendor(url: string | undefined, env?: Env, timeout?: number) { + return loadVendor( + url ? `${normalizeUrl(url)}ppms.js` : undefined, + () => getEnv(env).window._paq, + { timeout }, + ); +} + export const destinationPiwikPro: Destination = { type: 'piwikpro', config: {}, - init({ config, env, logger, collector }) { + async init({ config, env, collector }) { const { window } = getEnv(env); const { settings = {}, loadScript } = config; const { appId, url } = settings; - - // Set up the Piwik Pro interface _paq - window._paq = window._paq || []; - const queue = window._paq; - const paq = (command: unknown[]) => { - queue.push(command); - }; + const commands: unknown[][] = []; if (loadScript) { // Required only to load and address the tracker - if (!appId) return logger.throw('Config settings appId missing'); - if (!url) return logger.throw('Config settings url missing'); + if (!appId || !url) + throw new StepError( + `Piwik PRO with loadScript needs settings ${[ + ...(appId ? [] : ['appId']), + ...(url ? [] : ['url']), + ].join(' and ')}`, + { code: 'config' }, + ); const baseUrl = normalizeUrl(url); - // Load the JavaScript Tracking Client - addScript(baseUrl, env); - // Register the tracker url only with script loading - paq(['setTrackerUrl', baseUrl + 'ppms.php']); + commands.push(['setTrackerUrl', baseUrl + 'ppms.php']); // Register app id - paq(['setSiteId', appId]); + commands.push(['setSiteId', appId]); } // Start anonymous, before the tracker loads, unless identification is // granted already; a consent object resolves per event from then on. if (!isIdentified(settings.identified, collector)) - transition(collector, false).forEach(paq); + commands.push(...transition(collector, false)); + + // Set up the Piwik Pro interface _paq and load the tracking client + window._paq = window._paq || []; + if (loadScript) vendor(url, env, config.timeout); + await pushAll( + window._paq, + commands, + 'vendor-not-ready', + 'Piwik PRO did not take its setup commands', + ); }, - async push(event, { config, rule = {}, data, env, collector, logger, id }) { - const { window } = getEnv(env); - const queue = window._paq; - if (!queue) { - logger.warn('Piwik PRO _paq queue missing, init() not run'); - return; - } + async push(event, { config, rule = {}, data, env, collector }) { + const settings = config.settings || {}; + if (config.loadScript) vendor(settings.url, env, config.timeout).check(); + const queue = getEnv(env).window._paq; + if (!queue) + throw new StepError('Piwik PRO _paq queue missing: init did not run', { + code: 'vendor-not-ready', + }); + + // This event's commands, pushed in one block at the end. + const commands: unknown[][] = []; const paq = (command: unknown[]) => { - queue.push(command); + commands.push(command); }; - const settings = config.settings || {}; const linkTracking = settings.linkTracking !== false; // A tracking hit; the first one is followed by enableLinkTracking, as in @@ -98,15 +143,11 @@ export const destinationPiwikPro: Destination = { : implicitPageView ? 'trackPageView' : undefined; - if (method === undefined && goalId === undefined) { - logSkip( - logger, - `${id}|${event.name}|unmapped`, - `Event "${event.name}" skipped: no rule name, no goal and not a page view`, - { event: event.name, reason: 'unmapped' }, + if (method === undefined && goalId === undefined) + throw new StepError( + `Piwik PRO has no method for "${event.name}": map it with a rule name or a goal, or ignore it`, + { code: 'missing-input' }, ); - return; - } // The method hit, unless silent or goal-only const hitMethod = rule.silent === true ? undefined : method; @@ -115,15 +156,8 @@ export const destinationPiwikPro: Destination = { ? hitMethod : undefined; - if (hitMethod === undefined && goalId === undefined) { - logSkip( - logger, - `${id}|${event.name}|silent`, - `Event "${event.name}" skipped: silent rule without a goal`, - { event: event.name, reason: 'silent' }, - ); - return; - } + // A silent rule without a goal sends nothing, no transition either. + const sends = hitMethod !== undefined || goalId !== undefined; // Resolve everything first, so this event's commands are pushed in one // synchronous block below and never interleave with another event's. @@ -149,10 +183,11 @@ export const destinationPiwikPro: Destination = { : await getMappingValue(event, mapping.goalValue, { collector }); // Match the tracker's identified state before anything else is pushed - transition( - collector, - isIdentified(settings.identified, collector, event), - ).forEach(paq); + if (sends) + transition( + collector, + isIdentified(settings.identified, collector, event), + ).forEach(paq); if (hitMethod !== undefined) { if (portable === undefined) { @@ -195,6 +230,34 @@ export const destinationPiwikPro: Destination = { ), ]); } + + if (commands.length) + await pushAll( + queue, + commands, + 'transport', + 'Piwik PRO did not take the event commands', + ); + }, + + // The destination's required consent decides, as it gates the pushes: a + // revocation turns off the link tracking the destination enabled, which + // keeps sending on its own; the next hit after a grant turns it on again. + async on(type, { config, data, env, collector }) { + const required = config.consent; + if (type !== 'consent' || !required || !isObject(data)) return; + if (!Object.keys(required).some((key) => key in data)) return; + if (getGrantedConsent(required, collector.consent) !== false) return; + const queue = getEnv(env).window._paq; + if (!queue) return; + + linkTracked.delete(collector); + await pushAll( + queue, + [['disableLinkTracking']], + 'consent-not-applied', + 'Piwik PRO did not take the consent update', + ); }, }; @@ -239,14 +302,4 @@ function trimTrailing(args: unknown[]): unknown[] { return args.slice(0, end); } -function addScript(url: string, env?: Env) { - const { document } = getEnv(env); - const script = document.createElement('script'); - script.type = 'text/javascript'; - script.src = url + 'ppms.js'; - script.async = true; - script.defer = true; - document.head.appendChild(script); -} - export default destinationPiwikPro; diff --git a/packages/web/destinations/piwikpro/src/schemas/settings.ts b/packages/web/destinations/piwikpro/src/schemas/settings.ts index 2f85ded61..621f2f8ff 100644 --- a/packages/web/destinations/piwikpro/src/schemas/settings.ts +++ b/packages/web/destinations/piwikpro/src/schemas/settings.ts @@ -7,34 +7,36 @@ export const CustomDimensionsSchema = z.record( ValueSchema, ); -export const SettingsSchema = z.object({ - appId: z - .string() - .min(1) - .describe( - 'ID of the Piwik PRO site (like XXX-XXX-XXX-XXX-XXX). Required when loadScript is true', - ) - .optional(), - url: z - .string() - .url() - .describe( - 'URL of your Piwik PRO account, normalized to one trailing slash (like https://your_account_name.piwik.pro/). Required when loadScript is true', - ) - .optional(), - linkTracking: z - .boolean() - .default(true) - .describe( - 'Automatic outlink and download tracking, enabled right after the first hit this destination sends. Default: true', - ), - identified: z - .union([z.boolean(), z.record(z.string(), z.boolean())]) - .describe( - 'Identified or anonymous tracking. true (default) identifies every hit, false switches the tracker to anonymous mode without cookies (disableCookies, deleteCookies, setUserIsAnonymous) at init. A consent object like { marketing: true } identifies only while that consent is granted and calls enableCookies and deanonymizeUser once it is. Do not repeat these states in config.consent: a destination needs only one of its config.consent states granted, so next to another state the repeated one lets events through on its own, and as the only state it queues every event until granted, so no anonymous hit is sent.', - ) - .optional(), - customDimensions: CustomDimensionsSchema.describe( - 'Custom dimensions applied to every hit, keyed by bare dimension id (like { "1": "data.size" }). Rule-level customDimensions win per key.', - ).optional(), -}); +export const SettingsSchema = z + .object({ + appId: z + .string() + .min(1) + .describe( + 'ID of the Piwik PRO site (like XXX-XXX-XXX-XXX-XXX). Required when loadScript is true', + ) + .optional(), + url: z + .string() + .url() + .describe( + 'URL of your Piwik PRO account, normalized to one trailing slash (like https://your_account_name.piwik.pro/). Required when loadScript is true', + ) + .optional(), + linkTracking: z + .boolean() + .default(true) + .describe( + 'Automatic outlink and download tracking, enabled right after the first hit this destination sends. Default: true', + ), + identified: z + .union([z.boolean(), z.record(z.string(), z.boolean())]) + .describe( + 'Identified or anonymous tracking. true (default) identifies every hit, false switches the tracker to anonymous mode without cookies (disableCookies, deleteCookies, setUserIsAnonymous) at init. A consent object like { marketing: true } identifies only while that consent is granted and calls enableCookies and deanonymizeUser once it is. Do not repeat these states in config.consent: a destination needs only one of its config.consent states granted, so next to another state the repeated one lets events through on its own, and as the only state it queues every event until granted, so no anonymous hit is sent.', + ) + .optional(), + customDimensions: CustomDimensionsSchema.describe( + 'Custom dimensions applied to every hit, keyed by bare dimension id (like { "1": "data.size" }). Rule-level customDimensions win per key.', + ).optional(), + }) + .strict(); diff --git a/packages/web/destinations/piwikpro/src/skip.ts b/packages/web/destinations/piwikpro/src/skip.ts deleted file mode 100644 index 4508f2c59..000000000 --- a/packages/web/destinations/piwikpro/src/skip.ts +++ /dev/null @@ -1,23 +0,0 @@ -import type { Logger } from '@walkeros/core'; - -/** Most distinct skip keys that get a warning; beyond it every skip logs at debug. */ -export const SKIP_KEY_CAP = 1000; - -// Module-level and bounded, so unbounded event names cannot grow memory. -const warned = new Set(); - -/** Warns once per key, then logs the same skip at debug. */ -export function logSkip( - logger: Logger.Instance, - key: string, - message: string, - meta: Record, -): void { - if (!warned.has(key) && warned.size < SKIP_KEY_CAP) { - warned.add(key); - logger.warn(message, meta); - return; - } - - logger.debug(message, meta); -} diff --git a/packages/web/destinations/piwikpro/src/types/index.ts b/packages/web/destinations/piwikpro/src/types/index.ts index 02317bef3..f37026cd3 100644 --- a/packages/web/destinations/piwikpro/src/types/index.ts +++ b/packages/web/destinations/piwikpro/src/types/index.ts @@ -43,17 +43,6 @@ export interface Env extends DestinationWeb.Env { // Optional: the SDK queue may be absent until `init` installs it. _paq?: Array; }; - document: { - createElement: (tagName: string) => { - type: string; - src: string; - async?: boolean; - defer?: boolean; - }; - head: { - appendChild: (node: unknown) => void; - }; - }; } export type Types = CoreDestination.Types; diff --git a/packages/web/destinations/segment/src/__tests__/conformance.test.ts b/packages/web/destinations/segment/src/__tests__/conformance.test.ts new file mode 100644 index 000000000..c41097e22 --- /dev/null +++ b/packages/web/destinations/segment/src/__tests__/conformance.test.ts @@ -0,0 +1,41 @@ +import path from 'path'; +import { conformance } from '@walkeros/collector/dev'; +import { destinationSegment } from '..'; +import { examples, schemas } from '../dev'; + +// Analytics.js init options are forwarded into load() as a whole. +const passThrough = [ + 'initialPageview', + 'disableClientPersistence', + 'disableAutoISOConversion', + 'retryQueue', + 'obfuscate', + 'integrations', +]; + +describe('segment conformance', () => { + beforeEach(() => { + jest.useRealTimers(); + }); + + it('conforms', async () => { + await conformance('destination', destinationSegment, { + profile: 'web-sdk', + examples: examples.step, + schema: schemas.settings, + settings: { + apiKey: 'test-project', + cdnURL: 'https://cdn.example.com', + initialPageview: false, + disableClientPersistence: true, + disableAutoISOConversion: false, + retryQueue: false, + obfuscate: false, + integrations: {}, + }, + env: { push: examples.env.push, simulation: examples.env.simulation }, + root: path.join(__dirname, '..', '..'), + passThrough, + }); + }); +}); diff --git a/packages/web/destinations/segment/src/__tests__/index.test.ts b/packages/web/destinations/segment/src/__tests__/index.test.ts new file mode 100644 index 000000000..261b3e491 --- /dev/null +++ b/packages/web/destinations/segment/src/__tests__/index.test.ts @@ -0,0 +1,45 @@ +import { startFlow } from '@walkeros/collector'; +import { getEvent } from '@walkeros/core'; +import { destinationSegment } from '..'; +import type { SegmentAnalytics, SegmentSDK } from '../types'; + +describe('segment destination', () => { + it('passes cdnURL in the load() settings, where Analytics.js reads it', async () => { + const loads: unknown[][] = []; + const noop = () => undefined; + const instance: SegmentAnalytics = { + track: noop, + identify: noop, + group: noop, + page: noop, + alias: noop, + reset: noop, + setAnonymousId: noop, + }; + const analytics: SegmentSDK = { + load: (...args) => { + loads.push(args); + return instance; + }, + }; + const { elb } = await startFlow({ + destinations: { + segment: { + code: { ...destinationSegment, env: { analytics } }, + config: { + settings: { apiKey: 'key', cdnURL: 'https://cdn.example.com' }, + }, + }, + }, + }); + + await elb(getEvent('page view')); + + expect(loads).toEqual([ + [ + { writeKey: 'key', cdnURL: 'https://cdn.example.com' }, + { initialPageview: false }, + ], + ]); + }); +}); diff --git a/packages/web/destinations/segment/src/__tests__/stepExamples.test.ts b/packages/web/destinations/segment/src/__tests__/stepExamples.test.ts index f1e457741..dca4ab9bf 100644 --- a/packages/web/destinations/segment/src/__tests__/stepExamples.test.ts +++ b/packages/web/destinations/segment/src/__tests__/stepExamples.test.ts @@ -97,8 +97,9 @@ function trimUndefined(call: CallRecord): CallRecord { } describe('segment destination — step examples', () => { + // Non-public examples (silent, failures) run in the conformance test. const stepEntries = Object.entries(examples.step).filter( - ([name]) => name !== 'init', + ([name, example]) => name !== 'init' && example.public !== false, ); it('init', async () => { diff --git a/packages/web/destinations/segment/src/examples/step.ts b/packages/web/destinations/segment/src/examples/step.ts index 7927a60f7..91597beb6 100644 --- a/packages/web/destinations/segment/src/examples/step.ts +++ b/packages/web/destinations/segment/src/examples/step.ts @@ -495,29 +495,6 @@ export const consentContextOmitted: SegmentStepExample = { out: [['analytics.track', 'product view', {}]], }; -/** - * Consent granted → deferred load fires for the first time. The - * destination was initialized with consent requirement; on the walker - * consent command it calls analytics.load(writeKey, initOptions). - * - * Uses the canonical StepExample.command='consent' pattern. The full - * call shape includes both the settings arg and the initOptions arg. - */ -export const consentGrantDeferredLoad: SegmentStepExample = { - title: 'Consent deferred load', - description: - 'With consent: { analytics: true } in the destination config, a walker consent grant triggers the deferred Segment analytics.load with the configured writeKey.', - command: 'consent', - in: { analytics: true } as WalkerOS.Consent, - out: [ - [ - 'analytics.load', - { writeKey: 'test-project' }, - { initialPageview: false }, - ], - ], -}; - /** * Consent revoked → no SDK call. The walkerOS consent gate handles * blocking subsequent events. The destination's on('consent') handler @@ -529,3 +506,29 @@ export const consentRevokeNoOp: SegmentStepExample = { in: { analytics: false } as WalkerOS.Consent, out: [], }; + +/** A silent rule without settings of its own calls nothing. */ +export const silentRule: SegmentStepExample = { + public: false, + description: 'A rule with silent: true and no settings calls nothing.', + in: getEvent('page view', { timestamp: 1700000140 }), + mapping: { silent: true }, + out: [], +}; + +/** + * The silent group rule of companyUpdateGroup on an update without data: no + * groupId resolves and the rule suppresses the track, so nothing is sent. + */ +export const companyUpdateWithoutData: SegmentStepExample = { + public: false, + description: + 'A silent rule whose group mapping finds no groupId sends nothing, so the push fails with missing-input instead of counting as delivered.', + in: { name: 'company update', entity: 'company', action: 'update' }, + mapping: { + silent: true, + settings: { group: { map: { groupId: 'data.company_id' } } }, + }, + out: [], + outcome: { status: 'failed', code: 'missing-input' }, +}; diff --git a/packages/web/destinations/segment/src/index.ts b/packages/web/destinations/segment/src/index.ts index 5682343d6..f4fd4b386 100644 --- a/packages/web/destinations/segment/src/index.ts +++ b/packages/web/destinations/segment/src/index.ts @@ -1,5 +1,12 @@ import type { WalkerOS } from '@walkeros/core'; -import { getMappingValue, isArray, isObject, isString } from '@walkeros/core'; +import { + getMappingValue, + isArray, + isObject, + isString, + isThenable, + StepError, +} from '@walkeros/core'; import { AnalyticsBrowser } from '@segment/analytics-next'; import type { Destination, @@ -33,12 +40,14 @@ function getSegment(env: Env | undefined): SegmentSDK { } /** - * Call sdk.load() with destructured Segment settings. Strips walkerOS- - * specific keys from the options arg passed to the real SDK. + * Call sdk.load() with destructured Segment settings. The write key and the + * CDN host go into the first argument, the settings Analytics.js reads + * there; the rest are its init options. */ function loadAnalytics(sdk: SegmentSDK, settings: Settings): SegmentAnalytics { const { apiKey, + cdnURL, identify: _identify, group: _group, consent: _consent, @@ -52,23 +61,31 @@ function loadAnalytics(sdk: SegmentSDK, settings: Settings): SegmentAnalytics { ...initOptions, }; - return sdk.load({ writeKey: apiKey }, options); + return sdk.load( + { writeKey: apiKey, ...(cdnURL !== undefined ? { cdnURL } : {}) }, + options, + ); } -/** - * Resolve the live analytics instance, loading on demand if deferred. - * Returns undefined if load() was deferred and consent hasn't fired yet. - */ -function resolveAnalytics( - sdk: SegmentSDK, - settings: Settings, - state: RuntimeState, -): SegmentAnalytics | undefined { - if (state.analytics) return state.analytics; - if (state.loaded) return undefined; - state.analytics = loadAnalytics(sdk, settings); - state.loaded = true; - return state.analytics; +function isSegmentAnalytics(value: unknown): value is SegmentAnalytics { + return ( + isObject(value) && + ['track', 'identify', 'group', 'page', 'reset', 'setAnonymousId'].every( + (key) => typeof value[key] === 'function', + ) + ); +} + +/** Waits for one Analytics.js call; a throw or a rejection becomes a StepError. */ +async function send(fn: () => unknown): Promise { + try { + await fn(); + } catch (error) { + throw new StepError('Segment call failed', { + code: 'transport', + cause: error, + }); + } } /** @@ -96,7 +113,7 @@ function hashTraits(traits: Record | undefined): string { */ function buildEventOptions( event: WalkerOS.Event, - settings: Settings, + settings: Partial, ): SegmentEventOptions | undefined { const map = settings.consent; if (!map || Object.keys(map).length === 0) return undefined; @@ -124,14 +141,15 @@ function buildEventOptions( /** * Resolve and apply identity mapping. Calls analytics.identify() only - * when the resolved value differs from the last-fired identity. + * when the resolved value differs from the last-fired identity. Returns + * whether it called the SDK. */ -function applyIdentify( +async function applyIdentify( analytics: SegmentAnalytics, resolved: Record, state: RuntimeState, options?: SegmentEventOptions, -): void { +): Promise { const last = state.lastIdentity || {}; const userId = isString(resolved.userId) ? resolved.userId : undefined; const anonymousId = isString(resolved.anonymousId) @@ -139,11 +157,13 @@ function applyIdentify( : undefined; const traits = isObject(resolved.traits) && !isArray(resolved.traits) - ? (resolved.traits as Record) + ? resolved.traits : undefined; + let called = false; if (anonymousId && anonymousId !== last.anonymousId) { - analytics.setAnonymousId(anonymousId); + await send(() => analytics.setAnonymousId(anonymousId)); + called = true; } const traitsHash = hashTraits(traits); @@ -151,65 +171,69 @@ function applyIdentify( const traitsChanged = traitsHash !== (last.traitsHash ?? ''); const hasChange = userIdChanged || (traits !== undefined && traitsChanged); - if (!hasChange && !anonymousId) return; + if (!hasChange && !anonymousId) return called; - if (options) { - analytics.identify(userId, traits || {}, options); - } else { - analytics.identify(userId, traits || {}); - } + await send(() => + options + ? analytics.identify(userId, traits || {}, options) + : analytics.identify(userId, traits || {}), + ); state.lastIdentity = { userId, anonymousId: anonymousId ?? last.anonymousId, traitsHash, }; + return true; } -function applyGroup( +/** Resolve and apply the group mapping; returns whether it called the SDK. */ +async function applyGroup( analytics: SegmentAnalytics, resolved: Record, state: RuntimeState, options?: SegmentEventOptions, -): void { +): Promise { const groupId = isString(resolved.groupId) ? resolved.groupId : undefined; - if (!groupId) return; + if (!groupId) return false; const traits = isObject(resolved.traits) && !isArray(resolved.traits) - ? (resolved.traits as Record) + ? resolved.traits : undefined; const last = state.lastGroup || {}; const traitsHash = hashTraits(traits); const groupIdChanged = groupId !== last.groupId; const traitsChanged = traitsHash !== (last.traitsHash ?? ''); - if (!groupIdChanged && !traitsChanged) return; + if (!groupIdChanged && !traitsChanged) return false; - if (options) { - analytics.group(groupId, traits || {}, options); - } else { - analytics.group(groupId, traits || {}); - } + await send(() => + options + ? analytics.group(groupId, traits || {}, options) + : analytics.group(groupId, traits || {}), + ); state.lastGroup = { groupId, traitsHash }; + return true; } +/** Resolve and apply the page mapping; returns whether it called the SDK. */ async function applyPage( analytics: SegmentAnalytics, pageMapping: unknown, event: WalkerOS.Event, collector: unknown, options?: SegmentEventOptions, -): Promise { +): Promise { // Boolean true → call page() with no args (SDK auto-collect). if (pageMapping === true) { - if (options) { - analytics.page(undefined, undefined, undefined, options); - } else { - analytics.page(); - } - return; + await send(() => + options + ? analytics.page(undefined, undefined, undefined, options) + : analytics.page(), + ); + return true; } const resolved = await getMappingValue( @@ -217,29 +241,30 @@ async function applyPage( pageMapping as Parameters[1], { collector } as Parameters[2], ); - if (!isObject(resolved)) return; + if (!isObject(resolved)) return false; - const r = resolved as { - category?: unknown; - name?: unknown; - properties?: unknown; - }; - const category = isString(r.category) ? r.category : undefined; - const name = isString(r.name) ? r.name : undefined; + const category = isString(resolved.category) ? resolved.category : undefined; + const name = isString(resolved.name) ? resolved.name : undefined; const properties = - isObject(r.properties) && !isArray(r.properties) - ? (r.properties as Record) + isObject(resolved.properties) && !isArray(resolved.properties) + ? resolved.properties : undefined; - if (options) { - analytics.page(category ?? null, name ?? null, properties ?? {}, options); - } else if (properties !== undefined) { - analytics.page(category ?? null, name ?? null, properties); - } else if (name !== undefined || category !== undefined) { - analytics.page(category ?? null, name ?? null); - } else { - analytics.page(); - } + await send(() => { + if (options) + return analytics.page( + category ?? null, + name ?? null, + properties ?? {}, + options, + ); + if (properties !== undefined) + return analytics.page(category ?? null, name ?? null, properties); + if (name !== undefined || category !== undefined) + return analytics.page(category ?? null, name ?? null); + return analytics.page(); + }); + return true; } export const destinationSegment: Destination = { @@ -247,39 +272,63 @@ export const destinationSegment: Destination = { config: {}, - init({ config, env }) { - if (!config.settings?.apiKey) return false; - const settings = config.settings as Settings; - - const sdk = getSegment(env as Env | undefined); - const _state: RuntimeState = { - lastIdentity: {}, - lastGroup: {}, - loaded: false, - }; - - // If config.consent is declared, defer load() until on('consent'). - // Otherwise load immediately. - const requiresConsent = - !!config.consent && Object.keys(config.consent).length > 0; - - if (!requiresConsent) { - _state.analytics = loadAnalytics(sdk, settings); - _state.loaded = true; + // Analytics.js loads in init, so it starts with the page. A consent-gated + // destination is initialized only once its consent is granted. + async init({ config, env }) { + const apiKey = config.settings?.apiKey; + if (!apiKey) + throw new StepError('Segment needs settings.apiKey', { code: 'config' }); + const settings: Settings = { ...config.settings, apiKey }; + + const _state: RuntimeState = { lastIdentity: {}, lastGroup: {} }; + let loaded: unknown; + try { + loaded = loadAnalytics(getSegment(env), settings); + // load() answers the instance at once; a wrapped SDK may answer it + // through a promise. + if (!isSegmentAnalytics(loaded) && isThenable(loaded)) + loaded = await loaded; + } catch (error) { + throw new StepError('Segment load() failed', { + code: 'vendor-not-ready', + cause: error, + }); } + if (!isSegmentAnalytics(loaded)) + throw new StepError('Segment load() returned no Analytics instance', { + code: 'vendor-not-ready', + }); + + // The instance is also the promise of its load, settled once + // Analytics.js is ready: a failed load fails the later pushes. + if (isThenable(loaded)) + Promise.resolve(loaded).then( + () => undefined, + () => { + _state.failed = true; + }, + ); + _state.analytics = loaded; return { ...config, settings: { ...settings, _state } }; }, - async push(event, { config, rule, env, data, collector }) { - const sdk = getSegment(env as Env | undefined); - const settings = (config.settings || {}) as Settings; + async push(event, { config, rule, data, collector }) { + const settings: Partial = config.settings || {}; const state: RuntimeState = settings._state || {}; - const analytics = resolveAnalytics(sdk, settings, state); - if (!analytics) return; + const analytics = state.analytics; + if (state.failed) + throw new StepError('Segment Analytics.js failed to load', { + code: 'vendor-not-loaded', + }); + if (!analytics) + throw new StepError('Segment not ready: init did not load Analytics.js', { + code: 'not-ready', + }); const mappingSettings = rule?.settings || {}; const options = buildEventOptions(event, settings); + let calls = 0; // 1. Reset fires first so subsequent identity calls start clean. if (mappingSettings.reset !== undefined) { @@ -288,7 +337,8 @@ export const destinationSegment: Destination = { ? mappingSettings.reset : await getMappingValue(event, mappingSettings.reset, { collector }); if (resolved) { - analytics.reset(); + await send(() => analytics.reset()); + calls++; state.lastIdentity = {}; state.lastGroup = {}; } @@ -300,14 +350,11 @@ export const destinationSegment: Destination = { const resolved = await getMappingValue(event, identifyMapping, { collector, }); - if (isObject(resolved)) { - applyIdentify( - analytics, - resolved as Record, - state, - options, - ); - } + if ( + isObject(resolved) && + (await applyIdentify(analytics, resolved, state, options)) + ) + calls++; } // 3. Group - rule-level override wins over destination-level. @@ -316,66 +363,52 @@ export const destinationSegment: Destination = { const resolved = await getMappingValue(event, groupMapping, { collector, }); - if (isObject(resolved)) { - applyGroup( - analytics, - resolved as Record, - state, - options, - ); - } + if ( + isObject(resolved) && + (await applyGroup(analytics, resolved, state, options)) + ) + calls++; } // 4. Page (explicit first-class Segment pattern) - if (mappingSettings.page !== undefined) { - await applyPage( + if ( + mappingSettings.page !== undefined && + (await applyPage( analytics, mappingSettings.page, event, collector, options, - ); - } + )) + ) + calls++; // 5. Default track call unless rule opts out via silent. if (rule?.silent !== true) { const eventName = isString(rule?.name) ? rule.name : event.name; - const properties = isObject(data) - ? (data as Record) - : {}; - - if (options) { - analytics.track(eventName, properties, options); - } else { - analytics.track(eventName, properties); - } + const properties = isObject(data) ? data : {}; + + await send(() => + options + ? analytics.track(eventName, properties, options) + : analytics.track(eventName, properties), + ); + calls++; } settings._state = state; - }, - - on(type, context) { - if (type !== 'consent' || !context.data) return; - - const consent = context.data as WalkerOS.Consent; - const required = context.config?.consent; - if (!required || Object.keys(required).length === 0) return; - - const settings = (context.config?.settings || {}) as Settings; - const state: RuntimeState = settings._state || {}; - const sdk = getSegment(context.env as Env | undefined); - - const allGranted = Object.keys(required).every( - (key) => consent[key] === true, - ); - if (allGranted && !state.loaded) { - state.analytics = loadAnalytics(sdk, settings); - state.loaded = true; - settings._state = state; - } - // If not all granted, walkerOS's config.consent gate blocks subsequent - // push() calls - no SDK action needed (Segment has no opt-out method). + // A silent rule whose own settings resolved nothing sent nothing. + const ruleSettings = + mappingSettings.reset !== undefined || + mappingSettings.identify !== undefined || + mappingSettings.group !== undefined || + mappingSettings.page !== undefined; + if (!calls && ruleSettings) + throw new StepError( + `Segment: the silent rule of "${event.name}" resolved no identity, group or page`, + { code: 'missing-input' }, + ); }, }; diff --git a/packages/web/destinations/segment/src/types/index.ts b/packages/web/destinations/segment/src/types/index.ts index e8145018d..4883e8ac8 100644 --- a/packages/web/destinations/segment/src/types/index.ts +++ b/packages/web/destinations/segment/src/types/index.ts @@ -24,6 +24,8 @@ import type { export interface Settings extends Omit { /** Segment write key. Maps to `writeKey` in the load() settings arg. */ apiKey: string; + /** Custom CDN host for Analytics.js settings. Maps to `cdnURL` in the load() settings arg. */ + cdnURL?: string; /** walkerOS mapping value resolving to an identity object (userId, traits, anonymousId). */ identify?: WalkerOSMapping.Value; /** walkerOS mapping value resolving to a group object (groupId, traits). */ @@ -52,8 +54,8 @@ export interface RuntimeState { }; /** Holds the AnalyticsBrowser instance created in init(). */ analytics?: SegmentAnalytics; - /** True once load() has been called (may be deferred pending consent). */ - loaded?: boolean; + /** Set once the instance's load failed: later pushes fail. */ + failed?: boolean; } export type InitSettings = Partial; From 2251f63aa8d9304826228b5c9801d58e215c768f Mon Sep 17 00:00:00 2001 From: alexanderkirtzel Date: Wed, 21 Oct 2026 06:07:00 +0000 Subject: [PATCH 14/41] vendor attributes --- .../src/dev/__tests__/conformance.test.ts | 23 +++++++ .../__tests__/fixture-package/src/vendor.ts | 41 ++++++++++++ packages/collector/src/dev/runExamples.ts | 5 ++ .../web/core/src/__tests__/loadVendor.test.ts | 37 +++++++++++ packages/web/core/src/loadVendor.ts | 62 +++++++++++++++---- 5 files changed, 155 insertions(+), 13 deletions(-) diff --git a/packages/collector/src/dev/__tests__/conformance.test.ts b/packages/collector/src/dev/__tests__/conformance.test.ts index dfde8fb65..55538db89 100644 --- a/packages/collector/src/dev/__tests__/conformance.test.ts +++ b/packages/collector/src/dev/__tests__/conformance.test.ts @@ -27,6 +27,8 @@ import { consentAware, consentIgnoring, initCalling, + pageScriptIgnoring, + pageScriptVendor, scriptVendor, swallowingSdk, vendorGlobal, @@ -517,6 +519,27 @@ describe('conformance kit, vendor-global profile', () => { ).resolves.toBeUndefined(); }); + describe('a script injected once per page, loadScript on by default (item 2)', () => { + // No declared failure example: item 2 must reach the script error itself. + const { unreachable, ...scriptExamples } = examples; + const scriptOptions = { ...options, examples: scriptExamples }; + + it('reaches the script error although item 1 injected the src first', async () => { + await expect( + conformance('destination', pageScriptVendor, scriptOptions), + ).resolves.toBeUndefined(); + }); + + it('fails item 2 when the push ignores the script error', async () => { + await expect( + conformance('destination', pageScriptIgnoring, scriptOptions), + ).rejects.toMatchObject({ + items: [2], + message: expect.stringContaining('is not reported vendor-not-loaded'), + }); + }); + }); + describe('item 8 runs a revoke gated on its keys, granted first (I1)', () => { it('passes a destination that passes the revocation to its vendor', async () => { await expect( diff --git a/packages/collector/src/dev/__tests__/fixture-package/src/vendor.ts b/packages/collector/src/dev/__tests__/fixture-package/src/vendor.ts index a278294f5..480ed3612 100644 --- a/packages/collector/src/dev/__tests__/fixture-package/src/vendor.ts +++ b/packages/collector/src/dev/__tests__/fixture-package/src/vendor.ts @@ -152,3 +152,44 @@ export const scriptVendor: Destination.Instance = { vendor.consent(data.marketing === true); }, }; + +/** A script's error, kept with its element (the page is the registry). */ +const failedScripts = new WeakSet(); + +/** + * Injects its marked script once per page, as loadVendor does (a marked + * script with its src already on the page is the injected one), with + * `loadScript` on by default. A push calls the stub synchronously and, + * with `reports`, fails `vendor-not-loaded` once the script fired `error`. + */ +function pageScript(reports: boolean): Destination.Instance { + const src = 'https://vendor.example/page.js'; + const scriptOnPage = () => + Array.from(document.querySelectorAll('script[data-walkeros-vendor]')).find( + (script) => script.getAttribute('src') === src, + ); + return { + ...vendorGlobal, + type: reports ? 'page-script' : 'page-script-ignoring', + config: { loadScript: true }, + init: ({ config }) => { + if (!config.loadScript || scriptOnPage()) return; + const script = document.createElement('script'); + script.src = src; + script.setAttribute('data-walkeros-vendor', ''); + script.addEventListener('error', () => failedScripts.add(script)); + document.head.appendChild(script); + }, + push: async (event, context) => { + const script = scriptOnPage(); + if (reports && script && failedScripts.has(script)) + throw new StepError('Vendor script failed to load', { + code: 'vendor-not-loaded', + }); + await vendorGlobal.push(event, context); + }, + }; +} + +export const pageScriptVendor = pageScript(true); +export const pageScriptIgnoring = pageScript(false); diff --git a/packages/collector/src/dev/runExamples.ts b/packages/collector/src/dev/runExamples.ts index 4513edbc7..86c4884af 100644 --- a/packages/collector/src/dev/runExamples.ts +++ b/packages/collector/src/dev/runExamples.ts @@ -348,6 +348,11 @@ async function open( await collector.command('shutdown'); } finally { boundary.restore(); + // The page is loadVendor's once-per-src registry: the scripts this + // run added go, so the next run starts on the page it found. + documentScripts() + .filter((script) => !before.has(script)) + .forEach((script) => script.remove()); } }, }; diff --git a/packages/web/core/src/__tests__/loadVendor.test.ts b/packages/web/core/src/__tests__/loadVendor.test.ts index ddb9d9059..20f6e8ff6 100644 --- a/packages/web/core/src/__tests__/loadVendor.test.ts +++ b/packages/web/core/src/__tests__/loadVendor.test.ts @@ -37,6 +37,43 @@ describe('loadVendor', () => { expect(scripts[0].async).toBe(true); }); + it('sets the tag attributes before the script is inserted; src and the marker stay', () => { + const src = nextSrc(); + // Watching the head: an attribute set after the insertion would show. + const observer = new MutationObserver(() => undefined); + observer.observe(document.head, { + childList: true, + subtree: true, + attributes: true, + }); + loadVendor(src, get, { + attributes: { + 'data-domain': 'example.com', + src: 'https://other.example/x.js', + [VENDOR_SCRIPT_MARKER]: 'taken', + }, + }); + const records = observer.takeRecords(); + observer.disconnect(); + + expect(records.map(({ type }) => type)).toEqual(['childList']); + const [script] = scriptsOf(src); + expect(script.getAttribute('data-domain')).toBe('example.com'); + expect(script.getAttribute('src')).toBe(src); + expect(script.getAttribute(VENDOR_SCRIPT_MARKER)).toBe(''); + }); + + it('keeps the page as the registry: a script removed from the page is injected again', () => { + const src = nextSrc(); + loadVendor(src, get); + loadVendor(src, get); + expect(scriptsOf(src)).toHaveLength(1); + + scriptsOf(src).forEach((script) => script.remove()); + loadVendor(src, get); + expect(scriptsOf(src)).toHaveLength(1); + }); + it('resolves with get() after load', async () => { const src = nextSrc(); const ready = loadVendor(src, get).ready(); diff --git a/packages/web/core/src/loadVendor.ts b/packages/web/core/src/loadVendor.ts index 2b8b57486..4ec4069ff 100644 --- a/packages/web/core/src/loadVendor.ts +++ b/packages/web/core/src/loadVendor.ts @@ -6,6 +6,11 @@ export const VENDOR_SCRIPT_MARKER = 'data-walkeros-vendor'; export interface LoadVendorOptions { /** The destination's timeout; the load deadline is its resolved value. */ timeout?: number; + /** + * Attributes the vendor's tag needs (e.g. `data-domain`), set before the + * script is inserted. `src` and the marker cannot be overwritten. + */ + attributes?: Record; } export interface Vendor { @@ -34,20 +39,40 @@ interface Injected { waiting: Array<(outcome: Outcome) => void>; } -/** One script per src on the page: a failure is never re-injected. */ -const injected = new Map(); +/** The state of each vendor script, kept with its element. */ +const injected = new WeakMap(); -function inject(src: string, timeout: number): Injected { - const known = injected.get(src); +/** + * The page's script for `src`. The page itself is the registry: its one + * marked script with that src means the src was injected (once per page, a + * failure is never re-injected), and another document has its own. + */ +function scriptOf(src: string): HTMLScriptElement | undefined { + const marked = document.querySelectorAll( + `script[${VENDOR_SCRIPT_MARKER}]`, + ); + return Array.from(marked).find( + (script) => script.getAttribute('src') === src, + ); +} + +function inject( + src: string, + timeout: number, + attributes: Record = {}, +): Injected { + const existing = scriptOf(src); + const known = existing && injected.get(existing); if (known) return known; + // A marked script without state (another bundle's) is watched, not doubled. + const script = existing ?? document.createElement('script'); const entry: Injected = { state: 'loading', expired: false, timeout, waiting: [], }; - injected.set(src, entry); const settle = (outcome: Outcome) => { const waiting = entry.waiting; @@ -60,10 +85,7 @@ function inject(src: string, timeout: number): Injected { settle('expired'); }, timeout); - const script = document.createElement('script'); - script.src = src; - script.async = true; - script.setAttribute(VENDOR_SCRIPT_MARKER, ''); + injected.set(script, entry); script.addEventListener('load', () => { clearTimeout(timer); entry.state = 'loaded'; @@ -74,7 +96,16 @@ function inject(src: string, timeout: number): Injected { entry.state = 'failed'; settle('failed'); }); - document.head.appendChild(script); + + if (!existing) { + for (const [name, value] of Object.entries(attributes)) + script.setAttribute(name, value); + // After the attributes, so they never replace the src or the marker. + script.src = src; + script.async = true; + script.setAttribute(VENDOR_SCRIPT_MARKER, ''); + document.head.appendChild(script); + } return entry; } @@ -93,8 +124,9 @@ const timedOut = (timeout: number) => }); /** - * Injects a vendor script once per `src` (marked `data-walkeros-vendor`) and - * reports its readiness. The deadline runs from the injection and equals the + * Injects a vendor script once per `src` on the page (marked + * `data-walkeros-vendor`, with the `attributes` its tag needs) and reports + * its readiness. The deadline runs from the injection and equals the * destination's resolved timeout, so an awaited `ready()` settles before the * push's own timeout. Without `src` (the page loads the vendor) nothing is * injected and there is no deadline. @@ -105,7 +137,11 @@ export function loadVendor( options: LoadVendorOptions = {}, ): Vendor { const script = src - ? inject(src, resolveDestinationTimeout(options.timeout)) + ? inject( + src, + resolveDestinationTimeout(options.timeout), + options.attributes, + ) : undefined; const present = (): Promise => { From 75f4f34f7e417c6145bb0b34f735010354303b76 Mon Sep 17 00:00:00 2001 From: alexanderkirtzel Date: Wed, 21 Oct 2026 06:11:34 +0000 Subject: [PATCH 15/41] web recipe --- .../api/src/__tests__/conformance.test.ts | 20 ++ .../web/destinations/api/src/examples/step.ts | 9 + .../web/destinations/api/src/index.test.ts | 2 +- packages/web/destinations/api/src/index.ts | 89 +++--- .../destinations/api/src/schemas/settings.ts | 52 ++-- packages/web/destinations/d8a/jest.config.mjs | 11 +- .../d8a/src/__tests__/conformance.test.ts | 94 ++++++ .../web/destinations/d8a/src/examples/step.ts | 9 + packages/web/destinations/d8a/src/index.ts | 130 +++++--- .../d8a/src/schemas/primitives.ts | 153 +++++----- .../hotjar/src/__tests__/conformance.test.ts | 33 ++ .../destinations/hotjar/src/examples/step.ts | 20 ++ packages/web/destinations/hotjar/src/index.ts | 75 ++++- .../hotjar/src/schemas/settings.ts | 68 +++-- .../src/__tests__/conformance.test.ts | 21 ++ .../destinations/linkedin/src/examples/env.ts | 24 +- .../linkedin/src/examples/step.ts | 19 +- .../web/destinations/linkedin/src/index.ts | 107 ++++--- .../linkedin/src/schemas/settings.ts | 26 +- .../destinations/linkedin/src/types/index.ts | 2 - .../src/__tests__/conformance.test.ts | 66 ++++ .../mixpanel/src/examples/step.ts | 34 +++ .../web/destinations/mixpanel/src/index.ts | 155 ++++++++-- .../mixpanel/src/schemas/settings.ts | 202 ++++++------- .../src/__tests__/conformance.test.ts | 29 ++ .../src/__tests__/stepExamples.test.ts | 88 +++--- .../optimizely/src/examples/step.ts | 60 ++++ .../web/destinations/optimizely/src/index.ts | 123 ++++++-- .../optimizely/src/schemas/settings.ts | 102 ++++--- .../optimizely/src/types/index.ts | 2 - .../src/__tests__/conformance.test.ts | 24 ++ .../pinterest/src/__tests__/init.test.ts | 13 +- .../pinterest/src/examples/step.ts | 36 +++ .../web/destinations/pinterest/src/index.ts | 158 ++++++---- .../pinterest/src/schemas/settings.ts | 44 +-- .../src/__tests__/conformance.test.ts | 21 ++ .../src/__tests__/stepExamples.test.ts | 40 +-- .../plausible/src/examples/step.ts | 23 ++ .../destinations/plausible/src/index.test.ts | 70 +---- .../web/destinations/plausible/src/index.ts | 54 +++- .../plausible/src/schemas/settings.ts | 20 +- .../posthog/src/__tests__/conformance.test.ts | 68 +++++ .../destinations/posthog/src/examples/step.ts | 34 +++ .../web/destinations/posthog/src/index.ts | 121 ++++++-- .../posthog/src/schemas/settings.ts | 228 +++++++------- .../src/__tests__/conformance.test.ts | 24 ++ .../snowplow/src/examples/step.ts | 23 ++ .../destinations/snowplow/src/index.test.ts | 61 +--- .../web/destinations/snowplow/src/index.ts | 281 ++++++++++++------ .../web/destinations/snowplow/src/push.ts | 95 +++--- .../snowplow/src/schemas/settings.ts | 162 +++++++--- .../web/destinations/snowplow/src/setup.ts | 23 -- .../destinations/snowplow/src/types/index.ts | 9 - .../tiktok/src/__tests__/conformance.test.ts | 26 ++ .../tiktok/src/__tests__/stepExamples.test.ts | 19 +- .../destinations/tiktok/src/examples/step.ts | 37 ++- packages/web/destinations/tiktok/src/index.ts | 110 +++++-- .../tiktok/src/schemas/settings.ts | 58 ++-- packages/web/destinations/tiktok/src/setup.ts | 18 +- 59 files changed, 2501 insertions(+), 1224 deletions(-) create mode 100644 packages/web/destinations/api/src/__tests__/conformance.test.ts create mode 100644 packages/web/destinations/d8a/src/__tests__/conformance.test.ts create mode 100644 packages/web/destinations/hotjar/src/__tests__/conformance.test.ts create mode 100644 packages/web/destinations/linkedin/src/__tests__/conformance.test.ts create mode 100644 packages/web/destinations/mixpanel/src/__tests__/conformance.test.ts create mode 100644 packages/web/destinations/optimizely/src/__tests__/conformance.test.ts create mode 100644 packages/web/destinations/pinterest/src/__tests__/conformance.test.ts create mode 100644 packages/web/destinations/plausible/src/__tests__/conformance.test.ts create mode 100644 packages/web/destinations/posthog/src/__tests__/conformance.test.ts create mode 100644 packages/web/destinations/snowplow/src/__tests__/conformance.test.ts create mode 100644 packages/web/destinations/tiktok/src/__tests__/conformance.test.ts diff --git a/packages/web/destinations/api/src/__tests__/conformance.test.ts b/packages/web/destinations/api/src/__tests__/conformance.test.ts new file mode 100644 index 000000000..9bfc4d88c --- /dev/null +++ b/packages/web/destinations/api/src/__tests__/conformance.test.ts @@ -0,0 +1,20 @@ +import path from 'path'; +import { conformance } from '@walkeros/collector/dev'; +import { destinationAPI } from '..'; +import { examples, schemas } from '../dev'; + +describe('api conformance', () => { + beforeEach(() => { + jest.useRealTimers(); + }); + + it('conforms', async () => { + await conformance('destination', destinationAPI, { + profile: 'http', + examples: examples.step, + schema: schemas.settings, + settings: { url: 'https://api.example.com/events' }, + root: path.join(__dirname, '..', '..'), + }); + }); +}); diff --git a/packages/web/destinations/api/src/examples/step.ts b/packages/web/destinations/api/src/examples/step.ts index f9919791f..cf262ac8f 100644 --- a/packages/web/destinations/api/src/examples/step.ts +++ b/packages/web/destinations/api/src/examples/step.ts @@ -122,3 +122,12 @@ export const customTransform: Flow.StepExample = { ], ], }; + +/** A silent rule sends no request. */ +export const silentRule: Flow.StepExample = { + public: false, + description: 'A rule with silent: true sends no request.', + in: getEvent('page view', { timestamp: 1700000502 }), + mapping: { silent: true }, + out: [], +}; diff --git a/packages/web/destinations/api/src/index.test.ts b/packages/web/destinations/api/src/index.test.ts index 2540d03d4..0fb2b7ede 100644 --- a/packages/web/destinations/api/src/index.test.ts +++ b/packages/web/destinations/api/src/index.test.ts @@ -258,7 +258,7 @@ describe('Destination API', () => { id: 'test-api', }), ), - ).toThrow('Config settings url missing'); + ).toThrow('API destination needs settings.url'); }); test('succeeds when url provided', () => { diff --git a/packages/web/destinations/api/src/index.ts b/packages/web/destinations/api/src/index.ts index 0f2eca61d..4211b150c 100644 --- a/packages/web/destinations/api/src/index.ts +++ b/packages/web/destinations/api/src/index.ts @@ -1,6 +1,6 @@ import type { SendDataValue, SendHeaders, WalkerOS } from '@walkeros/core'; import type { Settings, Destination } from './types'; -import { isDefined } from '@walkeros/core'; +import { isDefined, StepError } from '@walkeros/core'; import { sendWeb } from '@walkeros/web-core'; // Types @@ -28,13 +28,13 @@ function withTraceparent( return { ...headers, traceparent: `00-${trace}-${id}-01` }; } -function send( +async function send( body: SendDataValue, settings: Settings, sendWebFn: typeof sendWeb, -): void { +): Promise { const { url, headers, method, transport = 'fetch' } = settings; - sendWebFn(url, body, { headers, method, transport }); + await sendWebFn(url, body, { headers, method, transport }); } /** A transformed body as a batch item: JSON object or array text becomes its value. */ @@ -53,55 +53,68 @@ export const destinationAPI: Destination = { config: {}, - init({ config, logger }) { + init({ config }) { const { url } = config.settings || {}; - if (!url) logger.throw('Config settings url missing'); + if (!url) + throw new StepError('API destination needs settings.url', { + code: 'config', + }); }, - push(event, { config, rule, data, env, logger }) { + async push(event, { config, rule, data, env }) { const { settings } = config; const { url, transform } = settings || {}; - if (!url) { - logger.throw('Config settings url missing'); - return; - } + if (!url) + throw new StepError('API destination needs settings.url', { + code: 'config', + }); - const eventData = isDefined(data) ? data : event; - // Transform returns body directly, otherwise stringify - const body = transform - ? transform(eventData, config, rule) - : JSON.stringify(eventData); + // A silent rule sends nothing + if (!rule?.silent) { + const eventData = isDefined(data) ? data : event; + // Transform returns body directly, otherwise stringify + const body = transform + ? transform(eventData, config, rule) + : JSON.stringify(eventData); - const headers = withTraceparent(event, settings?.headers); + const headers = withTraceparent(event, settings?.headers); - send(body, { ...settings, url, headers }, env.sendWeb || sendWeb); + await send(body, { ...settings, url, headers }, env.sendWeb || sendWeb); + } }, - pushBatch(batch, { config, rule, env, logger }) { + async pushBatch(batch, { config, rule, env }) { const { settings } = config; const { url, transform } = settings || {}; - if (!url) { - logger.throw('Config settings url missing'); - return; + if (!url) + throw new StepError('API destination needs settings.url', { + code: 'config', + }); + + // A silent rule sends nothing + if (!rule?.silent) { + const items = batch.entries.map((e) => + isDefined(e.data) ? e.data : e.event, + ); + + // Apply transform to each item if defined, then stringify the array. A + // transform returns one serialized body, so a JSON string joins the array + // as its value instead of as an escaped string. + const payload = transform + ? items.map((item) => fromBody(transform(item, config, rule))) + : items; + + // No traceparent here: a batch may aggregate events from distinct + // upstream traces, so there is no single identity to stamp. Per-event + // correlation is emitted on the non-batched push path only. + await send( + JSON.stringify(payload), + { ...settings, url }, + env.sendWeb || sendWeb, + ); } - - const items = batch.entries.map((e) => - isDefined(e.data) ? e.data : e.event, - ); - - // Apply transform to each item if defined, then stringify the array. A - // transform returns one serialized body, so a JSON string joins the array - // as its value instead of as an escaped string. - const payload = transform - ? items.map((item) => fromBody(transform(item, config, rule))) - : items; - - // No traceparent here: a batch may aggregate events from distinct - // upstream traces, so there is no single identity to stamp. Per-event - // correlation is emitted on the non-batched push path only. - send(JSON.stringify(payload), { ...settings, url }, env.sendWeb || sendWeb); }, }; diff --git a/packages/web/destinations/api/src/schemas/settings.ts b/packages/web/destinations/api/src/schemas/settings.ts index 88b29e9d7..29b885a04 100644 --- a/packages/web/destinations/api/src/schemas/settings.ts +++ b/packages/web/destinations/api/src/schemas/settings.ts @@ -1,29 +1,31 @@ import { z } from '@walkeros/core/dev'; -export const SettingsSchema = z.object({ - url: z - .string() - .url() - .describe( - 'The HTTP endpoint URL to send events to (like https://api.example.com/events)', - ), - headers: z - .record(z.string(), z.string()) - .describe( - "Additional HTTP headers to include with requests (like { 'Authorization': 'Bearer token', 'Content-Type': 'application/json' })", - ) - .optional(), - method: z.string().default('POST').describe('HTTP method for the request'), - transform: z - .any() - .describe( - 'Function to transform event data before sending (like (data, config, mapping) => JSON.stringify(data))', - ) - .optional(), - transport: z - .enum(['fetch', 'xhr', 'beacon']) - .default('fetch') - .describe('Transport method for sending requests'), -}); +export const SettingsSchema = z + .object({ + url: z + .string() + .url() + .describe( + 'The HTTP endpoint URL to send events to (like https://api.example.com/events)', + ), + headers: z + .record(z.string(), z.string()) + .describe( + "Additional HTTP headers to include with requests (like { 'Authorization': 'Bearer token', 'Content-Type': 'application/json' })", + ) + .optional(), + method: z.string().default('POST').describe('HTTP method for the request'), + transform: z + .any() + .describe( + 'Function to transform event data before sending (like (data, config, mapping) => JSON.stringify(data))', + ) + .optional(), + transport: z + .enum(['fetch', 'xhr', 'beacon']) + .default('fetch') + .describe('Transport method for sending requests'), + }) + .strict(); export type Settings = z.infer; diff --git a/packages/web/destinations/d8a/jest.config.mjs b/packages/web/destinations/d8a/jest.config.mjs index 9f5dcec3d..d5490e06c 100644 --- a/packages/web/destinations/d8a/jest.config.mjs +++ b/packages/web/destinations/d8a/jest.config.mjs @@ -1,5 +1,14 @@ import baseConfig from '@walkeros/config/jest/web.config'; -const config = {}; +// @d8a-tech/wt ships ESM only and exports no `require` condition: point jest +// at its module file and transform it like the workspace packages. +const config = { + moduleNameMapper: { + ...baseConfig.moduleNameMapper, + '^@d8a-tech/wt$': + '/../../../../node_modules/@d8a-tech/wt/dist/index.min.mjs', + }, + transformIgnorePatterns: ['/node_modules/(?!(@walkeros|@d8a-tech)/)'], +}; export default { ...baseConfig, ...config }; diff --git a/packages/web/destinations/d8a/src/__tests__/conformance.test.ts b/packages/web/destinations/d8a/src/__tests__/conformance.test.ts new file mode 100644 index 000000000..5397e8e0e --- /dev/null +++ b/packages/web/destinations/d8a/src/__tests__/conformance.test.ts @@ -0,0 +1,94 @@ +import path from 'path'; +import { conformance } from '@walkeros/collector/dev'; +import { destinationD8a } from '..'; +import { examples, schemas } from '../dev'; + +describe('d8a conformance', () => { + beforeEach(() => { + jest.useRealTimers(); + }); + + it('conforms', async () => { + await conformance('destination', destinationD8a, { + profile: 'web-sdk', + examples: examples.step, + schema: schemas.settings, + settings: { + property_id: '80e1d6d0-560d-419f-ac2a-fe9281e93386', + server_container_url: + 'https://global.t.d8a.tech/80e1d6d0-560d-419f-ac2a-fe9281e93386/d/c', + send_page_view: true, + debug_mode: false, + cookie_domain: 'auto', + cookie_path: '/', + cookie_expires: 63072000, + cookie_flags: 'SameSite=Lax', + cookie_prefix: 'd8a_', + cookie_update: true, + session_timeout_ms: 1800000, + session_engagement_time_sec: 10, + flush_interval_ms: 1000, + max_batch_size: 20, + user_id: 'user-1', + client_id: 'client-1', + campaign_id: 'c1', + campaign_source: 'newsletter', + campaign_medium: 'email', + campaign_name: 'spring', + campaign_term: 'term', + campaign_content: 'content', + page_location: 'https://example.com/', + page_title: 'Example', + page_referrer: 'https://example.org/', + content_group: 'docs', + language: 'en', + screen_resolution: '1920x1080', + ignore_referrer: false, + site_search_enabled: true, + site_search_query_params: ['q'], + outbound_clicks_enabled: true, + outbound_exclude_domains: ['example.com'], + file_downloads_enabled: true, + file_download_extensions: ['pdf'], + }, + env: { push: examples.env.push, simulation: examples.env.simulation }, + // Forwarded whole into d8a('config', ...) by buildConfigParams + passThrough: [ + 'send_page_view', + 'debug_mode', + 'cookie_domain', + 'cookie_path', + 'cookie_expires', + 'cookie_flags', + 'cookie_prefix', + 'cookie_update', + 'session_timeout_ms', + 'session_engagement_time_sec', + 'flush_interval_ms', + 'max_batch_size', + 'user_id', + 'client_id', + 'campaign_id', + 'campaign_source', + 'campaign_medium', + 'campaign_name', + 'campaign_term', + 'campaign_content', + 'page_location', + 'page_title', + 'page_referrer', + 'content_group', + 'language', + 'screen_resolution', + 'ignore_referrer', + 'site_search_enabled', + 'site_search_query_params', + 'outbound_clicks_enabled', + 'outbound_exclude_domains', + 'file_downloads_enabled', + 'file_download_extensions', + ], + root: path.join(__dirname, '..', '..'), + }); + }); +}); diff --git a/packages/web/destinations/d8a/src/examples/step.ts b/packages/web/destinations/d8a/src/examples/step.ts index e4af4beee..f7e5086a5 100644 --- a/packages/web/destinations/d8a/src/examples/step.ts +++ b/packages/web/destinations/d8a/src/examples/step.ts @@ -175,3 +175,12 @@ export const consentMode: Flow.StepExample = { ], ], }; + +/** A silent rule sends no d8a event. */ +export const silentRule: Flow.StepExample = { + public: false, + description: 'A rule with silent: true sends no d8a event.', + in: getEvent('page view', { timestamp: 1700000302 }), + mapping: { silent: true }, + out: [], +}; diff --git a/packages/web/destinations/d8a/src/index.ts b/packages/web/destinations/d8a/src/index.ts index b5ffcabb8..d4336939c 100644 --- a/packages/web/destinations/d8a/src/index.ts +++ b/packages/web/destinations/d8a/src/index.ts @@ -1,5 +1,5 @@ import type { WalkerOS } from '@walkeros/core'; -import { isObject } from '@walkeros/core'; +import { isObject, StepError } from '@walkeros/core'; import { getEnv } from '@walkeros/web-core'; import type { D8aConfigParams, D8aFn } from '@d8a-tech/wt'; import { installD8a } from '@d8a-tech/wt'; @@ -29,6 +29,32 @@ function getD8a( return windowRef[name] as D8aFn; } +/** Runs one d8a call; a throw or a rejection becomes a StepError of `code`. */ +async function call( + fn: () => unknown, + code: string, + message: string, +): Promise { + try { + await fn(); + } catch (error) { + throw new StepError(message, { code, cause: error }); + } +} + +/** The installed d8a function, or a StepError when it is not there. */ +function requireD8a( + windowRef: Window & Record, + name: string, +): D8aFn { + const d8a = getD8a(windowRef, name); + if (typeof d8a !== 'function') + throw new StepError(`d8a not ready: window.${name} is missing`, { + code: 'vendor-not-ready', + }); + return d8a; +} + function buildConfigParams(settings: Partial): D8aConfigParams { const { property_id: _property_id, @@ -48,13 +74,18 @@ export const destinationD8a: Destination = { config: { settings: {} }, - async init({ config, env, logger }) { + async init({ config, env }) { const settings = config.settings || {}; const { property_id, server_container_url } = settings; - if (!property_id) logger.throw('Config settings property_id missing'); - if (!server_container_url) - logger.throw('Config settings server_container_url missing'); + const missing = [ + !property_id && 'property_id', + !server_container_url && 'server_container_url', + ].filter(Boolean); + if (missing.length) + throw new StepError(`d8a needs settings.${missing.join(', ')}`, { + code: 'config', + }); const { window } = getEnv(env); const globalName = settings.globalName || DEFAULT_GLOBAL_NAME; @@ -67,40 +98,61 @@ export const destinationD8a: Destination = { dataLayerName, }); - const d8a = getD8a(window as Window & Record, globalName); - d8a('js', new Date()); - d8a('config', property_id, buildConfigParams(settings)); + const d8a = requireD8a( + window as Window & Record, + globalName, + ); + await call( + () => d8a('js', new Date()), + 'vendor-not-ready', + 'd8a init call failed', + ); + await call( + () => d8a('config', property_id, buildConfigParams(settings)), + 'vendor-not-ready', + 'd8a config call failed', + ); return config; }, - async push(event, { config, data, env, collector }) { - const settings = config.settings || {}; - const eventData = await getData( - event, - data as WalkerOS.AnyObject, - config, - settings, - collector, - ); - const eventParams = isObject(eventData) - ? { ...(eventData as Record) } - : {}; - - eventParams.send_to = settings.property_id; - - const eventName = normalizeEventName( - event.name, - settings.snakeCase !== false, - ); - const { window } = getEnv(env); - const globalName = settings.globalName || DEFAULT_GLOBAL_NAME; - const d8a = getD8a(window as Window & Record, globalName); - - d8a('event', eventName, eventParams); + async push(event, { config, rule, data, env, collector }) { + // A silent rule sends no d8a event + if (!rule?.silent) { + const settings = config.settings || {}; + const eventData = await getData( + event, + data as WalkerOS.AnyObject, + config, + settings, + collector, + ); + const eventParams = isObject(eventData) + ? { ...(eventData as Record) } + : {}; + + eventParams.send_to = settings.property_id; + + const eventName = normalizeEventName( + event.name, + settings.snakeCase !== false, + ); + const { window } = getEnv(env); + const globalName = settings.globalName || DEFAULT_GLOBAL_NAME; + const d8a = requireD8a( + window as Window & Record, + globalName, + ); + + await call( + () => d8a('event', eventName, eventParams), + 'transport', + 'd8a event call failed', + ); + } }, - on(type, context) { + async on(type, context) { if (type !== 'consent' || !context.data) return; const settings = (context.config?.settings || {}) as Partial; @@ -127,7 +179,11 @@ export const destinationD8a: Destination = { allParams.forEach((param) => { defaultConsent[param] = 'denied'; }); - d8a('consent', 'default', defaultConsent); + await call( + () => d8a('consent', 'default', defaultConsent), + 'consent-not-applied', + 'd8a did not take the consent default', + ); } defaultConsentSet = true; @@ -147,7 +203,11 @@ export const destinationD8a: Destination = { if (Object.keys(d8aConsent).length === 0) return; - d8a('consent', 'update', d8aConsent); + await call( + () => d8a('consent', 'update', d8aConsent), + 'consent-not-applied', + 'd8a did not take the consent update', + ); }, }; diff --git a/packages/web/destinations/d8a/src/schemas/primitives.ts b/packages/web/destinations/d8a/src/schemas/primitives.ts index 9d4c17937..d3536cfa6 100644 --- a/packages/web/destinations/d8a/src/schemas/primitives.ts +++ b/packages/web/destinations/d8a/src/schemas/primitives.ts @@ -7,79 +7,84 @@ export const ConsentModeSchema = z.union([ const csvOrArray = z.union([z.string(), z.array(z.string())]); -export const SettingsSchema = z.object({ - property_id: z.string().describe('d8a property ID'), - server_container_url: z - .string() - .url() - .describe('d8a collector URL for the property'), - como: ConsentModeSchema.describe( - 'Consent mode configuration: false (disabled), true (use defaults), or custom mapping', - ).optional(), - data: z.unknown().describe('Custom data mapping configuration').optional(), - dataLayerName: z - .string() - .describe('Name of the d8a command queue (default: d8aLayer)') - .optional(), - globalName: z - .string() - .describe('Name of the global d8a function (default: d8a)') - .optional(), - send_page_view: z - .boolean() - .describe('Enable automatic pageview tracking') - .optional(), - snakeCase: z - .boolean() - .describe('Convert event names to snake_case (like true)') - .optional(), - debug_mode: z.boolean().describe('Enable debug mode').optional(), - cookie_domain: z - .string() - .describe('Cookie domain strategy: auto, none, or a specific domain') - .optional(), - cookie_path: z.string().describe('Cookie path').optional(), - cookie_expires: z.number().describe('Cookie lifetime in seconds').optional(), - cookie_flags: z.string().describe('Raw cookie flags').optional(), - cookie_prefix: z.string().describe('Cookie name prefix').optional(), - cookie_update: z - .boolean() - .describe('Refresh cookie expirations on activity') - .optional(), - session_timeout_ms: z - .number() - .describe('Session timeout window in milliseconds') - .optional(), - session_engagement_time_sec: z - .number() - .describe('Minimum engaged time in seconds') - .optional(), - flush_interval_ms: z - .number() - .describe('Flush interval in milliseconds') - .optional(), - max_batch_size: z.number().describe('Maximum batch size').optional(), - user_id: z.string().describe('GA4-style user ID').optional(), - client_id: z.string().describe('Client ID override').optional(), - campaign_id: z.string().optional(), - campaign_source: z.string().optional(), - campaign_medium: z.string().optional(), - campaign_name: z.string().optional(), - campaign_term: z.string().optional(), - campaign_content: z.string().optional(), - page_location: z.string().optional(), - page_title: z.string().optional(), - page_referrer: z.string().optional(), - content_group: z.string().optional(), - language: z.string().optional(), - screen_resolution: z.string().optional(), - ignore_referrer: z.boolean().optional(), - site_search_enabled: z.boolean().optional(), - site_search_query_params: csvOrArray.optional(), - outbound_clicks_enabled: z.boolean().optional(), - outbound_exclude_domains: csvOrArray.optional(), - file_downloads_enabled: z.boolean().optional(), - file_download_extensions: csvOrArray.optional(), -}); +export const SettingsSchema = z + .object({ + property_id: z.string().describe('d8a property ID'), + server_container_url: z + .string() + .url() + .describe('d8a collector URL for the property'), + como: ConsentModeSchema.describe( + 'Consent mode configuration: false (disabled), true (use defaults), or custom mapping', + ).optional(), + data: z.unknown().describe('Custom data mapping configuration').optional(), + dataLayerName: z + .string() + .describe('Name of the d8a command queue (default: d8aLayer)') + .optional(), + globalName: z + .string() + .describe('Name of the global d8a function (default: d8a)') + .optional(), + send_page_view: z + .boolean() + .describe('Enable automatic pageview tracking') + .optional(), + snakeCase: z + .boolean() + .describe('Convert event names to snake_case (like true)') + .optional(), + debug_mode: z.boolean().describe('Enable debug mode').optional(), + cookie_domain: z + .string() + .describe('Cookie domain strategy: auto, none, or a specific domain') + .optional(), + cookie_path: z.string().describe('Cookie path').optional(), + cookie_expires: z + .number() + .describe('Cookie lifetime in seconds') + .optional(), + cookie_flags: z.string().describe('Raw cookie flags').optional(), + cookie_prefix: z.string().describe('Cookie name prefix').optional(), + cookie_update: z + .boolean() + .describe('Refresh cookie expirations on activity') + .optional(), + session_timeout_ms: z + .number() + .describe('Session timeout window in milliseconds') + .optional(), + session_engagement_time_sec: z + .number() + .describe('Minimum engaged time in seconds') + .optional(), + flush_interval_ms: z + .number() + .describe('Flush interval in milliseconds') + .optional(), + max_batch_size: z.number().describe('Maximum batch size').optional(), + user_id: z.string().describe('GA4-style user ID').optional(), + client_id: z.string().describe('Client ID override').optional(), + campaign_id: z.string().optional(), + campaign_source: z.string().optional(), + campaign_medium: z.string().optional(), + campaign_name: z.string().optional(), + campaign_term: z.string().optional(), + campaign_content: z.string().optional(), + page_location: z.string().optional(), + page_title: z.string().optional(), + page_referrer: z.string().optional(), + content_group: z.string().optional(), + language: z.string().optional(), + screen_resolution: z.string().optional(), + ignore_referrer: z.boolean().optional(), + site_search_enabled: z.boolean().optional(), + site_search_query_params: csvOrArray.optional(), + outbound_clicks_enabled: z.boolean().optional(), + outbound_exclude_domains: csvOrArray.optional(), + file_downloads_enabled: z.boolean().optional(), + file_download_extensions: csvOrArray.optional(), + }) + .strict(); export type Settings = z.infer; diff --git a/packages/web/destinations/hotjar/src/__tests__/conformance.test.ts b/packages/web/destinations/hotjar/src/__tests__/conformance.test.ts new file mode 100644 index 000000000..9d01368a5 --- /dev/null +++ b/packages/web/destinations/hotjar/src/__tests__/conformance.test.ts @@ -0,0 +1,33 @@ +import path from 'path'; +import { conformance } from '@walkeros/collector/dev'; +import { destinationHotjar } from '..'; +import { examples, schemas } from '../dev'; + +// The @hotjar/browser package is ESM -- stub the import so Jest can load it. +jest.mock('@hotjar/browser', () => ({ + __esModule: true, + default: { + init: jest.fn(() => true), + event: jest.fn(() => true), + identify: jest.fn(() => true), + stateChange: jest.fn(() => true), + isReady: jest.fn(() => true), + }, +})); + +describe('hotjar conformance', () => { + beforeEach(() => { + jest.useRealTimers(); + }); + + it('conforms', async () => { + await conformance('destination', destinationHotjar, { + profile: 'web-sdk', + examples: examples.step, + schema: schemas.settings, + settings: { siteId: 1234567 }, + env: { push: examples.env.push, simulation: examples.env.simulation }, + root: path.join(__dirname, '..', '..'), + }); + }); +}); diff --git a/packages/web/destinations/hotjar/src/examples/step.ts b/packages/web/destinations/hotjar/src/examples/step.ts index 28edf4625..5485af0d7 100644 --- a/packages/web/destinations/hotjar/src/examples/step.ts +++ b/packages/web/destinations/hotjar/src/examples/step.ts @@ -154,3 +154,23 @@ export const pageViewSilent: HotjarStepExample = { }, out: [], }; + +/** + * The silent stateChange rule of pageViewStateChange on a page view without + * data: no path resolves and the rule suppresses the event, so nothing is + * sent. + */ +export const pageViewStateChangeWithoutData: HotjarStepExample = { + public: false, + description: + 'A silent rule whose stateChange mapping finds no path sends nothing, so the push fails with missing-input instead of counting as delivered.', + in: { name: 'page view', entity: 'page', action: 'view' }, + mapping: { + silent: true, + settings: { + stateChange: 'data.id', + }, + }, + out: [], + outcome: { status: 'failed', code: 'missing-input' }, +}; diff --git a/packages/web/destinations/hotjar/src/index.ts b/packages/web/destinations/hotjar/src/index.ts index 4635ca9d7..a0ef410db 100644 --- a/packages/web/destinations/hotjar/src/index.ts +++ b/packages/web/destinations/hotjar/src/index.ts @@ -1,4 +1,4 @@ -import { getMappingValue, isObject, isString } from '@walkeros/core'; +import { getMappingValue, isObject, isString, StepError } from '@walkeros/core'; import Hotjar from '@hotjar/browser'; import type { HotjarSDK, Destination, Env, Mapping, Settings } from './types'; @@ -13,24 +13,49 @@ function getHotjar(env: Env | undefined): HotjarSDK { return env?.hotjar ?? (Hotjar as unknown as HotjarSDK); } +/** + * Runs one Hotjar call. The SDK reports failure by returning `false` (and + * logging to the console), so a `false` or a throw becomes a StepError. + */ +async function call( + fn: () => unknown, + code: string, + message: string, +): Promise { + let result: unknown; + try { + result = await fn(); + } catch (error) { + throw new StepError(message, { code, cause: error }); + } + if (result === false) throw new StepError(message, { code }); +} + export const destinationHotjar: Destination = { type: 'hotjar', config: {}, - init({ config, env }) { + async init({ config, env }) { const settings = config.settings; - if (!settings?.siteId) return false; + if (!settings?.siteId) + throw new StepError('Hotjar needs settings.siteId', { code: 'config' }); const hotjarVersion = settings.hotjarVersion ?? 6; const opts: { debug?: boolean; nonce?: string } = {}; if (settings.debug !== undefined) opts.debug = settings.debug; if (settings.nonce !== undefined) opts.nonce = settings.nonce; - getHotjar(env).init( - settings.siteId, - hotjarVersion, - Object.keys(opts).length > 0 ? opts : undefined, + const { siteId } = settings; + await call( + () => + getHotjar(env).init( + siteId, + hotjarVersion, + Object.keys(opts).length > 0 ? opts : undefined, + ), + 'vendor-not-ready', + 'Hotjar init failed', ); return config; @@ -41,6 +66,9 @@ export const destinationHotjar: Destination = { const settings = (config.settings || {}) as Partial; const mappingSettings = (rule?.settings || {}) as Mapping; + // Whether any Hotjar call went out for this event + let sent = false; + // 1. Identify -- rule-level override wins over destination-level. // Fires first per Hotjar's guidance so subsequent event calls // are associated with the right user. @@ -63,7 +91,12 @@ export const destinationHotjar: Destination = { userInfo[key] = val; } } - hotjar.identify(userId, userInfo); + await call( + () => hotjar.identify(userId, userInfo), + 'transport', + 'Hotjar identify failed', + ); + sent = true; } } } @@ -76,15 +109,37 @@ export const destinationHotjar: Destination = { { collector }, ); if (isString(resolved) && resolved) { - hotjar.stateChange(resolved); + await call( + () => hotjar.stateChange(resolved), + 'transport', + 'Hotjar stateChange failed', + ); + sent = true; } } // 3. Default event forwarding -- unless rule.silent is set if (rule?.silent !== true) { const eventName = isString(rule?.name) ? rule.name : event.name; - hotjar.event(eventName); + await call( + () => hotjar.event(eventName), + 'transport', + 'Hotjar event failed', + ); + sent = true; } + + // A silent rule that asked for identify or stateChange, which the event + // could not resolve, sent nothing at all + if ( + !sent && + (identifyMapping !== undefined || + mappingSettings.stateChange !== undefined) + ) + throw new StepError( + 'Hotjar identify or stateChange resolved to nothing for this event', + { code: 'missing-input' }, + ); }, }; diff --git a/packages/web/destinations/hotjar/src/schemas/settings.ts b/packages/web/destinations/hotjar/src/schemas/settings.ts index 83c46e8ab..98b273e72 100644 --- a/packages/web/destinations/hotjar/src/schemas/settings.ts +++ b/packages/web/destinations/hotjar/src/schemas/settings.ts @@ -1,37 +1,39 @@ import { z } from '@walkeros/core/dev'; -export const SettingsSchema = z.object({ - siteId: z - .number() - .int() - .positive() - .describe( - 'Your Hotjar site ID (e.g. 1234567). Find it in your Hotjar dashboard under Settings.', - ), - hotjarVersion: z - .number() - .int() - .positive() - .describe( - 'Hotjar snippet version. Defaults to 6 (current). Override only if Hotjar releases a new version.', - ) - .optional(), - debug: z - .boolean() - .describe('Enable Hotjar debug mode for development troubleshooting.') - .optional(), - nonce: z - .string() - .describe( - 'CSP nonce for the injected Hotjar script tag. Required when using strict Content-Security-Policy.', - ) - .optional(), - identify: z - .unknown() - .describe( - 'walkerOS mapping value resolving to { userId, ...attributes } for Hotjar.identify(). userId is extracted as the first argument; remaining keys become user attributes.', - ) - .optional(), -}); +export const SettingsSchema = z + .object({ + siteId: z + .number() + .int() + .positive() + .describe( + 'Your Hotjar site ID (e.g. 1234567). Find it in your Hotjar dashboard under Settings.', + ), + hotjarVersion: z + .number() + .int() + .positive() + .describe( + 'Hotjar snippet version. Defaults to 6 (current). Override only if Hotjar releases a new version.', + ) + .optional(), + debug: z + .boolean() + .describe('Enable Hotjar debug mode for development troubleshooting.') + .optional(), + nonce: z + .string() + .describe( + 'CSP nonce for the injected Hotjar script tag. Required when using strict Content-Security-Policy.', + ) + .optional(), + identify: z + .unknown() + .describe( + 'walkerOS mapping value resolving to { userId, ...attributes } for Hotjar.identify(). userId is extracted as the first argument; remaining keys become user attributes.', + ) + .optional(), + }) + .strict(); export type Settings = z.infer; diff --git a/packages/web/destinations/linkedin/src/__tests__/conformance.test.ts b/packages/web/destinations/linkedin/src/__tests__/conformance.test.ts new file mode 100644 index 000000000..a07bc36db --- /dev/null +++ b/packages/web/destinations/linkedin/src/__tests__/conformance.test.ts @@ -0,0 +1,21 @@ +import path from 'path'; +import { conformance } from '@walkeros/collector/dev'; +import { destinationLinkedIn } from '..'; +import { examples, schemas } from '../dev'; + +describe('linkedin conformance', () => { + beforeEach(() => { + jest.useRealTimers(); + }); + + it('conforms', async () => { + await conformance('destination', destinationLinkedIn, { + profile: 'vendor-global', + examples: examples.step, + schema: schemas.settings, + settings: { apiKey: '123456' }, + env: { push: examples.env.push, simulation: examples.env.simulation }, + root: path.join(__dirname, '..', '..'), + }); + }); +}); diff --git a/packages/web/destinations/linkedin/src/examples/env.ts b/packages/web/destinations/linkedin/src/examples/env.ts index 58f114090..0cb77bc56 100644 --- a/packages/web/destinations/linkedin/src/examples/env.ts +++ b/packages/web/destinations/linkedin/src/examples/env.ts @@ -5,7 +5,7 @@ import type { Env, Lintrk } from '../types'; * * Tests clone `push` and replace `window.lintrk` with a jest.fn() to collect * calls. Production leaves `env` undefined — the destination mutates the real - * `window` object and injects the LinkedIn CDN script via `addScript()`. + * `window` object and injects the LinkedIn CDN script through `loadVendor`. */ const noop = () => {}; @@ -20,17 +20,6 @@ export const init: Env | undefined = { _linkedin_data_partner_ids: undefined, lintrk: undefined, }, - document: { - createElement: () => ({ - src: '', - async: false, - dataset: {}, - setAttribute: () => {}, - removeAttribute: () => {}, - }), - head: { appendChild: () => {} }, - querySelector: () => null, - }, }; /** @@ -45,17 +34,6 @@ export const push: Env = { _linkedin_data_partner_ids: ['123456'], lintrk: Object.assign(noop, { q: [] as unknown[] }) as unknown as Lintrk, }, - document: { - createElement: () => ({ - src: '', - async: false, - dataset: {}, - setAttribute: () => {}, - removeAttribute: () => {}, - }), - head: { appendChild: () => {} }, - querySelector: () => null, - }, }; /** diff --git a/packages/web/destinations/linkedin/src/examples/step.ts b/packages/web/destinations/linkedin/src/examples/step.ts index 3a036c79a..6906e2d4e 100644 --- a/packages/web/destinations/linkedin/src/examples/step.ts +++ b/packages/web/destinations/linkedin/src/examples/step.ts @@ -203,7 +203,7 @@ export const conversionSilenced: LinkedInStepExample = { }; /** - * Falsy `id` → entire lintrk call is skipped. + * Falsy `id` → entire lintrk call is skipped and the push fails missing-input. * * If the resolved conversion object has no truthy `id`, the destination does * NOT call lintrk at all. This protects against misconfigured mappings @@ -214,6 +214,8 @@ export const conversionSilenced: LinkedInStepExample = { */ export const missingConversionIdIgnored: LinkedInStepExample = { public: false, + description: + 'A conversion mapping whose id resolves to nothing makes no lintrk call, so the push fails with missing-input instead of counting as delivered.', in: createEvent({ name: 'form submit', timestamp: 1700000106 }), mapping: { settings: { @@ -225,6 +227,7 @@ export const missingConversionIdIgnored: LinkedInStepExample = { }, }, out: [], + outcome: { status: 'failed', code: 'missing-input' }, }; /** @@ -262,3 +265,17 @@ export const partialFieldsOmitted: LinkedInStepExample = { ], ], }; + +/** + * A revocation of the consent the destination requires. The Insight Tag has + * no vendor consent mode, so the collector's consent gate is the only + * mechanism and no vendor call is made. + */ +export const consentRevoke: LinkedInStepExample = { + public: false, + description: + 'For a destination with config.consent { marketing: true }, revoking marketing makes no vendor call: the Insight Tag has no consent mode, the collector stops sending.', + command: 'consent', + in: { marketing: false }, + out: [], +}; diff --git a/packages/web/destinations/linkedin/src/index.ts b/packages/web/destinations/linkedin/src/index.ts index 37aaa601a..badc0843b 100644 --- a/packages/web/destinations/linkedin/src/index.ts +++ b/packages/web/destinations/linkedin/src/index.ts @@ -1,7 +1,12 @@ import type { WalkerOS } from '@walkeros/core'; -import { getMappingValue, isNumber, isObject, isString } from '@walkeros/core'; -import type { DestinationWeb } from '@walkeros/web-core'; -import { getEnv } from '@walkeros/web-core'; +import { + getMappingValue, + isNumber, + isObject, + isString, + StepError, +} from '@walkeros/core'; +import { getEnv, loadVendor } from '@walkeros/web-core'; import type { Destination, Env, @@ -55,13 +60,30 @@ function setPartnerGlobals(env: Env | undefined, partnerId: string): void { } } -function addScript(env: DestinationWeb.Env | undefined): void { - const { document } = getEnv(env); - const doc = document as Document; - const script = doc.createElement('script'); - script.src = LINKEDIN_SCRIPT_SRC; - script.async = true; - doc.head.appendChild(script); +/** + * The Insight Tag script `loadVendor` injects with `loadScript`. lintrk + * replays its own queue, so it is called synchronously and only a failed + * script is reported. + */ +function vendor(config: { loadScript?: boolean; timeout?: number }, env?: Env) { + return loadVendor( + config.loadScript ? LINKEDIN_SCRIPT_SRC : undefined, + () => (getEnv(env).window as Env['window']).lintrk, + { timeout: config.timeout }, + ); +} + +/** Runs one lintrk call; a throw or a rejection becomes a StepError of `code`. */ +async function call( + fn: () => unknown, + code: string, + message: string, +): Promise { + try { + await fn(); + } catch (error) { + throw new StepError(message, { code, cause: error }); + } } /** @@ -126,7 +148,8 @@ export const destinationLinkedIn: Destination = { init({ config, env }) { const settings = config.settings; - if (!settings?.apiKey) return false; + if (!settings?.apiKey) + throw new StepError('LinkedIn needs settings.apiKey', { code: 'config' }); setPartnerGlobals(env as Env | undefined, settings.apiKey); installLintrkQueue(env as Env | undefined); @@ -135,33 +158,46 @@ export const destinationLinkedIn: Destination = { // plausible / meta). Users who already embed the Insight Tag snippet in // their HTML should leave loadScript false - the destination only // installs the queue shim and mutates _linkedin_partner_id. - if (config.loadScript) addScript(env); + vendor(config, env as Env | undefined); return config; }, - async push(event, { rule, env, collector }) { + async push(event, { config, rule, env, collector }) { + vendor(config, env as Env | undefined).check(); + const { window } = getEnv(env); const w = window as Env['window']; const lintrk = w.lintrk; - if (!lintrk) return; // init must have run - - // Honor rule-level silent (core flag at packages/core/src/types/mapping.ts). - if (rule?.silent === true) return; + if (!lintrk) + throw new StepError('LinkedIn Insight Tag not ready: lintrk is missing', { + code: 'vendor-not-ready', + }); const mappingSettings = (rule?.settings || {}) as Mapping; - if (mappingSettings.conversion === undefined) return; // opt-in: no config = no call - - const resolved = await getMappingValue( - event as WalkerOS.Event, - mappingSettings.conversion, - { collector }, - ); - const data = buildLintrkData(resolved); - if (!data) return; - - lintrk('track', data); + // Opt-in: a rule without a conversion makes no call. Honor rule-level + // silent (core flag at packages/core/src/types/mapping.ts) the same way. + if (rule?.silent !== true && mappingSettings.conversion !== undefined) { + const resolved = await getMappingValue( + event as WalkerOS.Event, + mappingSettings.conversion, + { collector }, + ); + + const data = buildLintrkData(resolved); + if (!data) + throw new StepError( + 'LinkedIn conversion mapping resolved to no conversion id', + { code: 'missing-input' }, + ); + + await call( + () => lintrk('track', data), + 'transport', + 'LinkedIn lintrk call failed', + ); + } }, on(type, context) { @@ -170,23 +206,14 @@ export const destinationLinkedIn: Destination = { // LinkedIn has no vendor consent mode. The collector's config.consent gate // is the sole mechanism for blocking events. This handler exists for a // narrow case: config.loadScript was true BUT marketing consent was - // initially denied, so init() skipped addScript(). When consent later - // grants marketing, we inject the script. + // initially denied, so init() did not run. When consent later grants + // marketing, the script is injected; loadVendor injects it once per page. const consent = context.data as WalkerOS.Consent; const config = context.config; if (!config?.loadScript) return; if (consent.marketing !== true) return; - // Check whether the script tag is already in the DOM - addScript is not - // idempotent, and re-injecting would load the tag twice. - const { document } = getEnv(context.env); - const doc = document as Document; - const existing = doc.querySelector - ? doc.querySelector('script[src*="snap.licdn.com"]') - : null; - if (existing) return; - - addScript(context.env); + vendor(config, context.env as Env | undefined); }, }; diff --git a/packages/web/destinations/linkedin/src/schemas/settings.ts b/packages/web/destinations/linkedin/src/schemas/settings.ts index f3cce2e45..b5224a92b 100644 --- a/packages/web/destinations/linkedin/src/schemas/settings.ts +++ b/packages/web/destinations/linkedin/src/schemas/settings.ts @@ -1,16 +1,18 @@ import { z } from '@walkeros/core/dev'; -export const SettingsSchema = z.object({ - apiKey: z - .string() - .min(1) - .regex( - /^\d+$/, - 'Must be a numeric LinkedIn Partner ID (digits only, e.g. "123456")', - ) - .describe( - 'Your LinkedIn Partner ID (numeric string, typically 6 to 7 digits). Find it in Campaign Manager under Insight Tag. Assigned to window._linkedin_partner_id before the Insight Tag script loads (like 1234567).', - ), -}); +export const SettingsSchema = z + .object({ + apiKey: z + .string() + .min(1) + .regex( + /^\d+$/, + 'Must be a numeric LinkedIn Partner ID (digits only, e.g. "123456")', + ) + .describe( + 'Your LinkedIn Partner ID (numeric string, typically 6 to 7 digits). Find it in Campaign Manager under Insight Tag. Assigned to window._linkedin_partner_id before the Insight Tag script loads (like 1234567).', + ), + }) + .strict(); export type Settings = z.infer; diff --git a/packages/web/destinations/linkedin/src/types/index.ts b/packages/web/destinations/linkedin/src/types/index.ts index 7731b7147..5ddbce0bf 100644 --- a/packages/web/destinations/linkedin/src/types/index.ts +++ b/packages/web/destinations/linkedin/src/types/index.ts @@ -74,8 +74,6 @@ export interface Mapping { * `window._linkedin_partner_id` / `window._linkedin_data_partner_ids` and * installs `window.lintrk` in init; tests can pre-seed `window.lintrk` with * a spy to skip the script injection path. - * - * `document` is also mocked so `addScript()` can run headlessly. */ export interface Env extends DestinationWeb.Env { window: { diff --git a/packages/web/destinations/mixpanel/src/__tests__/conformance.test.ts b/packages/web/destinations/mixpanel/src/__tests__/conformance.test.ts new file mode 100644 index 000000000..6dbbee036 --- /dev/null +++ b/packages/web/destinations/mixpanel/src/__tests__/conformance.test.ts @@ -0,0 +1,66 @@ +import path from 'path'; +import { conformance } from '@walkeros/collector/dev'; +import { destinationMixpanel } from '..'; +import { examples, schemas } from '../dev'; + +// mixpanel-browser is ESM-style and Jest's CJS transformer can't parse it at +// import time. The conformance run wires examples.env, so the real module is +// never touched. +jest.mock('mixpanel-browser', () => ({ + __esModule: true, + default: {}, +})); + +// The mixpanel-browser config keys are forwarded into mp.init() as a whole. +const passThrough = [ + 'api_host', + 'persistence', + 'cross_subdomain_cookie', + 'cookie_expiration', + 'secure_cookie', + 'ip', + 'batch_requests', + 'batch_size', + 'batch_flush_interval_ms', + 'debug', + 'opt_out_tracking_by_default', + 'track_pageview', + 'autocapture', + 'record_sessions_percent', + 'record_mask_all_inputs', +]; + +describe('mixpanel conformance', () => { + beforeEach(() => { + jest.useRealTimers(); + }); + + it('conforms', async () => { + await conformance('destination', destinationMixpanel, { + profile: 'web-sdk', + examples: examples.step, + schema: schemas.settings, + settings: { + apiKey: 'a1b2c3d4e5f6789012345678abcdef12', + api_host: 'https://api-eu.mixpanel.com', + persistence: 'localStorage', + cross_subdomain_cookie: false, + cookie_expiration: 30, + secure_cookie: true, + ip: false, + batch_requests: true, + batch_size: 25, + batch_flush_interval_ms: 2000, + debug: false, + opt_out_tracking_by_default: false, + track_pageview: false, + autocapture: false, + record_sessions_percent: 0, + record_mask_all_inputs: true, + }, + env: { push: examples.env.push, simulation: examples.env.simulation }, + root: path.join(__dirname, '..', '..'), + passThrough, + }); + }); +}); diff --git a/packages/web/destinations/mixpanel/src/examples/step.ts b/packages/web/destinations/mixpanel/src/examples/step.ts index 5093a2f19..ab5c6f25b 100644 --- a/packages/web/destinations/mixpanel/src/examples/step.ts +++ b/packages/web/destinations/mixpanel/src/examples/step.ts @@ -441,3 +441,37 @@ export const consentGrantOptIn: MixpanelStepExample = { settings: {} as Partial, out: [['mixpanel.opt_in_tracking']], }; + +/** A silent rule sends no track call. */ +export const pageViewSilent: MixpanelStepExample = { + public: false, + description: 'A rule with silent: true sends no Mixpanel track call.', + in: getEvent('page view', { timestamp: 1700000108 }), + mapping: { silent: true }, + out: [], +}; + +/** + * The silent group profile rule of companyUpdateGroupProfile on an update + * without data: no group key and id resolve and the rule suppresses the + * track, so nothing is sent. + */ +export const companyUpdateWithoutData: MixpanelStepExample = { + public: false, + description: + 'A silent rule whose group profile mapping finds no group sends nothing, so the push fails with missing-input instead of counting as delivered.', + in: { name: 'company update', entity: 'company', action: 'update' }, + mapping: { + silent: true, + settings: { + groupProfile: { + map: { + key: { value: 'company_id' }, + id: 'data.company_id', + }, + }, + }, + }, + out: [], + outcome: { status: 'failed', code: 'missing-input' }, +}; diff --git a/packages/web/destinations/mixpanel/src/index.ts b/packages/web/destinations/mixpanel/src/index.ts index 3c856bd9a..c2d6ee513 100644 --- a/packages/web/destinations/mixpanel/src/index.ts +++ b/packages/web/destinations/mixpanel/src/index.ts @@ -5,6 +5,7 @@ import { isBoolean, isObject, isString, + StepError, } from '@walkeros/core'; import mixpanel from 'mixpanel-browser'; import type { @@ -28,21 +29,38 @@ function getMixpanel(env: Env | undefined): MixpanelSDK { return env?.mixpanel ?? (mixpanel as unknown as MixpanelSDK); } +/** Runs one Mixpanel call; a throw or a rejection becomes a StepError of `code`. */ +async function call( + fn: () => unknown, + code: string, + message: string, +): Promise { + try { + await fn(); + } catch (error) { + throw new StepError(message, { code, cause: error }); + } +} + /** * Resolve the distinctId from an identify mapping value and call * mixpanel.identify() only if the value changed since the last call. * Returns the updated lastIdentity snapshot. */ -function applyIdentify( +async function applyIdentify( mp: MixpanelSDK, resolved: Record, lastIdentity: RuntimeState['lastIdentity'] = {}, -): NonNullable { +): Promise> { const updated = { ...lastIdentity }; const distinctId = resolved.distinctId; if (!isString(distinctId) || distinctId === '') return updated; if (distinctId === lastIdentity.distinctId) return updated; - mp.identify(distinctId); + await call( + () => mp.identify(distinctId), + 'transport', + 'Mixpanel identify failed', + ); updated.distinctId = distinctId; return updated; } @@ -68,27 +86,50 @@ const PEOPLE_OBJECT_OPS: PeopleObjectOp[] = [ * Dispatch resolved people operations to the SDK. The resolved object * may contain any subset of the 8 ops. */ -function applyPeople(mp: MixpanelSDK, resolved: Record): void { +async function applyPeople( + mp: MixpanelSDK, + resolved: Record, +): Promise { + let applied = false; for (const op of PEOPLE_OBJECT_OPS) { const bag = resolved[op]; if (!isObject(bag)) continue; if (Object.keys(bag).length === 0) continue; // Typed dispatch - each op's first arg is a Record. - (mp.people[op] as (props: Record) => void)( - bag as Record, + await call( + () => + (mp.people[op] as (props: Record) => void)( + bag as Record, + ), + 'transport', + `Mixpanel people.${op} failed`, ); + applied = true; } if (isArray(resolved.unset)) { const names = (resolved.unset as unknown[]).filter((v): v is string => isString(v), ); - if (names.length > 0) mp.people.unset(names); + if (names.length > 0) + await call( + () => mp.people.unset(names), + 'transport', + 'Mixpanel people.unset failed', + ); + applied = names.length > 0 || applied; } if (resolved.delete_user === true) { - mp.people.delete_user(); + await call( + () => mp.people.delete_user(), + 'transport', + 'Mixpanel people.delete_user failed', + ); + applied = true; } + + return applied; } type GroupObjectOp = 'set' | 'set_once' | 'union' | 'remove'; @@ -100,12 +141,13 @@ const GROUP_OBJECT_OPS: GroupObjectOp[] = [ 'remove', ]; -function applyGroupProfile( +async function applyGroupProfile( mp: MixpanelSDK, resolved: Record, -): void { +): Promise { const { key, id } = resolved as { key?: unknown; id?: unknown }; - if (!isString(key) || key === '' || !isString(id) || id === '') return; + if (!isString(key) || key === '' || !isString(id) || id === '') return false; + let applied = false; const group = mp.get_group(key, id); @@ -113,21 +155,40 @@ function applyGroupProfile( const bag = resolved[op]; if (!isObject(bag)) continue; if (Object.keys(bag).length === 0) continue; - (group[op] as (props: Record) => void)( - bag as Record, + await call( + () => + (group[op] as (props: Record) => void)( + bag as Record, + ), + 'transport', + `Mixpanel group.${op} failed`, ); + applied = true; } if (isArray(resolved.unset)) { const names = (resolved.unset as unknown[]).filter((v): v is string => isString(v), ); - if (names.length > 0) group.unset(names); + if (names.length > 0) + await call( + () => group.unset(names), + 'transport', + 'Mixpanel group.unset failed', + ); + applied = names.length > 0 || applied; } if (resolved.delete === true) { - group.delete(); + await call( + () => group.delete(), + 'transport', + 'Mixpanel group.delete failed', + ); + applied = true; } + + return applied; } export const destinationMixpanel: Destination = { @@ -135,9 +196,11 @@ export const destinationMixpanel: Destination = { config: {}, - init({ config, env }) { - const settings = config.settings; - if (!settings?.apiKey) return false; + async init({ config, env }) { + // A copy, so the destructure below does not read keys off the live settings + const settings = { ...config.settings }; + if (!settings.apiKey) + throw new StepError('Mixpanel needs settings.apiKey', { code: 'config' }); const mp = getMixpanel(env as Env | undefined); // Destructure the walkerOS-specific keys out; the rest flow through to @@ -160,7 +223,11 @@ export const destinationMixpanel: Destination = { }; // Mixpanel's init() is synchronous (returns void). - mp.init(apiKey, initConfig); + await call( + () => mp.init(apiKey, initConfig), + 'vendor-not-ready', + 'Mixpanel init failed', + ); // Initialize runtime state. push() mutates this in place on subsequent // events to skip redundant identify() calls. @@ -173,6 +240,8 @@ export const destinationMixpanel: Destination = { const settings = (config.settings || {}) as Settings; const mappingSettings = rule?.settings || {}; const state: RuntimeState = settings._state || {}; + // Whether any Mixpanel call or state change came out of this event + let sent = false; // 1. Reset - fires first so subsequent identity calls start clean. if (mappingSettings.reset !== undefined) { @@ -180,8 +249,9 @@ export const destinationMixpanel: Destination = { ? mappingSettings.reset : await getMappingValue(event, mappingSettings.reset, { collector }); if (resolved) { - mp.reset(); + await call(() => mp.reset(), 'transport', 'Mixpanel reset failed'); state.lastIdentity = {}; + sent = true; } } @@ -192,11 +262,12 @@ export const destinationMixpanel: Destination = { collector, }); if (isObject(resolved)) { - state.lastIdentity = applyIdentify( + state.lastIdentity = await applyIdentify( mp, resolved as Record, state.lastIdentity, ); + sent = sent || isString(resolved.distinctId); } } @@ -206,7 +277,8 @@ export const destinationMixpanel: Destination = { collector, }); if (isObject(resolved)) { - applyPeople(mp, resolved as Record); + sent = + (await applyPeople(mp, resolved as Record)) || sent; } } @@ -224,7 +296,12 @@ export const destinationMixpanel: Destination = { (isString(id) || isArray(id)) && id !== '' ) { - mp.set_group(key, id as string | string[]); + await call( + () => mp.set_group(key, id as string | string[]), + 'transport', + 'Mixpanel set_group failed', + ); + sent = true; } } } @@ -237,7 +314,9 @@ export const destinationMixpanel: Destination = { { collector }, ); if (isObject(resolved)) { - applyGroupProfile(mp, resolved as Record); + sent = + (await applyGroupProfile(mp, resolved as Record)) || + sent; } } @@ -247,14 +326,28 @@ export const destinationMixpanel: Destination = { const properties = isObject(data) ? (data as Record) : {}; - mp.track(eventName, properties); + await call( + () => mp.track(eventName, properties), + 'transport', + 'Mixpanel track failed', + ); + sent = true; } + // A silent rule none of whose mappings resolved sent nothing at all + if (!sent && rule?.silent === true && Object.keys(mappingSettings).length) + throw new StepError( + 'Mixpanel mappings resolved to nothing for this event', + { + code: 'missing-input', + }, + ); + // Persist state mutations back onto config.settings. settings._state = state; }, - on(type, context) { + async on(type, context) { if (type !== 'consent' || !context.data) return; const mp = getMixpanel(context.env as Env | undefined); @@ -268,11 +361,11 @@ export const destinationMixpanel: Destination = { const allGranted = Object.keys(required).every( (key) => consent[key] === true, ); - if (allGranted) { - mp.opt_in_tracking(); - } else { - mp.opt_out_tracking(); - } + await call( + () => (allGranted ? mp.opt_in_tracking() : mp.opt_out_tracking()), + 'consent-not-applied', + 'Mixpanel did not take the consent update', + ); }, destroy({ config, env }) { diff --git a/packages/web/destinations/mixpanel/src/schemas/settings.ts b/packages/web/destinations/mixpanel/src/schemas/settings.ts index 9f0fdf1ea..5abcb871f 100644 --- a/packages/web/destinations/mixpanel/src/schemas/settings.ts +++ b/packages/web/destinations/mixpanel/src/schemas/settings.ts @@ -1,104 +1,106 @@ import { z } from '@walkeros/core/dev'; -export const SettingsSchema = z.object({ - apiKey: z - .string() - .min(1) - .describe( - 'Your Mixpanel project token. Find it in your Mixpanel project settings under "Project Settings" and "Access Keys". Passed to mixpanel.init() as the first argument (like a1b2c3d4e5f6789012345678abcdef12).', - ), - api_host: z - .string() - .describe( - 'Mixpanel ingestion host. Default: https://api-js.mixpanel.com. Use https://api-eu.mixpanel.com for EU residency.', - ) - .optional(), - persistence: z - .enum(['cookie', 'localStorage']) - .describe('Client-side persistence backend. Default: cookie.') - .optional(), - cross_subdomain_cookie: z - .boolean() - .describe('Share cookie across subdomains. Default: true.') - .optional(), - cookie_expiration: z - .number() - .int() - .nonnegative() - .describe('Cookie expiration in days. Default: 365.') - .optional(), - secure_cookie: z - .boolean() - .describe('Only send cookie over HTTPS. Default: false.') - .optional(), - ip: z - .boolean() - .describe('Enable server-side IP geolocation. Default: true.') - .optional(), - batch_requests: z - .boolean() - .describe( - 'Use the /batch endpoint instead of individual requests. Default: true.', - ) - .optional(), - batch_size: z - .number() - .int() - .positive() - .describe('Max events per batch. Default: 50.') - .optional(), - batch_flush_interval_ms: z - .number() - .int() - .positive() - .describe('Batch flush interval in ms. Default: 5000.') - .optional(), - debug: z - .boolean() - .describe('Enable verbose SDK logging. Default: false.') - .optional(), - opt_out_tracking_by_default: z - .boolean() - .describe( - 'Start in opted-out state until opt_in_tracking() is called. Default: false.', - ) - .optional(), - track_pageview: z - .union([z.boolean(), z.string()]) - .describe( - 'Enable Mixpanel auto-pageview tracking. walkerOS default: false (walkerOS sources handle page views).', - ) - .optional(), - autocapture: z - .unknown() - .describe( - 'Enable Mixpanel web autocapture. walkerOS default: false (walkerOS sources handle event capture).', - ) - .optional(), - record_sessions_percent: z - .number() - .min(0) - .max(100) - .describe( - 'Session replay sampling rate (0-100). Default: 0 (disabled). Session replay is bundled in the npm build.', - ) - .optional(), - record_mask_all_inputs: z - .boolean() - .describe('Mask all input values in session replay. Default: true.') - .optional(), - identify: z - .unknown() - .describe( - 'walkerOS mapping value resolving to an identity object. Keys: distinctId.', - ) - .optional(), - group: z - .unknown() - .describe( - 'walkerOS mapping value resolving to { key, id } → mixpanel.set_group(key, id). Resolved and sent with set_group on every push (set_group also updates the user profile). A rule-level group replaces it for that event.', - ) - .optional(), -}); +export const SettingsSchema = z + .object({ + apiKey: z + .string() + .min(1) + .describe( + 'Your Mixpanel project token. Find it in your Mixpanel project settings under "Project Settings" and "Access Keys". Passed to mixpanel.init() as the first argument (like a1b2c3d4e5f6789012345678abcdef12).', + ), + api_host: z + .string() + .describe( + 'Mixpanel ingestion host. Default: https://api-js.mixpanel.com. Use https://api-eu.mixpanel.com for EU residency.', + ) + .optional(), + persistence: z + .enum(['cookie', 'localStorage']) + .describe('Client-side persistence backend. Default: cookie.') + .optional(), + cross_subdomain_cookie: z + .boolean() + .describe('Share cookie across subdomains. Default: true.') + .optional(), + cookie_expiration: z + .number() + .int() + .nonnegative() + .describe('Cookie expiration in days. Default: 365.') + .optional(), + secure_cookie: z + .boolean() + .describe('Only send cookie over HTTPS. Default: false.') + .optional(), + ip: z + .boolean() + .describe('Enable server-side IP geolocation. Default: true.') + .optional(), + batch_requests: z + .boolean() + .describe( + 'Use the /batch endpoint instead of individual requests. Default: true.', + ) + .optional(), + batch_size: z + .number() + .int() + .positive() + .describe('Max events per batch. Default: 50.') + .optional(), + batch_flush_interval_ms: z + .number() + .int() + .positive() + .describe('Batch flush interval in ms. Default: 5000.') + .optional(), + debug: z + .boolean() + .describe('Enable verbose SDK logging. Default: false.') + .optional(), + opt_out_tracking_by_default: z + .boolean() + .describe( + 'Start in opted-out state until opt_in_tracking() is called. Default: false.', + ) + .optional(), + track_pageview: z + .union([z.boolean(), z.string()]) + .describe( + 'Enable Mixpanel auto-pageview tracking. walkerOS default: false (walkerOS sources handle page views).', + ) + .optional(), + autocapture: z + .unknown() + .describe( + 'Enable Mixpanel web autocapture. walkerOS default: false (walkerOS sources handle event capture).', + ) + .optional(), + record_sessions_percent: z + .number() + .min(0) + .max(100) + .describe( + 'Session replay sampling rate (0-100). Default: 0 (disabled). Session replay is bundled in the npm build.', + ) + .optional(), + record_mask_all_inputs: z + .boolean() + .describe('Mask all input values in session replay. Default: true.') + .optional(), + identify: z + .unknown() + .describe( + 'walkerOS mapping value resolving to an identity object. Keys: distinctId.', + ) + .optional(), + group: z + .unknown() + .describe( + 'walkerOS mapping value resolving to { key, id } → mixpanel.set_group(key, id). Resolved and sent with set_group on every push (set_group also updates the user profile). A rule-level group replaces it for that event.', + ) + .optional(), + }) + .strict(); export type Settings = z.infer; diff --git a/packages/web/destinations/optimizely/src/__tests__/conformance.test.ts b/packages/web/destinations/optimizely/src/__tests__/conformance.test.ts new file mode 100644 index 000000000..2e6c6b60c --- /dev/null +++ b/packages/web/destinations/optimizely/src/__tests__/conformance.test.ts @@ -0,0 +1,29 @@ +import path from 'path'; +import { conformance } from '@walkeros/collector/dev'; +import { destinationOptimizely } from '..'; +import { examples, schemas } from '../dev'; + +describe('optimizely conformance', () => { + beforeEach(() => { + jest.useRealTimers(); + }); + + it('conforms', async () => { + await conformance('destination', destinationOptimizely, { + profile: 'web-sdk', + examples: examples.step, + schema: schemas.settings, + settings: { + sdkKey: 'abc123SdkKey', + userId: 'user.id', + attributes: { map: { plan: 'user.plan' } }, + updateInterval: 60000, + autoUpdate: true, + batchSize: 10, + flushInterval: 1000, + }, + env: { push: examples.env.push, simulation: examples.env.simulation }, + root: path.join(__dirname, '..', '..'), + }); + }); +}); diff --git a/packages/web/destinations/optimizely/src/__tests__/stepExamples.test.ts b/packages/web/destinations/optimizely/src/__tests__/stepExamples.test.ts index 35fdeeb68..9aa8429d0 100644 --- a/packages/web/destinations/optimizely/src/__tests__/stepExamples.test.ts +++ b/packages/web/destinations/optimizely/src/__tests__/stepExamples.test.ts @@ -44,54 +44,56 @@ function spyEnv(): { env: Env; collected: () => CallRecord[] } { } describe('optimizely destination -- step examples', () => { - it.each<[string, OptimizelyStepExample]>(Object.entries(examples.step))( - '%s', - async (name, example) => { - const { env, collected } = spyEnv(); - const dest = jest.requireActual('../').default; - const { elb } = await startFlow(); + // Non-public examples (silent, failures) run in the conformance test. + it.each<[string, OptimizelyStepExample]>( + Object.entries(examples.step).filter( + ([, example]) => example.public !== false, + ), + )('%s', async (name, example) => { + const { env, collected } = spyEnv(); + const dest = jest.requireActual('../').default; + const { elb } = await startFlow(); - const baseSettings: Partial & { sdkKey: string } = { - sdkKey: 'test-sdk-key', - userId: 'user.id', - ...(example.settings || {}), - }; + const baseSettings: Partial & { sdkKey: string } = { + sdkKey: 'test-sdk-key', + userId: 'user.id', + ...(example.settings || {}), + }; - if (example.command === 'consent') { - await elb('walker destination', { - code: { ...dest, env }, - config: { - consent: { analytics: true }, - settings: baseSettings, - }, - }); - // Load the gated destination under a prior grant first when the example - // declares one, so a revoke acts on an already-granted destination. - if (example.before) await elb('walker consent', example.before); - await elb('walker consent', example.in as WalkerOS.Consent); - } else { - const event = example.in as WalkerOS.Event; - const mapping = example.mapping as WalkerOSMapping.Rule | undefined; - const mappingConfig = mapping - ? { [event.entity]: { [event.action]: mapping } } - : undefined; + if (example.command === 'consent') { + await elb('walker destination', { + code: { ...dest, env }, + config: { + consent: { analytics: true }, + settings: baseSettings, + }, + }); + // Load the gated destination under a prior grant first when the example + // declares one, so a revoke acts on an already-granted destination. + if (example.before) await elb('walker consent', example.before); + await elb('walker consent', example.in as WalkerOS.Consent); + } else { + const event = example.in as WalkerOS.Event; + const mapping = example.mapping as WalkerOSMapping.Rule | undefined; + const mappingConfig = mapping + ? { [event.entity]: { [event.action]: mapping } } + : undefined; - await elb('walker destination', { - code: { ...dest, env }, - config: { - settings: baseSettings, - mapping: mappingConfig, - }, - }); - await elb(event); - } + await elb('walker destination', { + code: { ...dest, env }, + config: { + settings: baseSettings, + mapping: mappingConfig, + }, + }); + await elb(event); + } - const expected = (example.out ?? []) as ReadonlyArray; - const actual = collected(); + const expected = (example.out ?? []) as ReadonlyArray; + const actual = collected(); - expect(actual).toEqual(expected); - }, - ); + expect(actual).toEqual(expected); + }); }); it('declares simulation paths that resolve', () => diff --git a/packages/web/destinations/optimizely/src/examples/step.ts b/packages/web/destinations/optimizely/src/examples/step.ts index f702924ab..ffd188550 100644 --- a/packages/web/destinations/optimizely/src/examples/step.ts +++ b/packages/web/destinations/optimizely/src/examples/step.ts @@ -183,3 +183,63 @@ export const consentGranted: OptimizelyStepExample = { settings: {} as Partial, out: [], }; + +/** A silent rule sends no trackEvent. */ +export const silentRule: OptimizelyStepExample = { + public: false, + description: + 'A rule with silent: true sends no trackEvent call; the user context the rule settings run on is still created.', + in: getEvent('page view', { timestamp: 1700000106 }), + mapping: { silent: true }, + out: [['optimizely.createInstance.createUserContext', 'us3r', {}]], +}; + +/** Page view without a user: settings.userId resolves to nothing, nothing is sent. */ +export const missingUserOnPageView: OptimizelyStepExample = { + public: false, + description: + 'Without a user id, settings.userId resolves to nothing: no user context can be created, so the push fails with missing-input instead of counting as delivered.', + in: { name: 'page view', entity: 'page', action: 'view' }, + out: [], + outcome: { status: 'failed', code: 'missing-input' }, +}; + +/** Product view without a user: settings.userId resolves to nothing, nothing is sent. */ +export const missingUserOnProductView: OptimizelyStepExample = { + public: false, + description: + 'Without a user id, settings.userId resolves to nothing: no user context can be created, so the push fails with missing-input instead of counting as delivered.', + in: { name: 'product view', entity: 'product', action: 'view' }, + out: [], + outcome: { status: 'failed', code: 'missing-input' }, +}; + +/** Order complete without a user: settings.userId resolves to nothing, nothing is sent. */ +export const missingUserOnOrderComplete: OptimizelyStepExample = { + public: false, + description: + 'Without a user id, settings.userId resolves to nothing: no user context can be created, so the push fails with missing-input instead of counting as delivered.', + in: { name: 'order complete', entity: 'order', action: 'complete' }, + out: [], + outcome: { status: 'failed', code: 'missing-input' }, +}; + +/** User signup without a user: settings.userId resolves to nothing, nothing is sent. */ +export const missingUserOnUserSignup: OptimizelyStepExample = { + public: false, + description: + 'Without a user id, settings.userId resolves to nothing: no user context can be created, so the push fails with missing-input instead of counting as delivered.', + in: { name: 'user signup', entity: 'user', action: 'signup' }, + out: [], + outcome: { status: 'failed', code: 'missing-input' }, +}; + +/** Profile update without a user: settings.userId resolves to nothing, nothing is sent. */ +export const missingUserOnProfileUpdate: OptimizelyStepExample = { + public: false, + description: + 'Without a user id, settings.userId resolves to nothing: no user context can be created, so the push fails with missing-input instead of counting as delivered.', + in: { name: 'profile update', entity: 'profile', action: 'update' }, + out: [], + outcome: { status: 'failed', code: 'missing-input' }, +}; diff --git a/packages/web/destinations/optimizely/src/index.ts b/packages/web/destinations/optimizely/src/index.ts index 3eafc7ff6..8508865e1 100644 --- a/packages/web/destinations/optimizely/src/index.ts +++ b/packages/web/destinations/optimizely/src/index.ts @@ -1,5 +1,11 @@ import type { WalkerOS } from '@walkeros/core'; -import { getMappingValue, isNumber, isObject, isString } from '@walkeros/core'; +import { + getMappingValue, + isNumber, + isObject, + isString, + StepError, +} from '@walkeros/core'; import * as optimizelySDK from '@optimizely/optimizely-sdk'; import type { Destination, @@ -21,6 +27,19 @@ function getOptimizely(env: Env | undefined): OptimizelySDKType { return optimizelySDK as unknown as OptimizelySDKType; } +/** Runs one Optimizely call; a throw or a rejection becomes a StepError of `code`. */ +async function call( + fn: () => T | Promise, + code: string, + message: string, +): Promise { + try { + return await fn(); + } catch (error) { + throw new StepError(message, { code, cause: error }); + } +} + export const destinationOptimizely: Destination = { type: 'optimizely', @@ -28,28 +47,54 @@ export const destinationOptimizely: Destination = { async init({ config, env }) { const settings = config.settings; - if (!settings?.sdkKey) return false; + if (!settings?.sdkKey) + throw new StepError('Optimizely needs settings.sdkKey', { + code: 'config', + }); const sdk = getOptimizely(env as Env | undefined); - const configManager = sdk.createPollingProjectConfigManager({ - sdkKey: settings.sdkKey, - autoUpdate: settings.autoUpdate ?? true, - updateInterval: settings.updateInterval ?? 60000, - }); + const { sdkKey } = settings; + const configManager = await call( + () => + sdk.createPollingProjectConfigManager({ + sdkKey, + autoUpdate: settings.autoUpdate ?? true, + updateInterval: settings.updateInterval ?? 60000, + }), + 'vendor-not-ready', + 'Optimizely config manager failed to start', + ); - const eventProcessor = sdk.createBatchEventProcessor({ - batchSize: settings.batchSize ?? 10, - flushInterval: settings.flushInterval ?? 1000, - }); + const eventProcessor = await call( + () => + sdk.createBatchEventProcessor({ + batchSize: settings.batchSize ?? 10, + flushInterval: settings.flushInterval ?? 1000, + }), + 'vendor-not-ready', + 'Optimizely event processor failed to start', + ); const instanceConfig: Record = { projectConfigManager: configManager, eventProcessor: eventProcessor, }; - const client = sdk.createInstance(instanceConfig); - await client.onReady(); + const client = await call( + () => sdk.createInstance(instanceConfig), + 'vendor-not-ready', + 'Optimizely client failed to start', + ); + const ready = await call( + () => client.onReady(), + 'vendor-not-ready', + 'Optimizely client did not become ready', + ); + if (isObject(ready) && ready.success === false) + throw new StepError('Optimizely client did not become ready', { + code: 'vendor-not-ready', + }); const _state: RuntimeState = { client }; @@ -61,7 +106,11 @@ export const destinationOptimizely: Destination = { const state: RuntimeState = settings._state || {}; const mappingSettings = rule?.settings || {}; - if (!state.client) return; + const { client } = state; + if (!client) + throw new StepError('Optimizely client is not running', { + code: 'not-ready', + }); // 1. Resolve userId const userIdMapping = settings.userId; @@ -74,7 +123,13 @@ export const destinationOptimizely: Destination = { } // Fallback: no userId means we cannot create a user context - if (!userId) return; + if (!userId) + throw new StepError( + 'Optimizely needs a userId, settings.userId resolved to nothing', + { + code: 'missing-input', + }, + ); // 2. Resolve destination-level attributes let baseAttributes: Record | undefined; @@ -89,17 +144,24 @@ export const destinationOptimizely: Destination = { // 3. Create or reuse user context if (userId !== state.lastUserId || !state.userContext) { - const userContext = state.client.createUserContext( - userId, - baseAttributes, + const userContext = await call( + () => client.createUserContext(userId, baseAttributes), + 'transport', + 'Optimizely createUserContext failed', ); - if (!userContext) return; + if (!userContext) + throw new StepError('Optimizely could not create a user context', { + code: 'transport', + }); state.userContext = userContext; state.lastUserId = userId; } const userContext = state.userContext; - if (!userContext) return; + if (!userContext) + throw new StepError('Optimizely has no user context', { + code: 'not-ready', + }); // 4. Per-event attributes via setAttribute if (mappingSettings.attributes !== undefined) { @@ -112,7 +174,11 @@ export const destinationOptimizely: Destination = { for (const [key, value] of Object.entries( resolved as Record, )) { - userContext.setAttribute(key, value); + await call( + () => userContext.setAttribute(key, value), + 'transport', + 'Optimizely setAttribute failed', + ); } } } @@ -156,14 +222,18 @@ export const destinationOptimizely: Destination = { } } - userContext.trackEvent(eventKey, eventTags); + await call( + () => userContext.trackEvent(eventKey, eventTags), + 'transport', + 'Optimizely trackEvent failed', + ); } // Persist state mutations settings._state = state; }, - on(type, context) { + async on(type, context) { if (type !== 'consent' || !context.data) return; const required = context.config?.consent; @@ -176,9 +246,14 @@ export const destinationOptimizely: Destination = { const state = (context.config?.settings as Settings)?._state; if (!allGranted && state?.client) { - state.client.close(); + const { client } = state; state.client = undefined; state.userContext = undefined; + await call( + () => client.close(), + 'consent-not-applied', + 'Optimizely did not stop on the consent revocation', + ); } }, diff --git a/packages/web/destinations/optimizely/src/schemas/settings.ts b/packages/web/destinations/optimizely/src/schemas/settings.ts index 1a0549ea2..0074dc090 100644 --- a/packages/web/destinations/optimizely/src/schemas/settings.ts +++ b/packages/web/destinations/optimizely/src/schemas/settings.ts @@ -1,56 +1,54 @@ import { z } from '@walkeros/core/dev'; -export const SettingsSchema = z.object({ - sdkKey: z - .string() - .min(1) - .describe( - 'Your Optimizely Feature Experimentation SDK key. Find it in your Optimizely project under Settings > Environments.', - ), - userId: z - .unknown() - .describe( - 'walkerOS mapping value to resolve userId for experiment bucketing. Example: "user.id" or { key: "user.id", value: "user.device" } for fallback.', - ) - .optional(), - attributes: z - .unknown() - .describe( - 'User attributes for audience targeting. Resolves to Record. Applied to every event via createUserContext().', - ) - .optional(), - updateInterval: z - .number() - .int() - .positive() - .describe( - 'Polling interval for datafile updates in milliseconds. Default: 60000.', - ) - .optional(), - autoUpdate: z - .boolean() - .describe('Auto-update datafile via polling. Default: true.') - .optional(), - batchSize: z - .number() - .int() - .positive() - .describe('Batch event processor: number of events per batch. Default: 10.') - .optional(), - flushInterval: z - .number() - .int() - .positive() - .describe( - 'Batch event processor: flush interval in milliseconds. Default: 1000.', - ) - .optional(), - skipOdp: z - .boolean() - .describe( - 'Skip Optimizely Data Platform (ODP) manager initialization. Default: true. Set to false only if you use ODP.', - ) - .optional(), -}); +export const SettingsSchema = z + .object({ + sdkKey: z + .string() + .min(1) + .describe( + 'Your Optimizely Feature Experimentation SDK key. Find it in your Optimizely project under Settings > Environments.', + ), + userId: z + .unknown() + .describe( + 'walkerOS mapping value to resolve userId for experiment bucketing. Example: "user.id" or { key: "user.id", value: "user.device" } for fallback.', + ) + .optional(), + attributes: z + .unknown() + .describe( + 'User attributes for audience targeting. Resolves to Record. Applied to every event via createUserContext().', + ) + .optional(), + updateInterval: z + .number() + .int() + .positive() + .describe( + 'Polling interval for datafile updates in milliseconds. Default: 60000.', + ) + .optional(), + autoUpdate: z + .boolean() + .describe('Auto-update datafile via polling. Default: true.') + .optional(), + batchSize: z + .number() + .int() + .positive() + .describe( + 'Batch event processor: number of events per batch. Default: 10.', + ) + .optional(), + flushInterval: z + .number() + .int() + .positive() + .describe( + 'Batch event processor: flush interval in milliseconds. Default: 1000.', + ) + .optional(), + }) + .strict(); export type Settings = z.infer; diff --git a/packages/web/destinations/optimizely/src/types/index.ts b/packages/web/destinations/optimizely/src/types/index.ts index 4e70f84df..8709174eb 100644 --- a/packages/web/destinations/optimizely/src/types/index.ts +++ b/packages/web/destinations/optimizely/src/types/index.ts @@ -22,8 +22,6 @@ export interface Settings { batchSize?: number; /** Batch event processor: flush interval in ms. Default: 1000. */ flushInterval?: number; - /** Skip ODP manager initialization. Default: true. */ - skipOdp?: boolean; /** Runtime state -- not user-facing. Mutated by init/push. */ _state?: RuntimeState; } diff --git a/packages/web/destinations/pinterest/src/__tests__/conformance.test.ts b/packages/web/destinations/pinterest/src/__tests__/conformance.test.ts new file mode 100644 index 000000000..02b3f64e3 --- /dev/null +++ b/packages/web/destinations/pinterest/src/__tests__/conformance.test.ts @@ -0,0 +1,24 @@ +import path from 'path'; +import { conformance } from '@walkeros/collector/dev'; +import { destinationPinterest } from '..'; +import { examples, schemas } from '../dev'; + +describe('pinterest conformance', () => { + beforeEach(() => { + jest.useRealTimers(); + }); + + it('conforms', async () => { + await conformance('destination', destinationPinterest, { + profile: 'vendor-global', + examples: examples.step, + schema: schemas.settings, + settings: { + apiKey: '2612345678901', + pageview: true, + }, + env: { push: examples.env.push, simulation: examples.env.simulation }, + root: path.join(__dirname, '..', '..'), + }); + }); +}); diff --git a/packages/web/destinations/pinterest/src/__tests__/init.test.ts b/packages/web/destinations/pinterest/src/__tests__/init.test.ts index 6ab130340..b0110e27b 100644 --- a/packages/web/destinations/pinterest/src/__tests__/init.test.ts +++ b/packages/web/destinations/pinterest/src/__tests__/init.test.ts @@ -63,13 +63,14 @@ describe('pinterest destination — init', () => { expect(calls).toEqual([['load', '2612345678901']]); }); - it('returns false when apiKey is missing', async () => { + it('throws a config error when apiKey is missing', async () => { const dest = jest.requireActual('../').default; const env = clone(examples.env.push) as Env; - const result = await dest.init({ - config: { settings: {} }, - env, - }); - expect(result).toBe(false); + await expect( + dest.init({ + config: { settings: {} }, + env, + }), + ).rejects.toMatchObject({ code: 'config' }); }); }); diff --git a/packages/web/destinations/pinterest/src/examples/step.ts b/packages/web/destinations/pinterest/src/examples/step.ts index bfb47cf67..9ac62fbca 100644 --- a/packages/web/destinations/pinterest/src/examples/step.ts +++ b/packages/web/destinations/pinterest/src/examples/step.ts @@ -326,6 +326,8 @@ export const identifyOnlySilent: Flow.StepExample = { */ export const consentRevoke: Flow.StepExample = { public: false, + description: + 'For a destination with config.consent { marketing: true }, revoking marketing makes no vendor call: the destination stops sending track calls itself.', command: 'consent', in: { marketing: false } as WalkerOS.Consent, out: [], @@ -340,3 +342,37 @@ export const consentGrant: Flow.StepExample = { in: { marketing: true } as WalkerOS.Consent, out: [], }; + +/** A silent rule sends no track call and no identity. */ +export const silentRule: Flow.StepExample = { + public: false, + description: 'A rule with silent: true sends no pintrk track call.', + in: createEvent({ name: 'page view', timestamp: 1700000109 }), + mapping: { silent: true }, + out: [], +}; + +/** + * The silent identify rule of identifyOnlySilent on an update without data: + * no identity resolves and the rule suppresses the track, so nothing is + * sent. + */ +export const identifyOnlySilentWithoutData: Flow.StepExample = { + public: false, + description: + 'A silent rule whose identify mapping finds no email or id sends nothing, so the push fails with missing-input instead of counting as delivered.', + in: { name: 'user update', entity: 'user', action: 'update' }, + mapping: { + silent: true, + settings: { + identify: { + map: { + em: 'data.email', + external_id: 'data.id', + }, + }, + }, + }, + out: [], + outcome: { status: 'failed', code: 'missing-input' }, +}; diff --git a/packages/web/destinations/pinterest/src/index.ts b/packages/web/destinations/pinterest/src/index.ts index e8fba9835..cf63dbe2c 100644 --- a/packages/web/destinations/pinterest/src/index.ts +++ b/packages/web/destinations/pinterest/src/index.ts @@ -1,6 +1,6 @@ import type { DestinationWeb } from '@walkeros/web-core'; -import { getEnv } from '@walkeros/web-core'; -import { getMappingValue, isObject, isString } from '@walkeros/core'; +import { getEnv, loadVendor } from '@walkeros/web-core'; +import { getMappingValue, isObject, isString, StepError } from '@walkeros/core'; import type { Collector, WalkerOS } from '@walkeros/core'; import type { Destination, @@ -34,16 +34,32 @@ function installPintrkQueue(env?: DestinationWeb.Env): void { } /** - * Insert the Pinterest core.js script tag. Async, non-blocking. Matches the - * plausible/meta precedent. + * The core.js script `loadVendor` injects unless `loadScript` is false. + * pintrk replays its own queue, so it is called synchronously and only a + * failed script is reported. */ -function addScript(env?: DestinationWeb.Env): void { - const { document } = getEnv(env); - const doc = document as Document; - const script = doc.createElement('script'); - script.src = PINTEREST_SCRIPT_SRC; - script.async = true; - doc.head.appendChild(script); +function vendor( + config: { loadScript?: boolean; timeout?: number }, + env?: DestinationWeb.Env, +) { + return loadVendor( + config.loadScript !== false ? PINTEREST_SCRIPT_SRC : undefined, + () => (getEnv(env).window as Window).pintrk, + { timeout: config.timeout }, + ); +} + +/** Runs one pintrk call; a throw or a rejection becomes a StepError of `code`. */ +async function call( + fn: () => unknown, + code: string, + message: string, +): Promise { + try { + await fn(); + } catch (error) { + throw new StepError(message, { code, cause: error }); + } } /** @@ -85,68 +101,104 @@ export const destinationPinterest: Destination = { config: {}, - init({ config, env }) { - const settings = config.settings; - if (!settings?.apiKey) return false; + async init({ config, env }) { + // A copy, so adding the runtime state below does not write onto the live settings + const settings = { ...config.settings }; + if (!settings.apiKey) + throw new StepError('Pinterest needs settings.apiKey', { + code: 'config', + }); + const { apiKey } = settings; const pageview = settings.pageview !== false; installPintrkQueue(env); - if (config.loadScript !== false) addScript(env); + vendor(config, env); const { window } = getEnv(env); const pintrk = (window as Window).pintrk as Pintrk; - pintrk('load', settings.apiKey); - if (pageview) pintrk('page'); + await call( + () => pintrk('load', apiKey), + 'vendor-not-ready', + 'Pinterest load call failed', + ); + if (pageview) + await call( + () => pintrk('page'), + 'vendor-not-ready', + 'Pinterest page call failed', + ); // Seed runtime state (mutation is allowed on Settings._state). - settings._state = { - lastIdentify: undefined, - consentGranted: true, + return { + ...config, + settings: { + ...settings, + _state: { lastIdentify: undefined, consentGranted: true }, + }, }; - - return config; }, async push(event, { config, rule, data, env, collector }) { + vendor(config, env).check(); + const settings = (config.settings || {}) as Settings; const state = (settings._state ||= { consentGranted: true }); - if (state.consentGranted === false) return; // Consent revoked - suppress const { window } = getEnv(env); const pintrk = (window as Window).pintrk as Pintrk; - if (!pintrk) return; - - // 1. Identity - per-event override falls back to destination-level. - const identifyMapping = - (rule?.settings?.identify as unknown) ?? settings.identify; - const identify = await resolveIdentify(event, identifyMapping, collector); - if (identify && !identifyEqual(identify, state.lastIdentify)) { - pintrk('set', identify); - state.lastIdentify = identify; - } - - // 2. Silent - process identity (above) but suppress the default track. - if (rule?.silent === true) return; - - // 3. Track - const eventName = isString(rule?.name) - ? (rule!.name as string) - : event.name; - - // context.data contains both include-flattened and mapping.data-resolved - // properties. Include is the bottom layer, mapping.data wins on conflict. - const contextData: Record = isObject(data) - ? (data as Record) - : {}; - - const eventData: EventData = { - ...(event.id ? { event_id: event.id } : {}), - ...contextData, - }; + if (!pintrk) + throw new StepError('Pinterest Tag not ready: pintrk is missing', { + code: 'vendor-not-ready', + }); + + // Consent revoked: suppress. This is the consent state, not a failure. + if (state.consentGranted !== false) { + // 1. Identity - per-event override falls back to destination-level. + const identifyMapping = + (rule?.settings?.identify as unknown) ?? settings.identify; + const identify = await resolveIdentify(event, identifyMapping, collector); + // A silent rule that resolves no identity sends nothing at all + if (!identify && rule?.silent === true && identifyMapping !== undefined) + throw new StepError( + 'Pinterest identify resolved to nothing for this event', + { code: 'missing-input' }, + ); + if (identify && !identifyEqual(identify, state.lastIdentify)) { + await call( + () => pintrk('set', identify), + 'transport', + 'Pinterest set call failed', + ); + state.lastIdentify = identify; + } - pintrk('track', eventName, eventData); + // 2. Silent - process identity (above) but suppress the default track. + if (rule?.silent !== true) { + // 3. Track + const eventName = isString(rule?.name) + ? (rule!.name as string) + : event.name; + + // context.data contains both include-flattened and mapping.data-resolved + // properties. Include is the bottom layer, mapping.data wins on conflict. + const contextData: Record = isObject(data) + ? (data as Record) + : {}; + + const eventData: EventData = { + ...(event.id ? { event_id: event.id } : {}), + ...contextData, + }; + + await call( + () => pintrk('track', eventName, eventData), + 'transport', + 'Pinterest track call failed', + ); + } + } }, on(type, context) { diff --git a/packages/web/destinations/pinterest/src/schemas/settings.ts b/packages/web/destinations/pinterest/src/schemas/settings.ts index beedab241..e4209b61c 100644 --- a/packages/web/destinations/pinterest/src/schemas/settings.ts +++ b/packages/web/destinations/pinterest/src/schemas/settings.ts @@ -1,25 +1,27 @@ import { z } from '@walkeros/core/dev'; -export const SettingsSchema = z.object({ - apiKey: z - .string() - .regex(/^\d+$/, 'Must be a numeric Pinterest Tag ID') - .min(6) - .describe( - 'Your Pinterest Tag ID (numeric string, e.g. "2612345678901"). Found in Pinterest Ads Manager under Conversions → Pinterest Tag. Passed to pintrk("load", tagId).', - ), - pageview: z - .boolean() - .describe( - 'Fire pintrk("page") once in init after core.js loads. Default true (matches Pinterest\'s base code convention). Set false when walkerOS sources already emit page view events and you would otherwise get a duplicate initial fire.', - ) - .optional(), - identify: z - .unknown() - .describe( - 'walkerOS mapping value resolving to enhanced matching fields: { em?: string, external_id?: string }. em is auto-hashed by the Pinterest JS tag — do not hash before passing. Passed to pintrk("load", tagId, data) on init and pintrk("set", data) when the resolved value changes.', - ) - .optional(), -}); +export const SettingsSchema = z + .object({ + apiKey: z + .string() + .regex(/^\d+$/, 'Must be a numeric Pinterest Tag ID') + .min(6) + .describe( + 'Your Pinterest Tag ID (numeric string, e.g. "2612345678901"). Found in Pinterest Ads Manager under Conversions → Pinterest Tag. Passed to pintrk("load", tagId).', + ), + pageview: z + .boolean() + .describe( + 'Fire pintrk("page") once in init after core.js loads. Default true (matches Pinterest\'s base code convention). Set false when walkerOS sources already emit page view events and you would otherwise get a duplicate initial fire.', + ) + .optional(), + identify: z + .unknown() + .describe( + 'walkerOS mapping value resolving to enhanced matching fields: { em?: string, external_id?: string }. em is auto-hashed by the Pinterest JS tag — do not hash before passing. Passed to pintrk("load", tagId, data) on init and pintrk("set", data) when the resolved value changes.', + ) + .optional(), + }) + .strict(); export type Settings = z.infer; diff --git a/packages/web/destinations/plausible/src/__tests__/conformance.test.ts b/packages/web/destinations/plausible/src/__tests__/conformance.test.ts new file mode 100644 index 000000000..ef9cf9c8d --- /dev/null +++ b/packages/web/destinations/plausible/src/__tests__/conformance.test.ts @@ -0,0 +1,21 @@ +import path from 'path'; +import { conformance } from '@walkeros/collector/dev'; +import { destinationPlausible } from '..'; +import { examples, schemas } from '../dev'; + +describe('plausible conformance', () => { + beforeEach(() => { + jest.useRealTimers(); + }); + + it('conforms', async () => { + await conformance('destination', destinationPlausible, { + profile: 'vendor-global', + examples: examples.step, + schema: schemas.settings, + settings: { domain: 'example.com' }, + env: { push: examples.env.push, simulation: examples.env.simulation }, + root: path.join(__dirname, '..', '..'), + }); + }); +}); diff --git a/packages/web/destinations/plausible/src/__tests__/stepExamples.test.ts b/packages/web/destinations/plausible/src/__tests__/stepExamples.test.ts index b335642dc..d5b1c3948 100644 --- a/packages/web/destinations/plausible/src/__tests__/stepExamples.test.ts +++ b/packages/web/destinations/plausible/src/__tests__/stepExamples.test.ts @@ -16,12 +16,7 @@ const initOut = (examples.step.init.out ?? []) as ReadonlyArray; const noopLogger = createLogger(); -/** - * Unified spy: captures both init-side `script.appendChild` calls and - * push-side `plausible(...)` calls into a single ordered array. This - * lets event tests bootstrap with the real init (triggering - * loadScript) and slice off `initOut.length` to isolate push calls. - */ +/** Spy: captures push-side `plausible(...)` calls into an ordered array. */ function spyEnv(): { env: Env; collected: () => CallRecord[] } { const calls: CallRecord[] = []; const spy = ( @@ -40,25 +35,6 @@ function spyEnv(): { env: Env; collected: () => CallRecord[] } { q: [] as IArguments[], }), }, - document: { - createElement: (_tag: string) => ({ src: '', dataset: {} }), - head: { - appendChild: (script: { - src: string; - dataset: { domain?: string }; - }) => { - calls.push([ - 'script.appendChild', - { - src: script.src, - domain: script.dataset.domain, - }, - ]); - return script; - }, - }, - querySelector: () => null, - }, }; return { env, collected: () => calls }; } @@ -85,7 +61,17 @@ describe('plausible destination -- step examples', () => { collector: {} as Destination.Context['collector'], }); - expect(collected()).toEqual(initOut); + // loadVendor injects into the page's document + const scripts = Array.from( + document.querySelectorAll( + 'script[data-walkeros-vendor]', + ), + ).map((script) => [ + 'script.appendChild', + { src: script.src, domain: script.dataset.domain }, + ]); + + expect([...scripts, ...collected()]).toEqual(initOut); }); // Plausible takes scalar custom props and revenue as { currency, amount } @@ -136,7 +122,7 @@ describe('plausible destination -- step examples', () => { await elb(event); const expected = (example.out ?? []) as ReadonlyArray; - const actual = collected().slice(initOut.length); + const actual = collected(); expect(actual).toEqual(expected); }); diff --git a/packages/web/destinations/plausible/src/examples/step.ts b/packages/web/destinations/plausible/src/examples/step.ts index 7a3329666..c8f3c9a82 100644 --- a/packages/web/destinations/plausible/src/examples/step.ts +++ b/packages/web/destinations/plausible/src/examples/step.ts @@ -92,3 +92,26 @@ export const customEvent: Flow.StepExample = { ], ], }; + +/** A silent rule sends no Plausible event. */ +export const silentRule: Flow.StepExample = { + public: false, + description: 'A rule with silent: true sends no Plausible event.', + in: getEvent('page view', { timestamp: 1700000600 }), + mapping: { silent: true }, + out: [], +}; + +/** + * A revocation of the consent the destination requires. Plausible has no + * consent mode, so the collector's consent gate is the only mechanism and no + * vendor call is made. + */ +export const consentRevoke: Flow.StepExample = { + public: false, + description: + 'For a destination with config.consent { marketing: true }, revoking marketing makes no vendor call: Plausible has no consent mode, the collector stops sending.', + command: 'consent', + in: { marketing: false }, + out: [], +}; diff --git a/packages/web/destinations/plausible/src/index.test.ts b/packages/web/destinations/plausible/src/index.test.ts index c7925f2c0..b3aace5ab 100644 --- a/packages/web/destinations/plausible/src/index.test.ts +++ b/packages/web/destinations/plausible/src/index.test.ts @@ -66,76 +66,24 @@ describe('destination plausible', () => { expect(initEnv?.window.plausible).toBeDefined(); }); - test('init with script load', async () => { - // For now, skip complex document mocking and focus on core functionality - // TODO: Fix document mocking with environment injection - const mockPlausible = jest.fn(); - const simpleEnv = { - ...testEnv, - window: { - plausible: mockPlausible, - }, - document: { - createElement: createElementMock, - head: { appendChild: appendChildMock }, - querySelector: jest.fn(), - }, - }; - - const destinationWithEnv = { - ...destination, - env: simpleEnv, - }; - elb('walker destination', { - code: destinationWithEnv, - config: { loadScript: true }, - }); - - await elb(event); - - // Verify script createElement was called - expect(createElementMock).toHaveBeenCalledWith('script'); - // Verify appendChild was called - expect(appendChildMock).toHaveBeenCalled(); - }); - - test('init with domain', async () => { + // loadVendor injects into the page's document, once per src + test('init with script load and domain', async () => { const domain = 'walkeros.io'; - const mockScript = { - src: '', - dataset: {} as Record, - setAttribute: jest.fn(), - removeAttribute: jest.fn(), - }; - createElementMock.mockReturnValue(mockScript); - - const mockPlausible = jest.fn(); - const simpleEnv = { - ...testEnv, - window: { - plausible: mockPlausible, - }, - document: { - createElement: createElementMock, - head: { appendChild: appendChildMock }, - querySelector: jest.fn(), - }, - }; - const destinationWithEnv = { ...destination, - env: simpleEnv, + env: { window: { plausible: jest.fn() } }, }; elb('walker destination', { code: destinationWithEnv, - config: { - loadScript: true, - settings: { domain }, - }, + config: { loadScript: true, settings: { domain } }, }); await elb(event); - expect((mockScript.dataset as Record).domain).toBe(domain); + const injected = document.querySelector( + 'script[data-walkeros-vendor]', + ); + expect(injected?.src).toBe(script); + expect(injected?.dataset.domain).toBe(domain); }); }); diff --git a/packages/web/destinations/plausible/src/index.ts b/packages/web/destinations/plausible/src/index.ts index 385f1bbc7..c799a2c0f 100644 --- a/packages/web/destinations/plausible/src/index.ts +++ b/packages/web/destinations/plausible/src/index.ts @@ -1,10 +1,12 @@ import type { Config, Destination, Env } from './types'; -import { isObject } from '@walkeros/core'; -import { getEnv } from '@walkeros/web-core'; +import { isObject, StepError } from '@walkeros/core'; +import { getEnv, loadVendor } from '@walkeros/web-core'; // Types export * as DestinationPlausible from './types'; +const SCRIPT_SRC = 'https://plausible.io/js/script.manual.js'; + export const destinationPlausible: Destination = { type: 'plausible', @@ -14,7 +16,7 @@ export const destinationPlausible: Destination = { const { window } = getEnv(env); const settings = config.settings || {}; - if (config.loadScript) addScript(settings.domain, env); + vendor(config, settings.domain, env); window.plausible = window.plausible || @@ -25,24 +27,48 @@ export const destinationPlausible: Destination = { return config; }, - push(event, { config, data, env }) { - const params = isObject(data) ? data : {}; + async push(event, { config, rule, data, env }) { + vendor(config, config.settings?.domain, env).check(); const { window } = getEnv(env); - window.plausible!(`${event.name}`, params); + const { plausible } = window; + if (!plausible) + throw new StepError('Plausible not ready: window.plausible is missing', { + code: 'vendor-not-ready', + }); + + // A silent rule sends no Plausible event + if (!rule?.silent) { + const params = isObject(data) ? data : {}; + try { + await plausible(`${event.name}`, params); + } catch (error) { + throw new StepError('Plausible call failed', { + code: 'transport', + cause: error, + }); + } + } }, }; -function addScript( - domain?: string, +/** + * The script `loadVendor` injects with `loadScript`. plausible replays its own + * queue, so it is called synchronously and only a failed script is reported. + */ +function vendor( + config: { loadScript?: boolean; timeout?: number }, + domain: string | undefined, env?: Env, - src = 'https://plausible.io/js/script.manual.js', ) { - const { document } = getEnv(env); - const script = document.createElement('script'); - script.src = src; - if (domain) script.dataset.domain = domain; - document.head.appendChild(script); + return loadVendor( + config.loadScript ? SCRIPT_SRC : undefined, + () => getEnv(env).window.plausible, + { + timeout: config.timeout, + attributes: domain ? { 'data-domain': domain } : undefined, + }, + ); } export default destinationPlausible; diff --git a/packages/web/destinations/plausible/src/schemas/settings.ts b/packages/web/destinations/plausible/src/schemas/settings.ts index fbb45147b..9575c8d62 100644 --- a/packages/web/destinations/plausible/src/schemas/settings.ts +++ b/packages/web/destinations/plausible/src/schemas/settings.ts @@ -3,14 +3,16 @@ import { z } from '@walkeros/core/dev'; const hostnameRegex = /^(?=.{1,253}$)(?!-)[A-Za-z0-9-]{1,63}(?; diff --git a/packages/web/destinations/posthog/src/__tests__/conformance.test.ts b/packages/web/destinations/posthog/src/__tests__/conformance.test.ts new file mode 100644 index 000000000..03071130f --- /dev/null +++ b/packages/web/destinations/posthog/src/__tests__/conformance.test.ts @@ -0,0 +1,68 @@ +import path from 'path'; +import { conformance } from '@walkeros/collector/dev'; +import { destinationPostHog } from '..'; +import { examples, schemas } from '../dev'; + +// posthog-js is ESM-first and Jest's CJS transformer can't parse it at import +// time. The conformance run wires examples.env, so the real module is never +// touched. +jest.mock('posthog-js', () => ({ + __esModule: true, + default: {}, +})); + +// The posthog-js options are forwarded into posthog.init() as a whole. +const passThrough = [ + 'api_host', + 'ui_host', + 'persistence', + 'person_profiles', + 'autocapture', + 'capture_pageview', + 'capture_pageleave', + 'capture_heatmaps', + 'capture_exceptions', + 'disable_surveys', + 'disable_session_recording', + 'advanced_disable_flags', + 'cookieless_mode', + 'debug', + 'session_recording', + 'bootstrap', +]; + +describe('posthog conformance', () => { + beforeEach(() => { + jest.useRealTimers(); + }); + + it('conforms', async () => { + await conformance('destination', destinationPostHog, { + profile: 'web-sdk', + examples: examples.step, + schema: schemas.settings, + settings: { + apiKey: 'phc_a1b2c3d4e5f6789012345678abcdef12', + api_host: 'https://eu.i.posthog.com', + ui_host: 'https://eu.posthog.com', + persistence: 'localStorage', + person_profiles: 'identified_only', + autocapture: false, + capture_pageview: false, + capture_pageleave: false, + capture_heatmaps: false, + capture_exceptions: false, + disable_surveys: true, + disable_session_recording: true, + advanced_disable_flags: false, + cookieless_mode: 'on_reject', + debug: false, + session_recording: { maskAllInputs: true }, + bootstrap: { distinctID: 'user-1' }, + }, + env: { push: examples.env.push, simulation: examples.env.simulation }, + root: path.join(__dirname, '..', '..'), + passThrough, + }); + }); +}); diff --git a/packages/web/destinations/posthog/src/examples/step.ts b/packages/web/destinations/posthog/src/examples/step.ts index 38398a133..d89ae2eba 100644 --- a/packages/web/destinations/posthog/src/examples/step.ts +++ b/packages/web/destinations/posthog/src/examples/step.ts @@ -364,3 +364,37 @@ export const consentGrantOptIn: PostHogStepExample = { settings: {} as Partial, out: [['posthog.opt_in_capturing']], }; + +/** A silent rule sends no capture call. */ +export const pageViewSilent: PostHogStepExample = { + public: false, + description: 'A rule with silent: true sends no PostHog capture call.', + in: getEvent('page view', { timestamp: 1700000109 }), + mapping: { silent: true }, + out: [], +}; + +/** + * The silent group rule of groupAssignmentWithProperties on an update + * without data: no group type and key resolve and the rule suppresses the + * capture, so nothing is sent. + */ +export const companyUpdateWithoutData: PostHogStepExample = { + public: false, + description: + 'A silent rule whose group mapping finds no group sends nothing, so the push fails with missing-input instead of counting as delivered.', + in: { name: 'company update', entity: 'company', action: 'update' }, + mapping: { + silent: true, + settings: { + group: { + map: { + type: { value: 'company' }, + key: 'data.company_id', + }, + }, + }, + }, + out: [], + outcome: { status: 'failed', code: 'missing-input' }, +}; diff --git a/packages/web/destinations/posthog/src/index.ts b/packages/web/destinations/posthog/src/index.ts index 2aa0e7d7f..8f59d7d45 100644 --- a/packages/web/destinations/posthog/src/index.ts +++ b/packages/web/destinations/posthog/src/index.ts @@ -1,5 +1,5 @@ import type { WalkerOS } from '@walkeros/core'; -import { getMappingValue, isObject, isString } from '@walkeros/core'; +import { getMappingValue, isObject, isString, StepError } from '@walkeros/core'; import posthogSDK from 'posthog-js'; import type { Destination, @@ -22,6 +22,19 @@ function getPostHog(env: Env | undefined): PostHogSDK { return env?.posthog ?? (posthogSDK as unknown as PostHogSDK); } +/** Runs one PostHog call; a throw or a rejection becomes a StepError of `code`. */ +async function call( + fn: () => unknown, + code: string, + message: string, +): Promise { + try { + await fn(); + } catch (error) { + throw new StepError(message, { code, cause: error }); + } +} + /** * Resolve the identify mapping value, call the right SDK method. * @@ -33,11 +46,15 @@ function getPostHog(env: Env | undefined): PostHogSDK { * * Returns the updated lastIdentity snapshot. */ -function applyIdentity( +async function applyIdentity( posthog: PostHogSDK, resolved: Record, lastIdentity: RuntimeState['lastIdentity'] = {}, -): NonNullable { +): Promise<{ + lastIdentity: NonNullable; + applied: boolean; +}> { + let applied = false; const updated = { ...lastIdentity }; const distinctId = resolved.distinctId; @@ -50,14 +67,24 @@ function applyIdentity( if (isString(distinctId) && distinctId !== '') { if (distinctId !== lastIdentity.distinctId || $set || $setOnce) { - posthog.identify(distinctId, $set, $setOnce); + await call( + () => posthog.identify(distinctId, $set, $setOnce), + 'transport', + 'PostHog identify failed', + ); updated.distinctId = distinctId; } + applied = true; } else if ($set || $setOnce) { - posthog.setPersonProperties($set, $setOnce); + await call( + () => posthog.setPersonProperties($set, $setOnce), + 'transport', + 'PostHog setPersonProperties failed', + ); + applied = true; } - return updated; + return { lastIdentity: updated, applied }; } /** @@ -65,11 +92,15 @@ function applyIdentity( * Like identity, group changes are tracked in runtime state and only re-fired * when type or key changes. */ -function applyGroup( +async function applyGroup( posthog: PostHogSDK, resolved: Record, lastGroup: RuntimeState['lastGroup'] = {}, -): NonNullable { +): Promise<{ + lastGroup: NonNullable; + applied: boolean; +}> { + let applied = false; const updated = { ...lastGroup }; const type = resolved.type; const key = resolved.key; @@ -79,13 +110,18 @@ function applyGroup( if (isString(type) && isString(key)) { if (type !== lastGroup.type || key !== lastGroup.key || properties) { - posthog.group(type, key, properties); + await call( + () => posthog.group(type, key, properties), + 'transport', + 'PostHog group failed', + ); updated.type = type; updated.key = key; } + applied = true; } - return updated; + return { lastGroup: updated, applied }; } export const destinationPostHog: Destination = { @@ -94,8 +130,10 @@ export const destinationPostHog: Destination = { config: {}, async init({ config, env }) { - const settings = config.settings; - if (!settings?.apiKey) return false; + // A copy, so the destructure below does not read keys off the live settings + const settings = { ...config.settings }; + if (!settings.apiKey) + throw new StepError('PostHog needs settings.apiKey', { code: 'config' }); const posthog = getPostHog(env as Env | undefined); @@ -122,12 +160,17 @@ export const destinationPostHog: Destination = { // destination is truly ready before returning; otherwise we resolve // immediately so tests with no-op mocks don't hang. let loadedFired = false; - posthog.init(apiKey, { - ...options, - loaded: () => { - loadedFired = true; - }, - }); + await call( + () => + posthog.init(apiKey, { + ...options, + loaded: () => { + loadedFired = true; + }, + }), + 'vendor-not-ready', + 'PostHog init failed', + ); if (!loadedFired) { await Promise.resolve(); } @@ -141,6 +184,8 @@ export const destinationPostHog: Destination = { const settings = (config.settings || {}) as Settings; const mappingSettings = rule?.settings || {}; const state: RuntimeState = settings._state || {}; + // Whether any PostHog call or state change came out of this event + let sent = false; // 1. Reset - fires first so subsequent identity calls start clean. if (mappingSettings.reset !== undefined) { @@ -151,9 +196,10 @@ export const destinationPostHog: Destination = { collector, }); if (resolved) { - posthog.reset(); + await call(() => posthog.reset(), 'transport', 'PostHog reset failed'); state.lastIdentity = {}; state.lastGroup = {}; + sent = true; } } @@ -164,11 +210,13 @@ export const destinationPostHog: Destination = { collector, }); if (isObject(resolved)) { - state.lastIdentity = applyIdentity( + const identity = await applyIdentity( posthog, resolved as Record, state.lastIdentity, ); + state.lastIdentity = identity.lastIdentity; + sent = sent || identity.applied; } } @@ -179,11 +227,13 @@ export const destinationPostHog: Destination = { collector, }); if (isObject(resolved)) { - state.lastGroup = applyGroup( + const group = await applyGroup( posthog, resolved as Record, state.lastGroup, ); + state.lastGroup = group.lastGroup; + sent = sent || group.applied; } } @@ -193,15 +243,29 @@ export const destinationPostHog: Destination = { const properties = isObject(data) ? (data as Record) : {}; - posthog.capture(eventName, properties); + await call( + () => posthog.capture(eventName, properties), + 'transport', + 'PostHog capture failed', + ); + sent = true; } + // A silent rule none of whose mappings resolved sent nothing at all + if (!sent && rule?.silent === true && Object.keys(mappingSettings).length) + throw new StepError( + 'PostHog mappings resolved to nothing for this event', + { + code: 'missing-input', + }, + ); + // Persist state. walkerOS destinations are long-lived singletons so // mutating settings in place is safe (Snowplow precedent). settings._state = state; }, - on(type, context) { + async on(type, context) { if (type !== 'consent' || !context?.data) return; const posthog = getPostHog(context.env as Env | undefined); @@ -216,11 +280,12 @@ export const destinationPostHog: Destination = { const allGranted = Object.keys(required).every( (key) => consent[key] === true, ); - if (allGranted) { - posthog.opt_in_capturing(); - } else { - posthog.opt_out_capturing(); - } + await call( + () => + allGranted ? posthog.opt_in_capturing() : posthog.opt_out_capturing(), + 'consent-not-applied', + 'PostHog did not take the consent update', + ); }, }; diff --git a/packages/web/destinations/posthog/src/schemas/settings.ts b/packages/web/destinations/posthog/src/schemas/settings.ts index e0f3c20cf..0807ece14 100644 --- a/packages/web/destinations/posthog/src/schemas/settings.ts +++ b/packages/web/destinations/posthog/src/schemas/settings.ts @@ -1,116 +1,120 @@ import { z } from '@walkeros/core/dev'; -export const SettingsSchema = z.object({ - apiKey: z - .string() - .min(1) - .describe( - 'Your PostHog project API key (starts with "phc_"). Find it in your PostHog project settings under "Project API Key" (like phc_a1b2c3d4e5f6789012345678abcdef12).', - ), - api_host: z - .string() - .describe( - 'PostHog API host. Defaults to https://us.i.posthog.com. Use https://eu.i.posthog.com for the EU region.', - ) - .optional(), - ui_host: z - .string() - .describe( - 'URL of the PostHog UI host, used by toolbar and replay player. Default: null.', - ) - .optional(), - persistence: z - .enum([ - 'localStorage+cookie', - 'cookie', - 'localStorage', - 'sessionStorage', - 'memory', - ]) - .describe('Where to persist user identity. Default: "localStorage+cookie".') - .optional(), - person_profiles: z - .enum(['always', 'never', 'identified_only']) - .describe( - 'When to create person profiles. "identified_only" (default) creates profiles only after identify() is called.', - ) - .optional(), - autocapture: z - .boolean() - .describe( - "Enable PostHog's autocapture of clicks, form submits, and pageviews. Default: false (walkerOS sources handle capture).", - ) - .optional(), - capture_pageview: z - .union([z.boolean(), z.literal('history_change')]) - .describe( - 'Whether PostHog should auto-capture pageview events. Default: false (walkerOS sources handle this).', - ) - .optional(), - capture_pageleave: z - .union([z.boolean(), z.literal('if_capture_pageview')]) - .describe( - 'Whether PostHog should auto-capture pageleave events. Default: false.', - ) - .optional(), - capture_heatmaps: z - .boolean() - .describe( - 'Enable heatmap data capture. Default: unset, which follows the heatmaps setting of the PostHog project.', - ) - .optional(), - capture_exceptions: z - .boolean() - .describe( - 'Enable automatic exception capture. Default: unset, which follows the exception autocapture setting of the PostHog project.', - ) - .optional(), - disable_surveys: z - .boolean() - .describe('Disable PostHog surveys. Default: false.') - .optional(), - disable_session_recording: z - .boolean() - .describe('Disable session recording. Default: false.') - .optional(), - advanced_disable_flags: z - .boolean() - .describe('Disable feature flag loading. Default: false.') - .optional(), - cookieless_mode: z - .enum(['always', 'on_reject']) - .describe( - '"always" never uses cookies/localStorage. "on_reject" falls back to cookieless when consent is rejected.', - ) - .optional(), - debug: z - .boolean() - .describe('Enable PostHog SDK debug logging. Default: false.') - .optional(), - session_recording: z - .unknown() - .describe( - 'Session Replay options (SessionRecordingOptions). Masking, blocking, sampling. See PostHog docs for full list.', - ) - .optional(), - bootstrap: z - .unknown() - .describe( - 'SSR bootstrap data. { distinctID?, featureFlags? } pre-populates identity and flag values to avoid first-render flicker.', - ) - .optional(), - identify: z - .unknown() - .describe( - 'walkerOS mapping value resolving to an identity object. Keys: distinctId, $set, $set_once. Resolved on every push: posthog.identify fires when distinctId changes, and on every push while $set or $set_once are present.', - ) - .optional(), - group: z - .unknown() - .describe( - 'walkerOS mapping value resolving to a group object. Keys: type, key, properties. Resolved on every push: posthog.group fires when type or key changes, and on every push while properties are present.', - ) - .optional(), -}); +export const SettingsSchema = z + .object({ + apiKey: z + .string() + .min(1) + .describe( + 'Your PostHog project API key (starts with "phc_"). Find it in your PostHog project settings under "Project API Key" (like phc_a1b2c3d4e5f6789012345678abcdef12).', + ), + api_host: z + .string() + .describe( + 'PostHog API host. Defaults to https://us.i.posthog.com. Use https://eu.i.posthog.com for the EU region.', + ) + .optional(), + ui_host: z + .string() + .describe( + 'URL of the PostHog UI host, used by toolbar and replay player. Default: null.', + ) + .optional(), + persistence: z + .enum([ + 'localStorage+cookie', + 'cookie', + 'localStorage', + 'sessionStorage', + 'memory', + ]) + .describe( + 'Where to persist user identity. Default: "localStorage+cookie".', + ) + .optional(), + person_profiles: z + .enum(['always', 'never', 'identified_only']) + .describe( + 'When to create person profiles. "identified_only" (default) creates profiles only after identify() is called.', + ) + .optional(), + autocapture: z + .boolean() + .describe( + "Enable PostHog's autocapture of clicks, form submits, and pageviews. Default: false (walkerOS sources handle capture).", + ) + .optional(), + capture_pageview: z + .union([z.boolean(), z.literal('history_change')]) + .describe( + 'Whether PostHog should auto-capture pageview events. Default: false (walkerOS sources handle this).', + ) + .optional(), + capture_pageleave: z + .union([z.boolean(), z.literal('if_capture_pageview')]) + .describe( + 'Whether PostHog should auto-capture pageleave events. Default: false.', + ) + .optional(), + capture_heatmaps: z + .boolean() + .describe( + 'Enable heatmap data capture. Default: unset, which follows the heatmaps setting of the PostHog project.', + ) + .optional(), + capture_exceptions: z + .boolean() + .describe( + 'Enable automatic exception capture. Default: unset, which follows the exception autocapture setting of the PostHog project.', + ) + .optional(), + disable_surveys: z + .boolean() + .describe('Disable PostHog surveys. Default: false.') + .optional(), + disable_session_recording: z + .boolean() + .describe('Disable session recording. Default: false.') + .optional(), + advanced_disable_flags: z + .boolean() + .describe('Disable feature flag loading. Default: false.') + .optional(), + cookieless_mode: z + .enum(['always', 'on_reject']) + .describe( + '"always" never uses cookies/localStorage. "on_reject" falls back to cookieless when consent is rejected.', + ) + .optional(), + debug: z + .boolean() + .describe('Enable PostHog SDK debug logging. Default: false.') + .optional(), + session_recording: z + .unknown() + .describe( + 'Session Replay options (SessionRecordingOptions). Masking, blocking, sampling. See PostHog docs for full list.', + ) + .optional(), + bootstrap: z + .unknown() + .describe( + 'SSR bootstrap data. { distinctID?, featureFlags? } pre-populates identity and flag values to avoid first-render flicker.', + ) + .optional(), + identify: z + .unknown() + .describe( + 'walkerOS mapping value resolving to an identity object. Keys: distinctId, $set, $set_once. Resolved on every push: posthog.identify fires when distinctId changes, and on every push while $set or $set_once are present.', + ) + .optional(), + group: z + .unknown() + .describe( + 'walkerOS mapping value resolving to a group object. Keys: type, key, properties. Resolved on every push: posthog.group fires when type or key changes, and on every push while properties are present.', + ) + .optional(), + }) + .strict(); export type Settings = z.infer; diff --git a/packages/web/destinations/snowplow/src/__tests__/conformance.test.ts b/packages/web/destinations/snowplow/src/__tests__/conformance.test.ts new file mode 100644 index 000000000..92b6349a8 --- /dev/null +++ b/packages/web/destinations/snowplow/src/__tests__/conformance.test.ts @@ -0,0 +1,24 @@ +import path from 'path'; +import { conformance } from '@walkeros/collector/dev'; +import { destinationSnowplow } from '..'; +import { examples, schemas } from '../dev'; + +describe('snowplow conformance', () => { + beforeEach(() => { + jest.useRealTimers(); + }); + + it('conforms', async () => { + await conformance('destination', destinationSnowplow, { + profile: 'vendor-global', + examples: examples.step, + schema: schemas.settings, + settings: { + collectorUrl: 'https://collector.example.com', + pageViewEvent: 'page view', + }, + env: { push: examples.env.push, simulation: examples.env.simulation }, + root: path.join(__dirname, '..', '..'), + }); + }); +}); diff --git a/packages/web/destinations/snowplow/src/examples/step.ts b/packages/web/destinations/snowplow/src/examples/step.ts index 87cc3a264..ceed2a6b2 100644 --- a/packages/web/destinations/snowplow/src/examples/step.ts +++ b/packages/web/destinations/snowplow/src/examples/step.ts @@ -460,3 +460,26 @@ export const contextLoop: Flow.StepExample = { ], ], }; + +/** A silent rule sends no Snowplow event. */ +export const silentRule: Flow.StepExample = { + public: false, + description: 'A rule with silent: true sends no Snowplow event.', + in: getEvent('product view', { timestamp: 1700000405 }), + mapping: { name: ACTIONS.PRODUCT_VIEW, silent: true }, + out: [], +}; + +/** + * A revocation of the consent the destination requires. Without + * settings.consent there is no Enhanced Consent call, so no vendor call is + * made and the collector's consent gate stops the sending. + */ +export const consentRevoke: Flow.StepExample = { + public: false, + description: + 'For a destination with config.consent { marketing: true } and no settings.consent, revoking marketing makes no vendor call: the collector stops sending.', + command: 'consent', + in: { marketing: false }, + out: [], +}; diff --git a/packages/web/destinations/snowplow/src/index.test.ts b/packages/web/destinations/snowplow/src/index.test.ts index 20e897e9e..2e998607a 100644 --- a/packages/web/destinations/snowplow/src/index.test.ts +++ b/packages/web/destinations/snowplow/src/index.test.ts @@ -13,7 +13,7 @@ import { MEDIA_ACTIONS, } from '.'; import { examples } from './dev'; -import { resetLoadedScripts, DEFAULT_SCRIPT_URL } from './setup'; +import { DEFAULT_SCRIPT_URL } from './setup'; // Step example `.mapping` is typed as `unknown` (it can be any rule shape). // Narrow to the destination's Rule type for use in `mappingConfig`. @@ -143,48 +143,36 @@ describe('destination snowplow', () => { await elb(getEvent()); - // logger.throw only throws; the collector catches and writes the one line + // init throws a StepError; the collector writes the one line expect(consoleErrorSpy).toHaveBeenCalledTimes(1); expect(consoleErrorSpy).toHaveBeenCalledWith( 'ERROR', expect.stringContaining('destination init failed'), expect.objectContaining({ type: 'snowplow', - error: 'Config settings collectorUrl missing', + error: 'Snowplow needs settings.collectorUrl', }), ); consoleErrorSpy.mockRestore(); }); describe('loadScript with scriptUrl', () => { - let createdScripts: Array<{ src: string }>; + // loadVendor injects into the page's document, once per src + const scriptSrcs = () => + Array.from(document.querySelectorAll('script[data-walkeros-vendor]')).map( + (script) => script.getAttribute('src'), + ); - beforeEach(() => { - createdScripts = []; - resetLoadedScripts(); - }); + const scriptEnv: DestinationSnowplow.Env = { + window: { + snowplow: Object.assign(() => {}, { q: [] }), + }, + }; test('uses custom scriptUrl when provided', async () => { const customScriptUrl = 'https://cdn.jsdelivr.net/npm/@snowplow/javascript-tracker@3.24.0/dist/sp.js'; - // Create env that captures created scripts - const scriptEnv: DestinationSnowplow.Env = { - window: { - snowplow: Object.assign(() => {}, { q: [] }), - }, - document: { - createElement: () => { - const element = { src: '', async: false }; - createdScripts.push(element); - return element; - }, - head: { - appendChild: () => {}, - }, - }, - }; - const destinationWithEnv = { ...destination, env: scriptEnv, @@ -202,28 +190,10 @@ describe('destination snowplow', () => { await elb(getEvent()); - expect(createdScripts.length).toBeGreaterThan(0); - expect(createdScripts[0].src).toBe(customScriptUrl); + expect(scriptSrcs()).toContain(customScriptUrl); }); test('uses default CDN URL when scriptUrl not provided', async () => { - // Create env that captures created scripts - const scriptEnv: DestinationSnowplow.Env = { - window: { - snowplow: Object.assign(() => {}, { q: [] }), - }, - document: { - createElement: () => { - const element = { src: '', async: false }; - createdScripts.push(element); - return element; - }, - head: { - appendChild: () => {}, - }, - }, - }; - const destinationWithEnv = { ...destination, env: scriptEnv, @@ -240,8 +210,7 @@ describe('destination snowplow', () => { await elb(getEvent()); - expect(createdScripts.length).toBeGreaterThan(0); - expect(createdScripts[0].src).toBe(DEFAULT_SCRIPT_URL); + expect(scriptSrcs()).toContain(DEFAULT_SCRIPT_URL); }); }); diff --git a/packages/web/destinations/snowplow/src/index.ts b/packages/web/destinations/snowplow/src/index.ts index 9bc14ee0c..056418eb1 100644 --- a/packages/web/destinations/snowplow/src/index.ts +++ b/packages/web/destinations/snowplow/src/index.ts @@ -1,4 +1,4 @@ -import type { WalkerOS, Logger } from '@walkeros/core'; +import type { WalkerOS } from '@walkeros/core'; import type { Settings, Destination, @@ -13,10 +13,11 @@ import { isCodeBasedPlugin, deriveEnableMethod, } from './types'; -import { addScript, setup } from './setup'; +import { DEFAULT_SCRIPT_URL, setup } from './setup'; import { pushSnowplowEvent } from './push'; import { createQueueAdapter, createBrowserTrackerAdapter } from './adapter'; -import { isObject } from '@walkeros/core'; +import { isObject, StepError } from '@walkeros/core'; +import { getEnv, loadVendor } from '@walkeros/web-core'; // Types export * as DestinationSnowplow from './types'; @@ -31,6 +32,41 @@ export { MEDIA_ACTIONS, } from './types'; +/** + * The sp.js script `loadVendor` injects with `loadScript`. The snowplow + * function replays its own queue, so it is called synchronously and only a + * failed script is reported. + */ +function vendor( + config: { loadScript?: boolean; timeout?: number; settings?: Settings }, + env?: Env, +) { + return loadVendor( + config.loadScript && !config.settings?.tracker + ? config.settings?.scriptUrl || DEFAULT_SCRIPT_URL + : undefined, + () => getEnv(env).window.snowplow, + { timeout: config.timeout }, + ); +} + +/** + * Runs one Snowplow call or block; a throw or a rejection that is not + * already a StepError becomes a StepError of `code`. + */ +async function guard( + fn: () => T | Promise, + code: string, + message: string, +): Promise { + try { + return await fn(); + } catch (error) { + if (error instanceof StepError) throw error; + throw new StepError(message, { code, cause: error }); + } +} + /** * Get the Snowplow function from the environment * @@ -182,114 +218,148 @@ export const destinationSnowplow: Destination = { * @param context - Initialization context * @returns Updated configuration */ - init({ config, env, logger }) { + async init({ config, env }) { const { settings = {}, loadScript } = config; const { collectorUrl, tracker: trackerFunctions } = settings; // Required collector URL - if (!collectorUrl) logger.throw('Config settings collectorUrl missing'); + if (!collectorUrl) + throw new StepError('Snowplow needs settings.collectorUrl', { + code: 'config', + }); let adapter: SnowplowAdapter | undefined; if (trackerFunctions) { // Browser-tracker mode: use imported functions directly - if (!trackerFunctions.newTracker) { - logger.throw('tracker.newTracker is required for browser-tracker mode'); - return false; - } - if (!trackerFunctions.trackSelfDescribingEvent) { - logger.throw( + if (!trackerFunctions.newTracker) + throw new StepError( + 'tracker.newTracker is required for browser-tracker mode', + { code: 'config' }, + ); + if (!trackerFunctions.trackSelfDescribingEvent) + throw new StepError( 'tracker.trackSelfDescribingEvent is required for browser-tracker mode', + { code: 'config' }, ); - return false; - } // Initialize tracker - trackerFunctions.newTracker(settings.trackerName || 'sp', collectorUrl!, { - appId: settings.appId || 'walkerOS', - platform: settings.platform || 'web', - discoverRootDomain: settings.discoverRootDomain, - cookieSameSite: settings.cookieSameSite, - appVersion: settings.appVersion, - contexts: settings.contexts, - anonymousTracking: settings.anonymousTracking, - }); + await guard( + () => + trackerFunctions.newTracker( + settings.trackerName || 'sp', + collectorUrl, + { + appId: settings.appId || 'walkerOS', + platform: settings.platform || 'web', + discoverRootDomain: settings.discoverRootDomain, + cookieSameSite: settings.cookieSameSite, + appVersion: settings.appVersion, + contexts: settings.contexts, + anonymousTracking: settings.anonymousTracking, + }, + ), + 'vendor-not-ready', + 'Snowplow newTracker failed', + ); // Create adapter from functions adapter = createBrowserTrackerAdapter(trackerFunctions); } else { // URL-based mode: load sp.js script - if (loadScript) { - addScript(collectorUrl!, env, settings.scriptUrl); - } - const snowplow = setup(env); - if (!snowplow) return false; + if (!snowplow) + throw new StepError('Snowplow not ready: window is missing', { + code: 'vendor-not-ready', + }); + + // The script, once per page; a failed one is reported by each push + vendor({ ...config, settings }, env); // Initialize tracker via queue - snowplow('newTracker', settings.trackerName || 'sp', collectorUrl!, { - appId: settings.appId || 'walkerOS', - platform: settings.platform || 'web', - discoverRootDomain: settings.discoverRootDomain, - cookieSameSite: settings.cookieSameSite, - appVersion: settings.appVersion, - contexts: settings.contexts, - anonymousTracking: settings.anonymousTracking, - }); + await guard( + () => + snowplow('newTracker', settings.trackerName || 'sp', collectorUrl, { + appId: settings.appId || 'walkerOS', + platform: settings.platform || 'web', + discoverRootDomain: settings.discoverRootDomain, + cookieSameSite: settings.cookieSameSite, + appVersion: settings.appVersion, + contexts: settings.contexts, + anonymousTracking: settings.anonymousTracking, + }), + 'vendor-not-ready', + 'Snowplow newTracker failed', + ); adapter = createQueueAdapter(snowplow); } - if (!adapter) return false; + const tracker = adapter; + if (!tracker) + throw new StepError('Snowplow adapter could not be created', { + code: 'vendor-not-ready', + }); // Enable activity tracking if configured if (settings.activityTracking) { - adapter.enableActivityTracking(settings.activityTracking); + const activityTracking = settings.activityTracking; + await guard( + () => tracker.enableActivityTracking(activityTracking), + 'vendor-not-ready', + 'Snowplow enableActivityTracking failed', + ); } - // Load plugins - if (settings.plugins) { - for (const plugin of settings.plugins) { - if (isCodeBasedPlugin(plugin)) { - // Code-based plugin: use directly or call factory with config - const pluginInstance = - typeof plugin.code === 'function' && plugin.config - ? (plugin.code as (...args: unknown[]) => BrowserPlugin)( - plugin.config, - ) - : plugin.code; - adapter.addPlugin({ plugin: pluginInstance as BrowserPlugin }); - } else if (isUrlBasedPlugin(plugin)) { - // URL-based plugin (sp.js approach only) - adapter.addPlugin([plugin.url, plugin.name]); - const enableMethod = - plugin.enableMethod ?? deriveEnableMethod(plugin.name[1]); - if (plugin.options) { - adapter.call(enableMethod, plugin.options); - } else { - adapter.call(enableMethod); + await guard( + () => { + // Load plugins + if (settings.plugins) { + for (const plugin of settings.plugins) { + if (isCodeBasedPlugin(plugin)) { + // Code-based plugin: use directly or call factory with config + const pluginInstance = + typeof plugin.code === 'function' && plugin.config + ? (plugin.code as (...args: unknown[]) => BrowserPlugin)( + plugin.config, + ) + : plugin.code; + tracker.addPlugin({ plugin: pluginInstance as BrowserPlugin }); + } else if (isUrlBasedPlugin(plugin)) { + // URL-based plugin (sp.js approach only) + tracker.addPlugin([plugin.url, plugin.name]); + const enableMethod = + plugin.enableMethod ?? deriveEnableMethod(plugin.name[1]); + if (plugin.options) { + tracker.call(enableMethod, plugin.options); + } else { + tracker.call(enableMethod); + } + } else { + // BrowserPlugin instance + tracker.addPlugin({ plugin: plugin as BrowserPlugin }); + } } - } else { - // BrowserPlugin instance - adapter.addPlugin({ plugin: plugin as BrowserPlugin }); } - } - } - // Register global contexts - if (settings.globalContexts && settings.globalContexts.length > 0) { - adapter.addGlobalContexts(settings.globalContexts); - } + // Register global contexts + if (settings.globalContexts && settings.globalContexts.length > 0) { + tracker.addGlobalContexts(settings.globalContexts); + } - // Track page view on init if configured - if (settings.trackPageView) { - adapter.trackPageView(); - } + // Track page view on init if configured + if (settings.trackPageView) { + tracker.trackPageView(); + } + }, + 'vendor-not-ready', + 'Snowplow setup call failed', + ); // Store adapter in runtime state const updatedSettings = { ...settings, - _state: { adapter } as RuntimeState, + _state: { adapter: tracker } as RuntimeState, }; return { ...config, settings: updatedSettings }; @@ -304,16 +374,24 @@ export const destinationSnowplow: Destination = { * @param event - The walkerOS event to process * @param context - Push context with config, data, rule, and env */ - async push(event, { config, data = {}, rule = {}, logger, collector }) { + async push(event, { config, env, data = {}, rule = {}, logger, collector }) { + vendor(config, env as Env | undefined).check(); + const eventMapping = rule.settings || {}; - await pushSnowplowEvent( - event, - eventMapping, - data as WalkerOS.AnyObject, - collector, - rule.name, - config, - logger, + await guard( + () => + pushSnowplowEvent( + event, + eventMapping, + data as WalkerOS.AnyObject, + collector, + rule.name, + config, + logger, + rule.silent === true, + ), + 'transport', + 'Snowplow call failed', ); }, @@ -327,7 +405,7 @@ export const destinationSnowplow: Destination = { * @param type - The event type ('consent', 'session', 'ready', 'run') * @param context - The destination context with config, data, env, logger */ - on(type, context) { + async on(type, context) { // Only handle consent events if (type !== 'consent' || !isObject(context.data)) return; @@ -370,20 +448,29 @@ export const destinationSnowplow: Destination = { baseParams.gdprApplies = consentConfig.gdprApplies; // Call the appropriate Enhanced Consent plugin method - if (allDenied) { - // For deny, use required scopes (the ones being denied) - snowplow('trackConsentDeny', { ...baseParams, consentScopes: required }); - } else if (allGranted) { - snowplow('trackConsentAllow', { - ...baseParams, - consentScopes: grantedScopes, - }); - } else { - snowplow('trackConsentSelected', { - ...baseParams, - consentScopes: grantedScopes, - }); - } + await guard( + () => { + if (allDenied) { + // For deny, use required scopes (the ones being denied) + snowplow('trackConsentDeny', { + ...baseParams, + consentScopes: required, + }); + } else if (allGranted) { + snowplow('trackConsentAllow', { + ...baseParams, + consentScopes: grantedScopes, + }); + } else { + snowplow('trackConsentSelected', { + ...baseParams, + consentScopes: grantedScopes, + }); + } + }, + 'consent-not-applied', + 'Snowplow did not take the consent update', + ); }, }; diff --git a/packages/web/destinations/snowplow/src/push.ts b/packages/web/destinations/snowplow/src/push.ts index bed661fa0..8d04e0e0e 100644 --- a/packages/web/destinations/snowplow/src/push.ts +++ b/packages/web/destinations/snowplow/src/push.ts @@ -17,6 +17,7 @@ import { getMappingValue, isString, isNumber, + StepError, } from '@walkeros/core'; import { SCHEMAS } from './types'; @@ -52,14 +53,15 @@ export async function pushSnowplowEvent( actionName?: string, config?: Config, logger?: Logger.Instance, + silent = false, ): Promise { const settings = config?.settings; const adapter = settings?._state?.adapter; - if (!adapter) { - logger?.throw('Tracker not initialized'); - return; - } + if (!adapter) + throw new StepError('Snowplow tracker not initialized', { + code: 'not-ready', + }); const runtimeState = settings?._state; // Set userId once on first event where value is available @@ -110,15 +112,13 @@ export async function pushSnowplowEvent( } } + // A silent rule runs the identity and page side effects above and sends + // no event + if (silent) return; + // Handle structured events (bypasses self-describing events) if (mapping.struct) { - await handleStructuredEvent( - event, - mapping.struct, - adapter, - collector, - logger, - ); + await handleStructuredEvent(event, mapping.struct, adapter, collector); return; } @@ -129,37 +129,35 @@ export async function pushSnowplowEvent( } // Handle self-describing events - if (actionName) { - const actionSchema = - mapping.snowplow?.actionSchema || - settings?.snowplow?.actionSchema || - SCHEMAS.ACTION; + if (!actionName) + throw new StepError('Snowplow rule has no action name', { + code: 'missing-input', + }); - const context = await buildContext(event, mapping, collector); + const actionSchema = + mapping.snowplow?.actionSchema || + settings?.snowplow?.actionSchema || + SCHEMAS.ACTION; - // Build event data based on schema type - let eventData: WalkerOS.AnyObject = {}; + const context = await buildContext(event, mapping, collector); - if (isObject(mapping.data) && 'map' in mapping.data) { - // Use mapped data for self-describing events (e.g., percent_progress) - const mapped = await getMappingValue(event, mapping.data, { collector }); - if (isObject(mapped)) eventData = mapped as WalkerOS.AnyObject; - } else if (actionSchema === SCHEMAS.ACTION) { - // Ecommerce pattern: include type field - eventData = { type: actionName }; - } - // else: empty data {} for marker events (media play/pause/etc.) + // Build event data based on schema type + let eventData: WalkerOS.AnyObject = {}; - adapter.trackSelfDescribingEvent({ - event: { schema: actionSchema, data: eventData as WalkerOS.Properties }, - context: context.length > 0 ? context : undefined, - }); - } else { - logger?.info('Event skipped: no action name in mapping', { - event: event.name, - hasContext: !!mapping.context?.length, - }); + if (isObject(mapping.data) && 'map' in mapping.data) { + // Use mapped data for self-describing events (e.g., percent_progress) + const mapped = await getMappingValue(event, mapping.data, { collector }); + if (isObject(mapped)) eventData = mapped as WalkerOS.AnyObject; + } else if (actionSchema === SCHEMAS.ACTION) { + // Ecommerce pattern: include type field + eventData = { type: actionName }; } + // else: empty data {} for marker events (media play/pause/etc.) + + adapter.trackSelfDescribingEvent({ + event: { schema: actionSchema, data: eventData as WalkerOS.Properties }, + context: context.length > 0 ? context : undefined, + }); } /** @@ -243,29 +241,20 @@ async function handleStructuredEvent( struct: StructuredEventMapping, adapter: SnowplowAdapter, collector: Collector.Instance, - logger?: Logger.Instance, ): Promise { // Resolve required fields const category = await getMappingValue(event, struct.category, { collector }); const action = await getMappingValue(event, struct.action, { collector }); - // Category and action are required - skip with warning if not present - if (!category || !isString(category)) { - logger?.info('Struct event skipped: invalid category', { - event: event.name, - category, - reason: !category ? 'missing' : 'not a string', + // Category and action are required: without them nothing can be sent + if (!category || !isString(category)) + throw new StepError('Snowplow struct event needs a category string', { + code: 'missing-input', }); - return; - } - if (!action || !isString(action)) { - logger?.info('Struct event skipped: invalid action', { - event: event.name, - action, - reason: !action ? 'missing' : 'not a string', + if (!action || !isString(action)) + throw new StepError('Snowplow struct event needs an action string', { + code: 'missing-input', }); - return; - } // Resolve optional fields const label = struct.label diff --git a/packages/web/destinations/snowplow/src/schemas/settings.ts b/packages/web/destinations/snowplow/src/schemas/settings.ts index f737af7e2..ae1cebe0c 100644 --- a/packages/web/destinations/snowplow/src/schemas/settings.ts +++ b/packages/web/destinations/snowplow/src/schemas/settings.ts @@ -109,53 +109,121 @@ export const MappedGlobalContextSchema = z.object({ /** * Configuration settings schema for Snowplow destination */ -export const SettingsSchema = z.object({ - collectorUrl: z - .string() - .optional() - .describe('Snowplow collector endpoint URL'), - appId: z.string().optional().describe('Application identifier'), - trackerName: z.string().optional().describe('Tracker instance name'), - platform: z.string().optional().describe('Platform identifier'), - pageViewTracking: z - .boolean() - .optional() - .describe('Enable automatic page view tracking'), - snowplow: SnowplowSettingsSchema.optional().describe( - 'Snowplow-specific ecommerce configuration', - ), - // New tracker configuration options - discoverRootDomain: z - .boolean() - .optional() - .describe('Discover root domain for cookies'), - cookieSameSite: z - .enum(['Strict', 'Lax', 'None']) - .optional() - .describe('Cookie SameSite attribute'), - appVersion: z.string().optional().describe('Application version'), - contexts: TrackerContextsSchema.optional().describe( - 'Built-in context entities', - ), - // Anonymous tracking - anonymousTracking: AnonymousTrackingSchema.optional().describe( - 'Enable anonymous tracking (no user identifiers)', - ), - // Plugins - validated loosely since BrowserPlugin is complex - plugins: z - .array(z.union([UrlBasedPluginSchema, z.any()])) - .optional() - .describe('Snowplow plugins'), - // Activity tracking - activityTracking: ActivityTrackingSchema.optional().describe( - 'Page ping configuration', - ), - // Global contexts - validated loosely to allow functions - globalContexts: z - .array(z.any()) - .optional() - .describe('Global context entities'), -}); +export const SettingsSchema = z + .object({ + collectorUrl: z.string().describe('Snowplow collector endpoint URL'), + scriptUrl: z + .string() + .optional() + .describe( + 'URL of the Snowplow JavaScript tracker script, used with loadScript. Defaults to the jsdelivr CDN with the @latest tag; pin a version in production.', + ), + tracker: z + .record(z.string(), z.any()) + .optional() + .describe( + 'Browser-tracker functions for bundled mode (newTracker, trackSelfDescribingEvent and the optional others), set with $code: references', + ), + appId: z.string().optional().describe('Application identifier'), + trackerName: z.string().optional().describe('Tracker instance name'), + platform: z.string().optional().describe('Platform identifier'), + snowplow: SnowplowSettingsSchema.optional().describe( + 'Snowplow-specific ecommerce configuration', + ), + // New tracker configuration options + discoverRootDomain: z + .boolean() + .optional() + .describe('Discover root domain for cookies'), + cookieSameSite: z + .enum(['Strict', 'Lax', 'None']) + .optional() + .describe('Cookie SameSite attribute'), + appVersion: z.string().optional().describe('Application version'), + contexts: TrackerContextsSchema.optional().describe( + 'Built-in context entities', + ), + // Anonymous tracking + anonymousTracking: AnonymousTrackingSchema.optional().describe( + 'Enable anonymous tracking (no user identifiers)', + ), + // Plugins - validated loosely since BrowserPlugin is complex + plugins: z + .array(z.union([UrlBasedPluginSchema, z.any()])) + .optional() + .describe('Snowplow plugins'), + // Activity tracking + activityTracking: ActivityTrackingSchema.optional().describe( + 'Page ping configuration', + ), + // Global contexts - validated loosely to allow functions + globalContexts: z + .array(z.any()) + .optional() + .describe('Global context entities'), + trackPageView: z + .boolean() + .optional() + .describe('Call trackPageView once right after the tracker starts'), + pageViewEvent: z + .string() + .optional() + .describe( + 'Event name that calls trackPageView instead of a self-describing event (like page view)', + ), + userId: z + .unknown() + .optional() + .describe( + 'walkerOS mapping value resolving to the user id, set once on the first event where it resolves', + ), + page: z + .object({ + type: z.unknown().describe('Mapping value resolving to the page type'), + language: z + .unknown() + .optional() + .describe('Mapping value resolving to the page language'), + locale: z + .unknown() + .optional() + .describe('Mapping value resolving to the page locale'), + }) + .optional() + .describe('Page context for setPageType (ecommerce plugin)'), + consent: z + .object({ + required: z + .array(z.string()) + .optional() + .describe('walkerOS consent groups to check'), + basisForProcessing: z + .string() + .optional() + .describe('Legal basis for processing under GDPR (default consent)'), + consentUrl: z + .string() + .optional() + .describe('URL of the privacy policy or consent document'), + consentVersion: z + .string() + .optional() + .describe('Version of the consent document'), + domainsApplied: z + .array(z.string()) + .optional() + .describe('Domains where this consent applies'), + gdprApplies: z + .boolean() + .optional() + .describe('Whether GDPR applies to this user'), + }) + .optional() + .describe( + 'Consent event tracking through the Enhanced Consent plugin, used by the consent handler', + ), + }) + .strict(); export type Settings = z.infer; export type SnowplowSettings = z.infer; diff --git a/packages/web/destinations/snowplow/src/setup.ts b/packages/web/destinations/snowplow/src/setup.ts index 894a71aa1..5b1bd5b66 100644 --- a/packages/web/destinations/snowplow/src/setup.ts +++ b/packages/web/destinations/snowplow/src/setup.ts @@ -10,29 +10,6 @@ import { getEnv } from '@walkeros/web-core'; export const DEFAULT_SCRIPT_URL = 'https://cdn.jsdelivr.net/npm/@snowplow/javascript-tracker@latest/dist/sp.js'; -const loadedScripts = new Set(); - -// For testing: allow resetting loaded scripts -export function resetLoadedScripts(): void { - loadedScripts.clear(); -} - -export function addScript( - collectorUrl: string, - env?: Env, - src = DEFAULT_SCRIPT_URL, -): void { - // Prevent loading the same script multiple times - if (loadedScripts.has(collectorUrl)) return; - - const { document } = getEnv(env); - const script = document.createElement('script'); - script.src = src; - script.async = true; - document.head.appendChild(script); - loadedScripts.add(collectorUrl); -} - export function setup(env?: Env): SnowplowFunction | undefined { const { window } = getEnv(env); diff --git a/packages/web/destinations/snowplow/src/types/index.ts b/packages/web/destinations/snowplow/src/types/index.ts index ae7cab0d7..744ec7096 100644 --- a/packages/web/destinations/snowplow/src/types/index.ts +++ b/packages/web/destinations/snowplow/src/types/index.ts @@ -467,15 +467,6 @@ export interface Settings { */ platform?: string; - /** - * Enable automatic page view tracking - * - * If true, page view events will be tracked automatically. - * - * @default false - */ - pageViewTracking?: boolean; - /** * Track page view on tracker initialization * diff --git a/packages/web/destinations/tiktok/src/__tests__/conformance.test.ts b/packages/web/destinations/tiktok/src/__tests__/conformance.test.ts new file mode 100644 index 000000000..5b187fd03 --- /dev/null +++ b/packages/web/destinations/tiktok/src/__tests__/conformance.test.ts @@ -0,0 +1,26 @@ +import path from 'path'; +import { conformance } from '@walkeros/collector/dev'; +import { destinationTikTok } from '..'; +import { examples, schemas } from '../dev'; + +describe('tiktok conformance', () => { + beforeEach(() => { + jest.useRealTimers(); + }); + + it('conforms', async () => { + await conformance('destination', destinationTikTok, { + profile: 'vendor-global', + examples: examples.step, + schema: schemas.settings, + settings: { + apiKey: 'C4A1B2C3D4E5F6A7B8C9', + auto_config: true, + limited_data_use: false, + }, + env: { push: examples.env.push, simulation: examples.env.simulation }, + root: path.join(__dirname, '..', '..'), + passThrough: ['auto_config', 'limited_data_use'], + }); + }); +}); diff --git a/packages/web/destinations/tiktok/src/__tests__/stepExamples.test.ts b/packages/web/destinations/tiktok/src/__tests__/stepExamples.test.ts index 7f0552ed7..83d00479e 100644 --- a/packages/web/destinations/tiktok/src/__tests__/stepExamples.test.ts +++ b/packages/web/destinations/tiktok/src/__tests__/stepExamples.test.ts @@ -14,6 +14,7 @@ type CallRecord = [string, ...unknown[]]; function spyTtq(env: Env): { env: Env; collected: () => CallRecord[]; + reset: () => void; } { const calls: CallRecord[] = []; const ttq = env.window.ttq; @@ -28,7 +29,7 @@ function spyTtq(env: Env): { ttq.identify = record('identify'); ttq.enableCookie = record('enableCookie'); ttq.disableCookie = record('disableCookie'); - return { env, collected: () => calls }; + return { env, collected: () => calls, reset: () => (calls.length = 0) }; } describe('tiktok destination — step examples', () => { @@ -37,7 +38,7 @@ describe('tiktok destination — step examples', () => { async (name, example) => { // Fresh clone per test so mutations don't bleed across cases. const env = clone(examples.env.push) as Env; - const { env: spiedEnv, collected } = spyTtq(env); + const { env: spiedEnv, collected, reset } = spyTtq(env); const dest = jest.requireActual('../').default; const { elb } = await startFlow(); @@ -58,7 +59,11 @@ describe('tiktok destination — step examples', () => { }); // Load the gated destination under a prior grant first when the example // declares one, so a revoke acts on an already-granted destination. - if (example.before) await elb('walker consent', example.before); + if (example.before) { + await elb('walker consent', example.before); + // The example's out is what the revoke itself calls + reset(); + } await elb('walker consent', example.in as WalkerOS.Consent); } else { const event = example.in as WalkerOS.Event; @@ -79,7 +84,13 @@ describe('tiktok destination — step examples', () => { } // Drop ttq.load — it's an init-time call, not part of per-push `out`. - const expected = (example.out ?? []) as ReadonlyArray; + // Consent examples name the vendor call by its env path (window.ttq.*). + const expected = ((example.out ?? []) as ReadonlyArray).map( + ([path, ...args]): CallRecord => [ + path.replace(/^window\./, ''), + ...args, + ], + ); const actual = collected().filter(([path]) => path !== 'ttq.load'); expect(actual).toEqual(expected); diff --git a/packages/web/destinations/tiktok/src/examples/step.ts b/packages/web/destinations/tiktok/src/examples/step.ts index d3dc70d50..a0f453348 100644 --- a/packages/web/destinations/tiktok/src/examples/step.ts +++ b/packages/web/destinations/tiktok/src/examples/step.ts @@ -389,5 +389,40 @@ export const consentRevokeDisableCookie: TikTokStepExample = { before: { marketing: true }, in: { marketing: false } as WalkerOS.Consent, settings: {} as Partial, - out: [['ttq.enableCookie'], ['ttq.disableCookie']], + out: [['window.ttq.disableCookie']], +}; + +/** A silent rule sends no track call. */ +export const silentRule: TikTokStepExample = { + public: false, + description: 'A rule with silent: true sends no ttq.track call.', + in: getEvent('page view', { timestamp: 1700000600 }), + mapping: { silent: true }, + out: [], +}; + +/** + * A silent rule that only sets an identity, on an event with no email, phone + * or id: nothing resolves and the rule suppresses the track, so nothing is + * sent. + */ +export const silentIdentifyWithoutData: TikTokStepExample = { + public: false, + description: + 'A silent rule whose identify mapping finds no email, phone or id sends nothing, so the push fails with missing-input instead of counting as delivered.', + in: { name: 'user update', entity: 'user', action: 'update' }, + mapping: { + silent: true, + settings: { + identify: { + map: { + email: 'data.email', + phone_number: 'data.phone', + external_id: 'data.id', + }, + }, + }, + }, + out: [], + outcome: { status: 'failed', code: 'missing-input' }, }; diff --git a/packages/web/destinations/tiktok/src/index.ts b/packages/web/destinations/tiktok/src/index.ts index 1a77421f1..4f7a4c688 100644 --- a/packages/web/destinations/tiktok/src/index.ts +++ b/packages/web/destinations/tiktok/src/index.ts @@ -1,7 +1,7 @@ import type { WalkerOS } from '@walkeros/core'; -import { getMappingValue, isObject, isString } from '@walkeros/core'; -import { getEnv } from '@walkeros/web-core'; -import { addScript, setup } from './setup'; +import { getMappingValue, isObject, isString, StepError } from '@walkeros/core'; +import { getEnv, loadVendor } from '@walkeros/web-core'; +import { SCRIPT_SRC, setup } from './setup'; import type { Destination, IdentifyParams, @@ -21,7 +21,44 @@ export * as DestinationTikTok from './types'; */ function getTTQ(env: unknown): TTQ { const { window } = getEnv(env as Parameters[0]); - return (window as unknown as { ttq: TTQ }).ttq; + const { ttq } = window as unknown as { ttq?: TTQ }; + if (!ttq) + throw new StepError('TikTok Pixel not ready: window.ttq is missing', { + code: 'vendor-not-ready', + }); + return ttq; +} + +/** + * The Pixel script `loadVendor` injects unless `loadScript` is false. ttq + * replays its own queue, so it is called synchronously and only a failed + * script is reported. + */ +function vendor( + config: { loadScript?: boolean; timeout?: number }, + env: unknown, +) { + return loadVendor( + config.loadScript !== false ? SCRIPT_SRC : undefined, + () => { + const { window } = getEnv(env as Parameters[0]); + return (window as unknown as { ttq?: TTQ }).ttq; + }, + { timeout: config.timeout }, + ); +} + +/** Runs one ttq call; a throw or a rejection becomes a StepError of `code`. */ +async function call( + fn: () => unknown, + code: string, + message: string, +): Promise { + try { + await fn(); + } catch (error) { + throw new StepError(message, { code, cause: error }); + } } /** @@ -32,11 +69,14 @@ function getTTQ(env: unknown): TTQ { * The destination never partial-updates identity - TikTok treats each * ttq.identify() call as a full replacement of Advanced Matching state. */ -function applyIdentify( +async function applyIdentify( ttq: TTQ, resolved: Record, lastIdentity: NonNullable = {}, -): NonNullable { +): Promise<{ + lastIdentity: NonNullable; + resolved: boolean; +}> { const next: NonNullable = { ...lastIdentity }; const params: IdentifyParams = {}; let changed = false; @@ -53,10 +93,11 @@ function applyIdentify( // Skip ttq.identify() entirely if the resolved object has no non-empty // values (anonymous visitor) or if nothing changed. - if (!changed || Object.keys(params).length === 0) return next; + if (!changed || Object.keys(params).length === 0) + return { lastIdentity: next, resolved: Object.keys(params).length > 0 }; - ttq.identify(params); - return next; + await call(() => ttq.identify(params), 'transport', 'TikTok identify failed'); + return { lastIdentity: next, resolved: true }; } export const destinationTikTok: Destination = { @@ -64,16 +105,18 @@ export const destinationTikTok: Destination = { config: {}, - init({ config, env }) { - const settings = config.settings; - if (!settings?.apiKey) return false; + async init({ config, env }) { + // A copy, so the destructure below does not read keys off the live settings + const settings = { ...config.settings }; + if (!settings.apiKey) + throw new StepError('TikTok needs settings.apiKey', { code: 'config' }); // Ensure window.ttq exists (queue-based stub) before any call. setup(env); // Load the real TikTok Pixel SDK from the CDN. The queue replays // once the script executes. - if (config.loadScript !== false) addScript(env); + vendor(config, env); const ttq = getTTQ(env); @@ -89,7 +132,11 @@ export const destinationTikTok: Destination = { // TikTok's SDK auto-fires ttq.page() on load; we let it fire (no knob). const loadOptions: Record = { ...passthrough }; - ttq.load(apiKey, loadOptions); + await call( + () => ttq.load(apiKey, loadOptions), + 'vendor-not-ready', + 'TikTok load call failed', + ); // Initialize runtime state. Identity is resolved on first push. const _state: RuntimeState = { lastIdentity: {} }; @@ -97,11 +144,14 @@ export const destinationTikTok: Destination = { }, async push(event, { config, rule, data, env, collector }) { + vendor(config, env).check(); const ttq = getTTQ(env); const settings = (config.settings || {}) as Settings; const mappingSettings = (rule?.settings || {}) as Mapping; const state: RuntimeState = settings._state || {}; + // Whether any ttq call or identity came out of this event + let sent = false; // Identity - rule-level override wins over destination-level. // Resolved and diffed against runtime state; only changes fire @@ -113,11 +163,13 @@ export const destinationTikTok: Destination = { collector, }); if (isObject(resolved)) { - state.lastIdentity = applyIdentify( + const identity = await applyIdentify( ttq, resolved as Record, state.lastIdentity, ); + state.lastIdentity = identity.lastIdentity; + sent = identity.resolved; } } @@ -129,16 +181,30 @@ export const destinationTikTok: Destination = { ? (data as Record) : {}; - ttq.track(eventName, params, { event_id: event.id }); + await call( + () => ttq.track(eventName, params, { event_id: event.id }), + 'transport', + 'TikTok track failed', + ); + sent = true; } + // A silent rule that resolved no identity sent nothing at all + if (!sent && rule?.silent === true && identifyMapping !== undefined) + throw new StepError( + 'TikTok identify resolved to nothing for this event', + { + code: 'missing-input', + }, + ); + // Persist runtime state mutations. walkerOS destinations are // long-lived singletons so mutating settings in place is safe and // matches the Snowplow precedent. settings._state = state; }, - on(type, context) { + async on(type, context) { if (type !== 'consent' || !context.data) return; const ttq = getTTQ(context.env); @@ -155,11 +221,11 @@ export const destinationTikTok: Destination = { (key) => consent[key] === true, ); - if (allGranted) { - ttq.enableCookie(); - } else { - ttq.disableCookie(); - } + await call( + () => (allGranted ? ttq.enableCookie() : ttq.disableCookie()), + 'consent-not-applied', + 'TikTok did not take the consent update', + ); }, }; diff --git a/packages/web/destinations/tiktok/src/schemas/settings.ts b/packages/web/destinations/tiktok/src/schemas/settings.ts index dd3fdaeec..a5f6a711a 100644 --- a/packages/web/destinations/tiktok/src/schemas/settings.ts +++ b/packages/web/destinations/tiktok/src/schemas/settings.ts @@ -1,32 +1,34 @@ import { z } from '@walkeros/core/dev'; -export const SettingsSchema = z.object({ - apiKey: z - .string() - .min(1) - .describe( - 'Your TikTok Pixel ID. Find it in TikTok Ads Manager under "Assets" → "Events" → "Web Events" → select your pixel → "Setup Web Event" → "Pixel ID".', - ), - // `pageview` removed 2026-04-09 — TikTok's SDK fires auto page view and - // the knob was unreliable. Let it fire; don't expose a toggle. - auto_config: z - .boolean() - .describe( - 'TikTok default: true. Enable automatic form field detection for Advanced Matching.', - ) - .optional(), - limited_data_use: z - .boolean() - .describe( - 'TikTok default: false. Restrict data use for compliance with U.S. state privacy laws.', - ) - .optional(), - identify: z - .unknown() - .describe( - 'walkerOS mapping value resolving to an Advanced Matching object with any of: email, phone_number, external_id. All values are auto-hashed (SHA256) by the TikTok SDK before sending.', - ) - .optional(), -}); +export const SettingsSchema = z + .object({ + apiKey: z + .string() + .min(1) + .describe( + 'Your TikTok Pixel ID. Find it in TikTok Ads Manager under "Assets" → "Events" → "Web Events" → select your pixel → "Setup Web Event" → "Pixel ID".', + ), + // `pageview` removed 2026-04-09 — TikTok's SDK fires auto page view and + // the knob was unreliable. Let it fire; don't expose a toggle. + auto_config: z + .boolean() + .describe( + 'TikTok default: true. Enable automatic form field detection for Advanced Matching.', + ) + .optional(), + limited_data_use: z + .boolean() + .describe( + 'TikTok default: false. Restrict data use for compliance with U.S. state privacy laws.', + ) + .optional(), + identify: z + .unknown() + .describe( + 'walkerOS mapping value resolving to an Advanced Matching object with any of: email, phone_number, external_id. All values are auto-hashed (SHA256) by the TikTok SDK before sending.', + ) + .optional(), + }) + .strict(); export type Settings = z.infer; diff --git a/packages/web/destinations/tiktok/src/setup.ts b/packages/web/destinations/tiktok/src/setup.ts index cc76c01dd..aca545568 100644 --- a/packages/web/destinations/tiktok/src/setup.ts +++ b/packages/web/destinations/tiktok/src/setup.ts @@ -1,22 +1,8 @@ import type { DestinationWeb } from '@walkeros/web-core'; import { getEnv } from '@walkeros/web-core'; -const DEFAULT_TIKTOK_SRC = 'https://analytics.tiktok.com/i18n/pixel/events.js'; - -/** - * Inject the TikTok Pixel CDN script. The real SDK replaces the queueing - * stub created by setup() with a full implementation. - */ -export function addScript( - env?: DestinationWeb.Env, - src = DEFAULT_TIKTOK_SRC, -): void { - const { document } = getEnv(env); - const script = (document as Document).createElement('script'); - script.src = src; - script.async = true; - (document as Document).head.appendChild(script); -} +/** The TikTok Pixel library, loaded through `loadVendor`. */ +export const SCRIPT_SRC = 'https://analytics.tiktok.com/i18n/pixel/events.js'; interface TTQFunction { (...args: unknown[]): void; From afd49db92803df375eaaa7c287c01db541440ec8 Mon Sep 17 00:00:00 2001 From: alexanderkirtzel Date: Wed, 21 Oct 2026 06:15:16 +0000 Subject: [PATCH 16/41] kit batch off --- .../src/dev/__tests__/conformance.test.ts | 20 +++++++++++++++++++ .../__tests__/fixture-package/src/index.ts | 14 +++++++++++++ packages/collector/src/dev/runExamples.ts | 3 +++ 3 files changed, 37 insertions(+) diff --git a/packages/collector/src/dev/__tests__/conformance.test.ts b/packages/collector/src/dev/__tests__/conformance.test.ts index 55538db89..a83147299 100644 --- a/packages/collector/src/dev/__tests__/conformance.test.ts +++ b/packages/collector/src/dev/__tests__/conformance.test.ts @@ -5,6 +5,7 @@ import { conformance } from '../conformance'; import { fixtures } from './fixtures'; import { batching, + batchingByDefault, conditional, echoing, floating, @@ -12,6 +13,7 @@ import { loud, rawHttps, swallowing, + swallowingByDefault, } from './fixture-package/src'; import { dropping, @@ -88,6 +90,24 @@ describe('conformance kit', () => { ).rejects.toMatchObject({ items: [9] }); }); + it('runs the push path of a destination that batches by default, and item 9 on its batch', async () => { + await expect( + conformance('destination', batchingByDefault, { + profile: 'http', + ...fixtures, + }), + ).resolves.toBeUndefined(); + }); + + it('still fails item 9 for a default batch that hides a failed entry', async () => { + await expect( + conformance('destination', swallowingByDefault, { + profile: 'http', + ...fixtures, + }), + ).rejects.toMatchObject({ items: [9] }); + }); + it('fails a destination without a silent example on item 11 only', async () => { const { silent, ...examples } = fixtures.examples; await expect( diff --git a/packages/collector/src/dev/__tests__/fixture-package/src/index.ts b/packages/collector/src/dev/__tests__/fixture-package/src/index.ts index c5ff8b2bd..69fb6566e 100644 --- a/packages/collector/src/dev/__tests__/fixture-package/src/index.ts +++ b/packages/collector/src/dev/__tests__/fixture-package/src/index.ts @@ -141,3 +141,17 @@ export const loud: Destination.Instance = { } }, }; + +/** Batches by default (shared rule 7) and reports the failed entries. */ +export const batchingByDefault: Destination.Instance = { + ...batching, + type: 'batching-by-default', + config: { batch: { size: 15, wait: 10_000 } }, +}; + +/** Batches by default and hides a failed entry (item 9). */ +export const swallowingByDefault: Destination.Instance = { + ...swallowing, + type: 'swallowing-by-default', + config: { batch: { size: 15, wait: 10_000 } }, +}; diff --git a/packages/collector/src/dev/runExamples.ts b/packages/collector/src/dev/runExamples.ts index 86c4884af..00d4ad0e6 100644 --- a/packages/collector/src/dev/runExamples.ts +++ b/packages/collector/src/dev/runExamples.ts @@ -381,6 +381,9 @@ export function openDestination( [STEP_ID]: { code: instrumentDestination(step, setPhase, setup), config: { + // The push path: a destination's default `config.batch` is off + // unless the run sets its own batch (item 9 tests pushBatch). + batch: undefined, ...setup.config, ...(setup.settings !== undefined ? { settings: settingsOf(setup) } From 0551558e4440cef76dc63b27504072512956ea84 Mon Sep 17 00:00:00 2001 From: alexanderkirtzel Date: Wed, 21 Oct 2026 06:20:49 +0000 Subject: [PATCH 17/41] kit declared skips --- .../src/dev/__tests__/conformance.test.ts | 70 +++++++++++++ .../__tests__/fixture-package/src/index.ts | 15 +++ packages/collector/src/dev/conformance.ts | 99 ++++++++++++++----- 3 files changed, 159 insertions(+), 25 deletions(-) diff --git a/packages/collector/src/dev/__tests__/conformance.test.ts b/packages/collector/src/dev/__tests__/conformance.test.ts index a83147299..6770b9c00 100644 --- a/packages/collector/src/dev/__tests__/conformance.test.ts +++ b/packages/collector/src/dev/__tests__/conformance.test.ts @@ -10,6 +10,7 @@ import { echoing, floating, good, + grouping, loud, rawHttps, swallowing, @@ -108,6 +109,75 @@ describe('conformance kit', () => { ).rejects.toMatchObject({ items: [9] }); }); + describe('item 10 accepts a declared skip for a silent group rule', () => { + const groupRule = { silent: true }; + const groupWithoutData: Flow.StepExample = { + public: false, + description: 'A group rule without a group sends nothing.', + in: { name: 'company update' }, + mapping: groupRule, + out: [], + }; + const groupExamples: Flow.StepExamples = { + ...fixtures.examples, + groupWithData: { + in: { name: 'company update', data: { group: 'acme' } }, + mapping: groupRule, + out: [ + [ + 'fetch', + 'https://127.0.0.1:9/e', + { method: 'POST', body: '{"group":"acme"}' }, + ], + ], + }, + groupWithoutData, + }; + + it('passes with the declared skip, confirmed by its own run', async () => { + await expect( + conformance('destination', grouping, { + profile: 'http', + ...fixtures, + examples: groupExamples, + }), + ).resolves.toBeUndefined(); + }); + + it('fails item 10 without the declared skip', async () => { + const { groupWithoutData: _, ...undeclared } = groupExamples; + await expect( + conformance('destination', grouping, { + profile: 'http', + ...fixtures, + examples: undeclared, + }), + ).rejects.toMatchObject({ items: [10] }); + }); + + // The declared skip carries the silent rule, so item 11 runs it too. + it('fails items 10 and 11 when the declared skip sends when it runs', async () => { + await expect( + conformance('destination', grouping, { + profile: 'http', + ...fixtures, + examples: { + ...groupExamples, + groupWithoutData: { + ...groupWithoutData, + in: { name: 'company update', custom: { group: 'acme' } }, + }, + }, + }), + ).rejects.toMatchObject({ + items: [10, 11], + message: expect.stringContaining( + 'declared skip groupWithoutData: out differs', + ), + }); + }); + }); + it('fails a destination without a silent example on item 11 only', async () => { const { silent, ...examples } = fixtures.examples; await expect( diff --git a/packages/collector/src/dev/__tests__/fixture-package/src/index.ts b/packages/collector/src/dev/__tests__/fixture-package/src/index.ts index 69fb6566e..4bc37c213 100644 --- a/packages/collector/src/dev/__tests__/fixture-package/src/index.ts +++ b/packages/collector/src/dev/__tests__/fixture-package/src/index.ts @@ -155,3 +155,18 @@ export const swallowingByDefault: Destination.Instance = { type: 'swallowing-by-default', config: { batch: { size: 15, wait: 10_000 } }, }; + +/** + * A silent group rule: no default call, a group call when the event names a + * group (in data or custom), nothing at all without one (a declared skip). + */ +export const grouping: Destination.Instance = { + ...good, + type: 'grouping', + push: async (event, context) => { + if (!context.rule?.silent) return good.push(event, context); + const group = event.data.group ?? event.custom.group; + if (typeof group === 'string') + await post(requireUrl(context.config), JSON.stringify({ group })); + }, +}; diff --git a/packages/collector/src/dev/conformance.ts b/packages/collector/src/dev/conformance.ts index 57afd3e30..bb5cbdbad 100644 --- a/packages/collector/src/dev/conformance.ts +++ b/packages/collector/src/dev/conformance.ts @@ -669,13 +669,22 @@ async function destinationItem( return batchProblem(step, ctx); case 10: - return strippedProblem(ctx, async (target, event) => { - const run = await runDestination(step, ctx, target, base, event); - return { - step: run, - delivered: Boolean(run.result.done?.[STEP_ID]) && !run.effects.length, - }; - }); + return strippedProblem( + ctx, + async (target, event) => { + const run = await runDestination(step, ctx, target, base, event); + return { + step: run, + delivered: + Boolean(run.result.done?.[STEP_ID]) && !run.effects.length, + }; + }, + (example) => + checkDestinationExample(step, example, { + settings: options.settings, + env: ctx.runEnv ? options.env : undefined, + }), + ); case 11: { const silent = Object.entries(options.examples).filter( @@ -818,6 +827,7 @@ async function strippedProblem( target: EventExample, event: WalkerOS.DeepPartialEvent, ) => Promise<{ step: Step; delivered: boolean }>, + confirm: (example: Flow.StepExample) => Promise, ): Promise { // An example whose own out is [] sends nothing by design (silent, ignore). const sending = ctx.events.filter(({ example }) => sends(example)); @@ -827,38 +837,61 @@ async function strippedProblem( const { user, data, globals, context, ...stripped } = target.event; const { step, delivered } = await run(target, stripped); if (failedCode(step) === 'missing-input') { - if (!declaresMissingInput(ctx.options.examples, stripped)) + if (!declaredFor(ctx, target, stripped, 'missing-input')) problems.push( `example ${target.name} fails missing-input without an out: [] example for that shape with a description`, ); - } else if (delivered) + continue; + } + if (!delivered) continue; + // Recipe R-d's other answer: a declared skip for that shape, confirmed. + const skip = declaredFor(ctx, target, stripped, 'skip'); + if (!skip) { problems.push( - `example ${target.name} without user, data, globals and context counts as delivered but nothing was sent`, + `example ${target.name} without user, data, globals and context counts as delivered but nothing was sent (declare missing-input, or a skip: public: false, the same rule, out: [], a description)`, ); + continue; + } + for (const problem of await confirm(skip.example)) + problems.push(`declared skip ${skip.name}: ${problem}`); } return problems.length ? problems.join(', ') : undefined; } const SHAPE_KEYS = ['user', 'data', 'globals', 'context']; -function declaresMissingInput( - examples: Flow.StepExamples, +/** + * The example a package declares for a stripped event's shape (an event `in` + * of that name without user, data, globals and context, `out: []`, a + * description): with `outcome.code: 'missing-input'` (a reported failure), + * or as a declared skip (`public: false`, no outcome, the target's rule). + */ +function declaredFor( + ctx: Context, + target: EventExample, stripped: WalkerOS.DeepPartialEvent, -): boolean { - return Object.values(examples).some( - (example) => + as: 'missing-input' | 'skip', +): { name: string; example: Flow.StepExample } | undefined { + const found = Object.entries(ctx.options.examples).find( + ([, example]) => isEvent(example.in) && example.in.name === stripped.name && - example.out?.length === 0 && + Array.isArray(example.out) && + example.out.length === 0 && Boolean(example.description) && - example.outcome?.code === 'missing-input' && SHAPE_KEYS.every((key) => { const value = isObject(example.in) ? example.in[key] : undefined; return ( value === undefined || (isObject(value) && !Object.keys(value).length) ); - }), + }) && + (as === 'missing-input' + ? example.outcome?.code === 'missing-input' + : example.public === false && + !example.outcome && + isEqual(example.mapping, target.example.mapping)), ); + return found && { name: found[0], example: found[1] }; } /* ------------------------------------------------------------------------ */ @@ -1015,13 +1048,29 @@ async function transformerItem( case 10: if (!ctx.events.length) return ingestInputProblem(step, ctx); - return strippedProblem(ctx, async (_, stripped) => { - const run = await runTransformer(step, ctx, base, stripped); - return { - step: run, - delivered: run.result.ok && !run.effects.length, - }; - }); + return strippedProblem( + ctx, + async (_, stripped) => { + const run = await runTransformer(step, ctx, base, stripped); + return { + step: run, + delivered: run.result.ok && !run.effects.length, + }; + }, + (example) => + runExamples( + 'transformer', + step, + { skip: example }, + { + settings: options.settings, + env: options.env, + }, + ).then( + () => [], + (error: unknown) => [textOf(error)], + ), + ); } return undefined; } From 52a4a717da4d53f2f7869b5cfa71caaafeed290c Mon Sep 17 00:00:00 2001 From: alexanderkirtzel Date: Wed, 21 Oct 2026 06:32:56 +0000 Subject: [PATCH 18/41] server steps --- packages/cli/examples/flow-complete.json | 2 +- .../examples/flow-complete-simulate.test.ts | 1 + .../amplitude/src/__tests__/config.test.ts | 15 +- .../src/__tests__/conformance.test.ts | 45 +++ .../amplitude/src/__tests__/failure.test.ts | 82 ++++ .../src/__tests__/stepExamples.test.ts | 2 + .../destinations/amplitude/src/config.ts | 12 +- .../amplitude/src/examples/env.ts | 20 +- .../amplitude/src/examples/step.ts | 29 ++ .../destinations/amplitude/src/index.ts | 17 +- .../server/destinations/amplitude/src/push.ts | 215 ++++++++-- .../amplitude/src/schemas/settings.ts | 158 ++++---- .../destinations/amplitude/src/types/index.ts | 25 +- .../clickhouse/src/__tests__/classify.test.ts | 75 ++-- .../clickhouse/src/__tests__/config.test.ts | 20 +- .../src/__tests__/conformance.test.ts | 22 ++ .../clickhouse/src/__tests__/init.test.ts | 8 +- .../clickhouse/src/__tests__/insert.test.ts | 10 +- .../clickhouse/src/__tests__/push.test.ts | 56 ++- .../src/__tests__/pushBatch.test.ts | 15 +- .../destinations/clickhouse/src/classify.ts | 38 +- .../destinations/clickhouse/src/config.ts | 34 +- .../destinations/clickhouse/src/deliver.ts | 49 ++- .../clickhouse/src/examples/step.ts | 9 + .../destinations/clickhouse/src/insert.ts | 4 +- .../destinations/clickhouse/src/push.ts | 5 +- .../destinations/clickhouse/src/pushBatch.ts | 10 +- .../clickhouse/src/schemas/settings.ts | 78 ++-- .../file/src/__tests__/conformance.test.ts | 31 ++ .../file/src/__tests__/failure.test.ts | 47 +++ .../file/src/__tests__/stepExamples.test.ts | 10 +- .../destinations/file/src/examples/env.ts | 3 +- .../destinations/file/src/examples/step.ts | 10 + .../server/destinations/file/src/helpers.ts | 56 ++- .../server/destinations/file/src/index.ts | 104 ++--- .../destinations/file/src/schemas/settings.ts | 50 +-- .../destinations/file/src/types/index.ts | 15 +- .../gcp/src/__tests__/conformance.test.ts | 43 ++ .../gcp/src/bigquery/__tests__/index.test.ts | 244 ++++++------ .../bigquery/__tests__/stepExamples.test.ts | 7 +- .../destinations/gcp/src/bigquery/config.ts | 12 +- .../gcp/src/bigquery/examples/step.ts | 9 + .../destinations/gcp/src/bigquery/index.ts | 82 ++-- .../destinations/gcp/src/bigquery/push.ts | 68 ++-- .../gcp/src/bigquery/pushBatch.ts | 131 +++--- .../gcp/src/bigquery/schemas/settings.ts | 70 ++-- .../destinations/gcp/src/bigquery/writer.ts | 39 +- .../destinations/gcp/src/pubsub/config.ts | 40 +- .../gcp/src/pubsub/examples/step.ts | 9 + .../destinations/gcp/src/pubsub/push.ts | 56 ++- .../gcp/src/pubsub/schemas/settings.ts | 86 ++-- .../destinations/gcp/src/pubsub/setup.ts | 5 +- .../hubspot/src/__tests__/batch.test.ts | 56 +++ .../hubspot/src/__tests__/conformance.test.ts | 51 +++ .../hubspot/src/__tests__/init.test.ts | 67 +--- .../server/destinations/hubspot/src/config.ts | 23 +- .../destinations/hubspot/src/examples/step.ts | 57 ++- .../server/destinations/hubspot/src/index.ts | 47 +-- .../server/destinations/hubspot/src/push.ts | 264 ++++++++----- .../hubspot/src/schemas/settings.ts | 93 ++--- .../destinations/hubspot/src/types/index.ts | 14 +- .../kafka/src/__tests__/conformance.test.ts | 30 ++ .../kafka/src/__tests__/init.test.ts | 2 +- .../src/__tests__/push-missing-topic.test.ts | 72 +++- .../server/destinations/kafka/src/config.ts | 26 +- .../destinations/kafka/src/examples/step.ts | 11 + .../server/destinations/kafka/src/index.ts | 25 +- .../server/destinations/kafka/src/push.ts | 82 ++-- .../kafka/src/schemas/settings.ts | 266 +++++++------ .../destinations/kafka/src/types/index.ts | 2 + .../src/__tests__/conformance.test.ts | 32 ++ .../pinterest/src/__tests__/index.test.ts | 2 +- .../src/__tests__/stepExamples.test.ts | 7 +- .../destinations/pinterest/src/config.ts | 20 +- .../pinterest/src/examples/step.ts | 32 ++ .../destinations/pinterest/src/index.ts | 5 +- .../server/destinations/pinterest/src/push.ts | 35 +- .../pinterest/src/schemas/settings.ts | 102 ++--- .../posthog/src/__tests__/conformance.test.ts | 35 ++ .../posthog/src/__tests__/failure.test.ts | 121 ++++++ .../src/__tests__/stepExamples.test.ts | 2 + .../server/destinations/posthog/src/config.ts | 8 +- .../destinations/posthog/src/examples/step.ts | 29 ++ .../server/destinations/posthog/src/index.ts | 16 +- .../server/destinations/posthog/src/push.ts | 150 ++++++- .../posthog/src/schemas/settings.ts | 140 ++++--- .../destinations/posthog/src/types/index.ts | 9 +- .../redis/src/__tests__/conformance.test.ts | 21 + .../server/destinations/redis/src/config.ts | 17 +- .../destinations/redis/src/examples/step.ts | 11 + .../server/destinations/redis/src/index.ts | 29 +- .../server/destinations/redis/src/push.ts | 42 +- .../redis/src/schemas/settings.ts | 94 ++--- .../destinations/redis/src/types/index.ts | 4 + .../src/__tests__/conformance.test.ts | 54 +++ .../rudderstack/src/__tests__/failure.test.ts | 83 ++++ .../rudderstack/src/__tests__/init.test.ts | 11 +- .../src/__tests__/stepExamples.test.ts | 20 +- .../destinations/rudderstack/src/config.ts | 20 +- .../rudderstack/src/examples/env.ts | 28 +- .../rudderstack/src/examples/step.ts | 60 +++ .../destinations/rudderstack/src/index.ts | 48 +-- .../destinations/rudderstack/src/push.ts | 374 +++++++++++++----- .../rudderstack/src/schemas/settings.ts | 184 ++++----- .../rudderstack/src/types/index.ts | 40 +- .../segment/src/__tests__/conformance.test.ts | 51 +++ .../segment/src/__tests__/failure.test.ts | 76 ++++ .../segment/src/__tests__/init.test.ts | 11 +- .../src/__tests__/stepExamples.test.ts | 18 +- .../server/destinations/segment/src/config.ts | 12 +- .../destinations/segment/src/examples/env.ts | 17 +- .../destinations/segment/src/examples/step.ts | 56 +++ .../server/destinations/segment/src/index.ts | 45 +-- .../server/destinations/segment/src/push.ts | 327 +++++++++++---- .../segment/src/schemas/settings.ts | 166 ++++---- .../destinations/segment/src/types/index.ts | 36 +- .../sqlite/src/__tests__/conformance.test.ts | 23 ++ .../sqlite/src/__tests__/init.test.ts | 78 +++- .../server/destinations/sqlite/src/config.ts | 14 +- .../destinations/sqlite/src/examples/step.ts | 11 + .../server/destinations/sqlite/src/index.ts | 58 +-- .../server/destinations/sqlite/src/push.ts | 117 ++++-- .../sqlite/src/schemas/settings.ts | 62 +-- .../destinations/sqlite/src/types/index.ts | 10 +- .../tiktok/src/__tests__/conformance.test.ts | 32 ++ .../tiktok/src/__tests__/index.test.ts | 6 +- .../tiktok/src/__tests__/stepExamples.test.ts | 7 +- .../server/destinations/tiktok/src/config.ts | 20 +- .../destinations/tiktok/src/examples/step.ts | 41 ++ .../server/destinations/tiktok/src/index.ts | 5 +- .../server/destinations/tiktok/src/push.ts | 41 +- .../tiktok/src/schemas/settings.ts | 76 ++-- .../fs/src/__tests__/conformance.test.ts | 39 ++ .../stores/fs/src/__tests__/store.test.ts | 28 +- .../server/stores/fs/src/examples/step.ts | 10 + .../server/stores/fs/src/schemas/settings.ts | 18 +- packages/server/stores/fs/src/store.ts | 111 ++++-- packages/server/stores/fs/src/types.ts | 13 + .../gcs/src/__tests__/conformance.test.ts | 29 ++ .../stores/gcs/src/__tests__/store.test.ts | 77 +++- .../server/stores/gcs/src/examples/step.ts | 10 + .../server/stores/gcs/src/schemas/settings.ts | 40 +- packages/server/stores/gcs/src/store.ts | 111 +++--- .../s3/src/__tests__/conformance.test.ts | 69 ++++ .../stores/s3/src/__tests__/store.test.ts | 75 +++- .../server/stores/s3/src/examples/step.ts | 10 + .../server/stores/s3/src/schemas/settings.ts | 34 +- packages/server/stores/s3/src/store.ts | 172 ++++---- .../sheets/src/__tests__/conformance.test.ts | 34 ++ .../stores/sheets/src/__tests__/store.test.ts | 76 +++- .../server/stores/sheets/src/examples/step.ts | 10 + .../stores/sheets/src/schemas/settings.ts | 74 ++-- packages/server/stores/sheets/src/store.ts | 151 ++++--- .../server/transformers/fingerprint/README.md | 6 +- .../src/__tests__/conformance.test.ts | 32 ++ .../fingerprint/src/__tests__/privacy.test.ts | 29 +- .../fingerprint/src/__tests__/schemas.test.ts | 11 +- .../src/__tests__/stepExamples.test.ts | 18 +- .../src/__tests__/transformer.test.ts | 35 +- .../fingerprint/src/examples/step.ts | 31 +- .../transformers/fingerprint/src/hints.ts | 2 +- .../fingerprint/src/schemas/settings.ts | 5 +- .../fingerprint/src/transformer.ts | 61 +-- .../transformers/fingerprint/src/types.ts | 5 +- .../ga4/src/__tests__/conformance.test.ts | 25 ++ .../ga4/src/__tests__/examples.test.ts | 7 +- .../ga4/src/__tests__/transformer.test.ts | 41 +- .../transformers/ga4/src/examples/step.ts | 9 + packages/transformers/ga4/src/transformer.ts | 27 +- 169 files changed, 5607 insertions(+), 2700 deletions(-) create mode 100644 packages/server/destinations/amplitude/src/__tests__/conformance.test.ts create mode 100644 packages/server/destinations/amplitude/src/__tests__/failure.test.ts create mode 100644 packages/server/destinations/clickhouse/src/__tests__/conformance.test.ts create mode 100644 packages/server/destinations/file/src/__tests__/conformance.test.ts create mode 100644 packages/server/destinations/file/src/__tests__/failure.test.ts create mode 100644 packages/server/destinations/gcp/src/__tests__/conformance.test.ts create mode 100644 packages/server/destinations/hubspot/src/__tests__/batch.test.ts create mode 100644 packages/server/destinations/hubspot/src/__tests__/conformance.test.ts create mode 100644 packages/server/destinations/kafka/src/__tests__/conformance.test.ts create mode 100644 packages/server/destinations/pinterest/src/__tests__/conformance.test.ts create mode 100644 packages/server/destinations/posthog/src/__tests__/conformance.test.ts create mode 100644 packages/server/destinations/posthog/src/__tests__/failure.test.ts create mode 100644 packages/server/destinations/redis/src/__tests__/conformance.test.ts create mode 100644 packages/server/destinations/rudderstack/src/__tests__/conformance.test.ts create mode 100644 packages/server/destinations/rudderstack/src/__tests__/failure.test.ts create mode 100644 packages/server/destinations/segment/src/__tests__/conformance.test.ts create mode 100644 packages/server/destinations/segment/src/__tests__/failure.test.ts create mode 100644 packages/server/destinations/sqlite/src/__tests__/conformance.test.ts create mode 100644 packages/server/destinations/tiktok/src/__tests__/conformance.test.ts create mode 100644 packages/server/stores/fs/src/__tests__/conformance.test.ts create mode 100644 packages/server/stores/gcs/src/__tests__/conformance.test.ts create mode 100644 packages/server/stores/s3/src/__tests__/conformance.test.ts create mode 100644 packages/server/stores/sheets/src/__tests__/conformance.test.ts create mode 100644 packages/server/transformers/fingerprint/src/__tests__/conformance.test.ts create mode 100644 packages/transformers/ga4/src/__tests__/conformance.test.ts diff --git a/packages/cli/examples/flow-complete.json b/packages/cli/examples/flow-complete.json index d082aad2a..872ffda35 100644 --- a/packages/cli/examples/flow-complete.json +++ b/packages/cli/examples/flow-complete.json @@ -4273,7 +4273,7 @@ [ "PubSub.topic.publishMessage", { - "data": "{\"name\":\"order complete\",\"data\":{\"id\":\"ORD-100\",\"value\":149.8,\"currency\":\"EUR\",\"coupon\":\"WELCOME10\"},\"context\":{},\"globals\":{\"language\":\"en\",\"pageGroup\":\"checkout\"},\"custom\":{},\"user\":{\"device\":\"d3v1c3\",\"session\":\"s3ss10n\",\"id\":\"cust-42\",\"email\":\"4cdea64501d6109d0a25750a7ce1d5c87e5fc927c61b674f191b73e28b582c93\",\"hash\":\"cea3ca84340feb4a\",\"botScore\":null,\"botCategory\":\"unknown\",\"ltv\":420,\"segment\":\"loyal\"},\"nested\":[{\"entity\":\"product\",\"data\":{\"id\":\"SKU-1\",\"name\":\"Trail Runner\",\"price\":129.9,\"quantity\":1}},{\"entity\":\"product\",\"data\":{\"id\":\"SKU-2\",\"name\":\"Running Socks\",\"price\":9.95,\"quantity\":2}}],\"consent\":{\"functional\":true,\"marketing\":true},\"id\":\"ev-order-complete\",\"trigger\":\"load\",\"entity\":\"order\",\"action\":\"complete\",\"timestamp\":1700000000000,\"timing\":0,\"source\":{\"type\":\"express\",\"platform\":\"server\",\"url\":\"https://www.example.com/checkout/thanks\",\"valid\":true,\"count\":1,\"release\":{\"server\":\"4.6.1\"}}}" + "data": "{\"name\":\"order complete\",\"data\":{\"id\":\"ORD-100\",\"value\":149.8,\"currency\":\"EUR\",\"coupon\":\"WELCOME10\"},\"context\":{},\"globals\":{\"language\":\"en\",\"pageGroup\":\"checkout\"},\"custom\":{},\"user\":{\"device\":\"d3v1c3\",\"session\":\"s3ss10n\",\"id\":\"cust-42\",\"email\":\"4cdea64501d6109d0a25750a7ce1d5c87e5fc927c61b674f191b73e28b582c93\",\"hash\":\"7f1b2860213c9d3d\",\"botScore\":0,\"botCategory\":\"human\",\"ltv\":420,\"segment\":\"loyal\"},\"nested\":[{\"entity\":\"product\",\"data\":{\"id\":\"SKU-1\",\"name\":\"Trail Runner\",\"price\":129.9,\"quantity\":1}},{\"entity\":\"product\",\"data\":{\"id\":\"SKU-2\",\"name\":\"Running Socks\",\"price\":9.95,\"quantity\":2}}],\"consent\":{\"functional\":true,\"marketing\":true},\"id\":\"ev-order-complete\",\"trigger\":\"load\",\"entity\":\"order\",\"action\":\"complete\",\"timestamp\":1700000000000,\"timing\":0,\"source\":{\"type\":\"express\",\"platform\":\"server\",\"url\":\"https://www.example.com/checkout/thanks\",\"valid\":true,\"count\":1,\"release\":{\"server\":\"4.6.1\"}}}" } ] ] diff --git a/packages/cli/src/__tests__/integration/examples/flow-complete-simulate.test.ts b/packages/cli/src/__tests__/integration/examples/flow-complete-simulate.test.ts index 5e18d15b5..ca9770f17 100644 --- a/packages/cli/src/__tests__/integration/examples/flow-complete-simulate.test.ts +++ b/packages/cli/src/__tests__/integration/examples/flow-complete-simulate.test.ts @@ -103,6 +103,7 @@ const INGEST: Record> = { 'server.destinations.meta.purchase': CHROME, 'server.destinations.piwikpro.orderWithGoal': CHROME, 'server.destinations.datamanager.conversionWithGclid': CHROME, + 'server.destinations.pubsub.everyEvent': CHROME, }; /** diff --git a/packages/server/destinations/amplitude/src/__tests__/config.test.ts b/packages/server/destinations/amplitude/src/__tests__/config.test.ts index 1d7834c66..8e0bf27b6 100644 --- a/packages/server/destinations/amplitude/src/__tests__/config.test.ts +++ b/packages/server/destinations/amplitude/src/__tests__/config.test.ts @@ -2,18 +2,21 @@ jest.mock('@amplitude/analytics-node', () => ({ __esModule: true, })); -import { createMockLogger } from '@walkeros/core'; import { getConfig } from '../config'; describe('getConfig', () => { - it('throws when apiKey is missing', () => { - const logger = createMockLogger(); - expect(() => getConfig({}, logger)).toThrow('apiKey missing'); + it('throws a config StepError when apiKey is missing', () => { + expect(() => getConfig({})).toThrow( + expect.objectContaining({ + name: 'StepError', + code: 'config', + message: 'Config settings apiKey missing', + }), + ); }); it('returns config with defaults when apiKey is provided', () => { - const logger = createMockLogger(); - const config = getConfig({ settings: { apiKey: 'test-key' } }, logger); + const config = getConfig({ settings: { apiKey: 'test-key' } }); expect(config.settings.apiKey).toBe('test-key'); }); }); diff --git a/packages/server/destinations/amplitude/src/__tests__/conformance.test.ts b/packages/server/destinations/amplitude/src/__tests__/conformance.test.ts new file mode 100644 index 000000000..9d5981481 --- /dev/null +++ b/packages/server/destinations/amplitude/src/__tests__/conformance.test.ts @@ -0,0 +1,45 @@ +import path from 'path'; +import { conformance } from '@walkeros/collector/dev'; +import { destinationAmplitude } from '..'; +import { examples, schemas } from '../dev'; + +// The runs without env use the SDK module: a stand-in that answers every +// call as sent. +jest.mock('@amplitude/analytics-node', () => { + const sent = () => ({ promise: Promise.resolve({ code: 200 }) }); + class Chain { + set() { + return this; + } + } + return { + __esModule: true, + init: () => ({ promise: Promise.resolve() }), + track: sent, + identify: sent, + revenue: sent, + setGroup: sent, + groupIdentify: sent, + setOptOut: () => undefined, + flush: () => ({ promise: Promise.resolve() }), + Identify: Chain, + Revenue: Chain, + }; +}); + +describe('amplitude conformance', () => { + beforeEach(() => { + jest.useRealTimers(); + }); + + it('conforms', async () => { + await conformance('destination', destinationAmplitude, { + profile: 'sdk-env', + examples: examples.step, + schema: schemas.settings, + settings: { apiKey: 'test-api-key' }, + env: { push: examples.env.push, simulation: examples.env.simulation }, + root: path.join(__dirname, '..', '..'), + }); + }); +}); diff --git a/packages/server/destinations/amplitude/src/__tests__/failure.test.ts b/packages/server/destinations/amplitude/src/__tests__/failure.test.ts new file mode 100644 index 000000000..243ab4fcd --- /dev/null +++ b/packages/server/destinations/amplitude/src/__tests__/failure.test.ts @@ -0,0 +1,82 @@ +jest.mock('@amplitude/analytics-node', () => ({ __esModule: true })); + +import { startFlow } from '@walkeros/collector'; +import { clone, createEvent, getEvent } from '@walkeros/core'; +import { destinationAmplitude } from '..'; +import { examples } from '../dev'; +import type { Env } from '../types'; + +/** + * The examples env, whose track answers the way the SDK does: it resolves + * with the HTTP code, 503 for the events `fails` picks. + */ +function envFailing(fails: (options?: { user_id?: string }) => boolean): Env { + const env: Env = clone(examples.env.push); + const amplitude = env.amplitude; + if (!amplitude) throw new Error('examples env has no amplitude'); + amplitude.track = (_type, _properties, options) => ({ + promise: Promise.resolve( + fails(options) + ? { code: 503, message: 'Service Unavailable' } + : { code: 200, message: 'OK' }, + ), + }); + return env; +} + +describe('amplitude send failures', () => { + it('fails a single push (batching off) when Amplitude answers with an error code', async () => { + const { collector } = await startFlow({ + logger: { level: 'ERROR', handler: () => undefined }, + destinations: { + amplitude: { + code: destinationAmplitude, + config: { settings: { apiKey: 'test' }, batch: undefined }, + env: envFailing(() => true), + }, + }, + }); + + const result = await collector.push(getEvent('page view')); + + expect(result.ok).toBe(false); + expect(result.failed?.amplitude?.reason).toMatchObject({ + code: 'transport', + status: 503, + retryable: true, + message: 'Amplitude track failed (code 503)', + }); + expect(collector.destinations.amplitude.dlq).toHaveLength(1); + + await collector.command('shutdown'); + }); + + it('gives each entry of a batch its own outcome', async () => { + const { collector } = await startFlow({ + logger: { level: 'ERROR', handler: () => undefined }, + destinations: { + amplitude: { + code: destinationAmplitude, + config: { + settings: { + apiKey: 'test', + identify: { map: { user_id: 'user.id' } }, + }, + }, + env: envFailing((options) => options?.user_id === 'bad'), + }, + }, + }); + + for (const id of ['user-a', 'bad', 'user-b']) + await collector.push(createEvent({ name: 'page view', user: { id } })); + await collector.command('flush'); + + const status = collector.status.destinations.amplitude; + expect(status.count).toBe(2); + expect(status.failed).toBe(1); + expect(collector.destinations.amplitude.dlq).toHaveLength(1); + + await collector.command('shutdown'); + }); +}); diff --git a/packages/server/destinations/amplitude/src/__tests__/stepExamples.test.ts b/packages/server/destinations/amplitude/src/__tests__/stepExamples.test.ts index d4e4650bc..38bf5f19f 100644 --- a/packages/server/destinations/amplitude/src/__tests__/stepExamples.test.ts +++ b/packages/server/destinations/amplitude/src/__tests__/stepExamples.test.ts @@ -207,6 +207,8 @@ describe('amplitude server destination -- step examples', () => { }, }); await elb(event); + // The destination batches by default: send what it holds. + await elb('walker flush'); } const actual = collected().filter(([path]) => path !== 'amplitude.init'); diff --git a/packages/server/destinations/amplitude/src/config.ts b/packages/server/destinations/amplitude/src/config.ts index da19188df..ec8f05ce4 100644 --- a/packages/server/destinations/amplitude/src/config.ts +++ b/packages/server/destinations/amplitude/src/config.ts @@ -1,14 +1,12 @@ import type { Config, Settings, PartialConfig } from './types'; -import type { Logger } from '@walkeros/core'; +import { StepError } from '@walkeros/core'; -export function getConfig( - partialConfig: PartialConfig = {}, - logger: Logger.Instance, -): Config { - const settings = (partialConfig.settings || {}) as Partial; +export function getConfig(partialConfig: PartialConfig = {}): Config { + const settings: Partial = partialConfig.settings || {}; const { apiKey } = settings; - if (!apiKey) logger.throw('Config settings apiKey missing'); + if (!apiKey) + throw new StepError('Config settings apiKey missing', { code: 'config' }); const settingsConfig: Settings = { ...settings, diff --git a/packages/server/destinations/amplitude/src/examples/env.ts b/packages/server/destinations/amplitude/src/examples/env.ts index eeafc7ee5..1e8d20e77 100644 --- a/packages/server/destinations/amplitude/src/examples/env.ts +++ b/packages/server/destinations/amplitude/src/examples/env.ts @@ -69,12 +69,12 @@ class MockRevenue implements RevenueInstance { export const init: Env | undefined = { amplitude: { init: noopPromise, - track: noop, - identify: noop, - revenue: noop, + track: noopPromise, + identify: noopPromise, + revenue: noopPromise, setOptOut: noop, - setGroup: noop, - groupIdentify: noop, + setGroup: noopPromise, + groupIdentify: noopPromise, flush: noopPromise, Identify: MockIdentify, Revenue: MockRevenue, @@ -84,12 +84,12 @@ export const init: Env | undefined = { export const push: Env = { amplitude: { init: noopPromise, - track: noop, - identify: noop, - revenue: noop, + track: noopPromise, + identify: noopPromise, + revenue: noopPromise, setOptOut: noop, - setGroup: noop, - groupIdentify: noop, + setGroup: noopPromise, + groupIdentify: noopPromise, flush: noopPromise, Identify: MockIdentify, Revenue: MockRevenue, diff --git a/packages/server/destinations/amplitude/src/examples/step.ts b/packages/server/destinations/amplitude/src/examples/step.ts index 6610d3d6f..b7f9468af 100644 --- a/packages/server/destinations/amplitude/src/examples/step.ts +++ b/packages/server/destinations/amplitude/src/examples/step.ts @@ -429,3 +429,32 @@ export const consentGrantOptIn: AmplitudeStepExample = { settings: {} as Partial, out: [['amplitude.setOptOut', false]], }; + +/** A silent rule without identify, group or revenue mappings calls nothing. */ +export const silentRule: AmplitudeStepExample = { + public: false, + description: + 'A rule with silent: true and no identify, group or revenue mapping sends nothing to Amplitude.', + in: getEvent('page view', { timestamp: 1700000110 }), + mapping: { silent: true }, + out: [], +}; + +/** + * The silent group rule of groupAssignmentWithProperties on an event without + * data: no group name resolves and the rule suppresses the track, so nothing + * is sent. + */ +export const groupAssignmentWithoutData: AmplitudeStepExample = { + public: false, + description: + 'A silent group rule whose name finds no data sends nothing: an unmet mapping, not a failure, so the push returns normally.', + in: { + name: 'company update', + entity: 'company', + action: 'update', + timestamp: 1700000111, + }, + mapping: groupAssignmentWithProperties.mapping, + out: [], +}; diff --git a/packages/server/destinations/amplitude/src/index.ts b/packages/server/destinations/amplitude/src/index.ts index b359238f7..1db6ac21b 100644 --- a/packages/server/destinations/amplitude/src/index.ts +++ b/packages/server/destinations/amplitude/src/index.ts @@ -1,17 +1,18 @@ import type { WalkerOS } from '@walkeros/core'; import type { Destination, Env } from './types'; import { getConfig } from './config'; -import { push, getAmplitude } from './push'; +import { push, pushBatch, getAmplitude } from './push'; export * as DestinationAmplitude from './types'; export const destinationAmplitude: Destination = { type: 'amplitude', - config: {}, + // Batched by default, at the SDK's own former defaults (200 events, 10 s). + config: { batch: { size: 200, wait: 10_000 } }, - async init({ config: partialConfig, env, logger }) { - const config = getConfig(partialConfig, logger); + async init({ config: partialConfig, env }) { + const config = getConfig(partialConfig); const amp = getAmplitude(env as Record | undefined); // Destructure walkerOS-specific keys; the rest flow through to the SDK. @@ -23,13 +24,15 @@ export const destinationAmplitude: Destination = { ...nodeOptions } = config.settings; + // Each push and each batch flushes the SDK and waits until Amplitude + // answered its calls. await amp.init(apiKey, nodeOptions).promise; return config; }, - async push(event, context) { - return await push(event, context); - }, + push, + + pushBatch, on(type, context) { if (type !== 'consent' || !context.data) return; diff --git a/packages/server/destinations/amplitude/src/push.ts b/packages/server/destinations/amplitude/src/push.ts index f4d69ed96..65e1bc75c 100644 --- a/packages/server/destinations/amplitude/src/push.ts +++ b/packages/server/destinations/amplitude/src/push.ts @@ -1,8 +1,12 @@ +import type { Collector, Destination, WalkerOS } from '@walkeros/core'; import type { PushFn, + PushBatchFn, AmplitudeSDK, EventOptions, IdentifyInstance, + Rule, + Settings, } from './types'; import { getMappingValue, @@ -11,7 +15,9 @@ import { isNumber, isObject, isString, + isThenable, flattenIncludeSections, + StepError, } from '@walkeros/core'; import * as amplitudeSDK from '@amplitude/analytics-node'; @@ -67,6 +73,33 @@ export function getAmplitude( return amplitude ?? (amplitudeSDK as unknown as AmplitudeSDK); } +/** + * Hand one call to the SDK; the promise settles once Amplitude answered it. + * The SDK resolves the call's promise after the send with a result whose + * `code` is the HTTP status (0 for a failed request), never rejecting, so a + * code outside 2xx fails. An injected client may answer with a promise of its + * own. + */ +async function call(message: string, send: () => unknown): Promise { + let result: unknown; + try { + const answer: unknown = await send(); + result = + isObject(answer) && isThenable(answer.promise) + ? await answer.promise + : answer; + } catch (error) { + throw new StepError(message, { code: 'transport', cause: error }); + } + const code = + isObject(result) && isNumber(result.code) ? result.code : undefined; + if (code !== undefined && (code < 200 || code >= 300)) + throw new StepError(`${message} (code ${code})`, { + code: 'transport', + status: code || undefined, + }); +} + /** * Deterministic string -> positive integer hash (djb2). * Used for session_id when walkerOS provides a string session. @@ -172,13 +205,32 @@ function buildEventOptions( return opts; } -export const push: PushFn = async function ( - event, - { config, rule, data, collector, env }, -) { - const amp = getAmplitude(env as Record | undefined); - const settings = config.settings!; +/** A call's outcome, marked as handled until its batch reads it. */ +function held(sent: Promise): Promise { + sent.catch(() => undefined); + return sent; +} + +/** The settled calls of one event: the first failure, if any. */ +async function delivered(calls: Promise[]>) { + for (const result of await calls) + if (result.status === 'rejected') throw result.reason; +} + +/** + * Hand the calls of one event to the SDK (identify, group, group identify, + * revenue, track); each returned promise settles on its answer. + */ +async function enqueue( + amp: AmplitudeSDK, + event: WalkerOS.Event, + rule: Rule | undefined, + data: Destination.Data, + settings: Partial, + collector: Collector.Instance, +): Promise[]> { const mappingSettings = rule?.settings || {}; + const calls: Promise[] = []; // 1. Resolve identity -- rule-level override wins over destination-level. const identifyMapping = mappingSettings.identify ?? settings.identify; @@ -215,7 +267,14 @@ export const push: PushFn = async function ( // 4. Identify operations -- fires amplitude.identify() if op keys are present. if (identifyResolved) { const id = buildIdentify(amp, identifyResolved); - if (id) amp.identify(id, hasEventOptions ? eventOptions : undefined); + if (id) + calls.push( + held( + call('Amplitude identify failed', () => + amp.identify(id, hasEventOptions ? eventOptions : undefined), + ), + ), + ); } // 5. Group assignment @@ -228,13 +287,18 @@ export const push: PushFn = async function ( type?: unknown; name?: unknown; }; - if (isString(type) && (isString(name) || isArray(name))) { - amp.setGroup( - type, - name as string | string[], - hasEventOptions ? eventOptions : undefined, + if (isString(type) && (isString(name) || isArray(name))) + calls.push( + held( + call('Amplitude setGroup failed', () => + amp.setGroup( + type, + name as string | string[], + hasEventOptions ? eventOptions : undefined, + ), + ), + ), ); - } } } @@ -253,11 +317,17 @@ export const push: PushFn = async function ( if (isString(type) && isString(name)) { const id = buildIdentify(amp, resolved as Record); if (id) - amp.groupIdentify( - type, - name, - id, - hasEventOptions ? eventOptions : undefined, + calls.push( + held( + call('Amplitude groupIdentify failed', () => + amp.groupIdentify( + type, + name, + id, + hasEventOptions ? eventOptions : undefined, + ), + ), + ), ); } } @@ -275,9 +345,15 @@ export const push: PushFn = async function ( : []; for (const item of items) { if (!isObject(item)) continue; - amp.revenue( - buildRevenue(amp, item), - hasEventOptions ? eventOptions : undefined, + calls.push( + held( + call('Amplitude revenue failed', () => + amp.revenue( + buildRevenue(amp, item), + hasEventOptions ? eventOptions : undefined, + ), + ), + ), ); } } @@ -294,10 +370,99 @@ export const push: PushFn = async function ( ? (data as Record) : {}; - amp.track( - eventType, - eventProperties, - hasEventOptions ? eventOptions : undefined, + calls.push( + held( + call('Amplitude track failed', () => + amp.track( + eventType, + eventProperties, + hasEventOptions ? eventOptions : undefined, + ), + ), + ), ); } + + return calls; +} + +/** Ask the SDK to send what it holds. */ +async function flush(amp: AmplitudeSDK): Promise { + await amp.flush().promise; +} + +/** The settings init resolved, or a StepError. */ +function settingsOf( + settings: Partial | undefined, +): Partial { + if (!settings) + throw new StepError('Amplitude settings missing, init() not run', { + code: 'not-ready', + }); + return settings; +} + +/** One event: its calls handed to the SDK, sent, and their answers awaited. */ +export const push: PushFn = async function ( + event, + { config, rule, data, collector, env }, +) { + const amp = getAmplitude(env as Record | undefined); + const settings = settingsOf(config.settings); + const calls = await enqueue(amp, event, rule, data, settings, collector); + if (calls.length) { + const settled = Promise.allSettled(calls); + await flush(amp); + await delivered(settled); + } +}; + +/** + * A flushed `config.batch` batch: every entry's calls handed to the SDK, one + * flush, and each entry's own outcome from its answers. When nothing landed, + * the batch throws. + */ +export const pushBatch: PushBatchFn = async function ( + batch, + { config, collector, env }, +) { + const amp = getAmplitude(env as Record | undefined); + const settings = settingsOf(config.settings); + + // Each entry's answers, or why its calls could not be handed over. + const outcomes: Array<{ + settled?: Promise[]>; + error?: unknown; + }> = []; + for (const entry of batch.entries) { + try { + const calls = await enqueue( + amp, + entry.event, + entry.rule, + entry.data, + settings, + collector, + ); + outcomes.push({ settled: Promise.allSettled(calls) }); + } catch (error) { + outcomes.push({ error }); + } + } + + await flush(amp); + + const failed: Destination.BatchFailure[] = []; + for (const [index, { settled, error }] of outcomes.entries()) { + if (!settled) failed.push({ index, error }); + else + await delivered(settled).catch((reason: unknown) => + failed.push({ index, error: reason }), + ); + } + + // Nothing landed: a whole-batch failure, which the breaker counts. + if (failed.length && failed.length === batch.entries.length) + throw failed[0].error; + if (failed.length) return { failed }; }; diff --git a/packages/server/destinations/amplitude/src/schemas/settings.ts b/packages/server/destinations/amplitude/src/schemas/settings.ts index 7f3a104c3..3b5bc7a96 100644 --- a/packages/server/destinations/amplitude/src/schemas/settings.ts +++ b/packages/server/destinations/amplitude/src/schemas/settings.ts @@ -1,81 +1,85 @@ import { z } from '@walkeros/core/dev'; -export const SettingsSchema = z.object({ - apiKey: z - .string() - .min(1) - .describe( - 'Your Amplitude project API key. Find it in your Amplitude project settings under "General" -> "API Keys".', - ), - serverZone: z - .enum(['US', 'EU']) - .describe( - 'Amplitude data residency zone. Use EU for European data residency. Default: US.', - ) - .optional(), - flushIntervalMillis: z - .number() - .int() - .positive() - .describe('How often (in ms) to flush the event queue. Default: 10000.') - .optional(), - flushQueueSize: z - .number() - .int() - .positive() - .describe('Max queued events before a flush. Default: 200.') - .optional(), - flushMaxRetries: z - .number() - .int() - .nonnegative() - .describe('Max retries on failed flush. Default: 12.') - .optional(), - useBatch: z - .boolean() - .describe( - 'Use the Amplitude batch endpoint for higher rate limits. Recommended for high-volume server flows. Default: false.', - ) - .optional(), - minIdLength: z - .number() - .int() - .positive() - .describe('Minimum length for user_id and device_id fields.') - .optional(), - serverUrl: z - .string() - .url() - .describe('Custom server URL for proxies or self-hosted endpoints.') - .optional(), - optOut: z - .boolean() - .describe( - 'Initial opt-out state. When true, no events are sent. Default: false.', - ) - .optional(), - enableRequestBodyCompression: z - .boolean() - .describe('Enable gzip compression for request payloads. Default: false.') - .optional(), - identify: z - .unknown() - .describe( - 'walkerOS mapping value resolving to per-event identity. Keys: user_id, device_id, session_id, set, setOnce, add, append, prepend, preInsert, postInsert, remove, unset, clearAll.', - ) - .optional(), - eventOptions: z - .unknown() - .describe( - 'walkerOS mapping value resolving to per-event EventOptions. Keys: time, insert_id, ip, city, country, region, language, platform, os_name, os_version, device_brand, device_model, app_version, user_agent.', - ) - .optional(), - include: z - .array(z.string()) - .describe( - "walkerOS event sections to include as event_properties (like ['data', 'globals']).", - ) - .optional(), -}); +export const SettingsSchema = z + .object({ + apiKey: z + .string() + .min(1) + .describe( + 'Your Amplitude project API key. Find it in your Amplitude project settings under "General" -> "API Keys".', + ), + serverZone: z + .enum(['US', 'EU']) + .describe( + 'Amplitude data residency zone. Use EU for European data residency. Default: US.', + ) + .optional(), + flushIntervalMillis: z + .number() + .int() + .positive() + .describe('How often (in ms) to flush the event queue. Default: 10000.') + .optional(), + flushQueueSize: z + .number() + .int() + .positive() + .describe( + 'Events the SDK sends in one request. Each push and each config.batch batch flushes the SDK and waits until Amplitude answered its calls. Default: 200.', + ) + .optional(), + flushMaxRetries: z + .number() + .int() + .nonnegative() + .describe('Max retries on failed flush. Default: 12.') + .optional(), + useBatch: z + .boolean() + .describe( + 'Use the Amplitude batch endpoint for higher rate limits. Recommended for high-volume server flows. Default: false.', + ) + .optional(), + minIdLength: z + .number() + .int() + .positive() + .describe('Minimum length for user_id and device_id fields.') + .optional(), + serverUrl: z + .string() + .url() + .describe('Custom server URL for proxies or self-hosted endpoints.') + .optional(), + optOut: z + .boolean() + .describe( + 'Initial opt-out state. When true, no events are sent. Default: false.', + ) + .optional(), + enableRequestBodyCompression: z + .boolean() + .describe('Enable gzip compression for request payloads. Default: false.') + .optional(), + identify: z + .unknown() + .describe( + 'walkerOS mapping value resolving to per-event identity. Keys: user_id, device_id, session_id, set, setOnce, add, append, prepend, preInsert, postInsert, remove, unset, clearAll.', + ) + .optional(), + eventOptions: z + .unknown() + .describe( + 'walkerOS mapping value resolving to per-event EventOptions. Keys: time, insert_id, ip, city, country, region, language, platform, os_name, os_version, device_brand, device_model, app_version, user_agent.', + ) + .optional(), + include: z + .array(z.string()) + .describe( + "walkerOS event sections to include as event_properties (like ['data', 'globals']).", + ) + .optional(), + }) + .strict(); export type Settings = z.infer; diff --git a/packages/server/destinations/amplitude/src/types/index.ts b/packages/server/destinations/amplitude/src/types/index.ts index ffb9841be..04378f834 100644 --- a/packages/server/destinations/amplitude/src/types/index.ts +++ b/packages/server/destinations/amplitude/src/types/index.ts @@ -12,6 +12,14 @@ import type { DestinationServer } from '@walkeros/server-core'; * Key difference from web: no setUserId/setDeviceId/setSessionId. * Identity is per-event via EventOptions on every SDK call. */ +/** + * What an SDK call answers: a promise of its result, which resolves after + * the event was sent (with its HTTP `code`) instead of rejecting. + */ +export interface AmplitudeReturn { + promise: Promise; +} + export interface AmplitudeSDK { init: ( apiKey: string, @@ -21,20 +29,26 @@ export interface AmplitudeSDK { eventType: string, eventProperties?: Record, eventOptions?: EventOptions, - ) => void; - identify: (identify: IdentifyInstance, eventOptions?: EventOptions) => void; - revenue: (revenue: RevenueInstance, eventOptions?: EventOptions) => void; + ) => AmplitudeReturn; + identify: ( + identify: IdentifyInstance, + eventOptions?: EventOptions, + ) => AmplitudeReturn; + revenue: ( + revenue: RevenueInstance, + eventOptions?: EventOptions, + ) => AmplitudeReturn; setGroup: ( groupType: string, groupName: string | string[], eventOptions?: EventOptions, - ) => void; + ) => AmplitudeReturn; groupIdentify: ( groupType: string, groupName: string, identify: IdentifyInstance, eventOptions?: EventOptions, - ) => void; + ) => AmplitudeReturn; setOptOut: (optOut: boolean) => void; flush: () => { promise: Promise }; Identify: new () => IdentifyInstance; @@ -163,6 +177,7 @@ export type Config = { export type InitFn = DestinationServer.InitFn; export type PushFn = DestinationServer.PushFn; +export type PushBatchFn = CoreDestination.PushBatchFn; export type PartialConfig = DestinationServer.PartialConfig; diff --git a/packages/server/destinations/clickhouse/src/__tests__/classify.test.ts b/packages/server/destinations/clickhouse/src/__tests__/classify.test.ts index 1eed1f0c2..0da33c53b 100644 --- a/packages/server/destinations/clickhouse/src/__tests__/classify.test.ts +++ b/packages/server/destinations/clickhouse/src/__tests__/classify.test.ts @@ -1,19 +1,11 @@ -import type { MockLogger } from '@walkeros/core'; -import { createMockLogger } from '@walkeros/core'; import { ClickHouseError } from '../__mocks__/@clickhouse/client'; -import { classify, errorCode } from '../classify'; +import { adviceFor, classify, errorCode } from '../classify'; /** A rejection as the server produces it: the class, carrying a string code. */ function serverError(code: string, message = 'clickhouse insert failed') { return new ClickHouseError({ message, code }); } -let logger: MockLogger; - -beforeEach(() => { - logger = createMockLogger(); -}); - describe('classify', () => { // Every condition that ends on its own within seconds. TIMEOUT_EXCEEDED and // QUERY_WAS_CANCELLED are the two that may have landed anyway, and the @@ -31,7 +23,7 @@ describe('classify', () => { ['999', 'KEEPER_EXCEPTION'], ['1017', 'ASYNC_INSERT_FLUSH_TIMEOUT'], ])('retries %s %s', (code) => { - expect(classify(serverError(code), logger)).toBe('retryable'); + expect(classify(serverError(code))).toBe('retryable'); }); // Everything a second attempt cannot change: the batch is wrong, the table is @@ -54,20 +46,18 @@ describe('classify', () => { ['516', 'AUTHENTICATION_FAILED'], ['565', 'TOO_MANY_PARTITIONS_FOR_SINGLE_INSERT_BLOCK'], ])('gives up on %s %s', (code) => { - expect(classify(serverError(code), logger)).toBe('terminal'); + expect(classify(serverError(code))).toBe('terminal'); }); // Nothing answered, so nothing was rejected. The token makes the retry free. it('retries an error carrying no clickhouse code', () => { - expect(classify(new Error('unparsed server response'), logger)).toBe( - 'retryable', - ); + expect(classify(new Error('unparsed server response'))).toBe('retryable'); }); it.each([undefined, null, 'unparsed server response', 500])( 'retries a %s rejection', (reason) => { - expect(classify(reason, logger)).toBe('retryable'); + expect(classify(reason)).toBe('retryable'); }, ); @@ -87,7 +77,7 @@ describe('classify', () => { ])('retries a socket failure carrying errno %s', (errno) => { const error = Object.assign(new Error('socket hang up'), { code: errno }); - expect(classify(error, logger)).toBe('retryable'); + expect(classify(error)).toBe('retryable'); }); // Retrying a batch the server did reject costs one attempt. Dropping one that @@ -98,7 +88,7 @@ describe('classify', () => { // would answer 'terminal' here, so reading 'retryable' is what proves a // number is not read as a ClickHouse code at all. it('treats a numeric code as no code at all', () => { - expect(classify({ code: 16 }, logger)).toBe('retryable'); + expect(classify({ code: 16 })).toBe('retryable'); }); // ClickHouse maps a few dozen codes to their own status and sends everything @@ -107,58 +97,39 @@ describe('classify', () => { // 4xx: a classifier that read the status would call it terminal, so this // fails if one is ever introduced. it('reads no classification out of a bare http status', () => { - expect(classify({ status: 400 }, logger)).toBe('retryable'); + expect(classify({ status: 400 })).toBe('retryable'); }); it('ignores a retryable-looking status beside a terminal code', () => { const error = Object.assign(serverError('16'), { status: 503 }); - expect(classify(error, logger)).toBe('terminal'); + expect(classify(error)).toBe('terminal'); }); }); +// The advice travels in the failure's message (deliver.ts); the collector logs +// it once per cause, and the package writes no line of its own. describe('operator guidance', () => { it('names the batch size as the fix for too many parts', () => { - expect(classify(serverError('252'), logger)).toBe('terminal'); - - expect(logger.error).toHaveBeenCalledTimes(1); - expect(logger.error).toHaveBeenCalledWith( - expect.stringContaining('config.batch.size'), - { code: '252' }, - ); - expect(logger.error).toHaveBeenCalledWith( - expect.stringContaining('Raising parts_to_throw_insert is not the fix'), - { code: '252' }, + expect(adviceFor('252')).toContain('config.batch.size'); + expect(adviceFor('252')).toContain( + 'Raising parts_to_throw_insert is not the fix', ); }); it('points an unknown table at the table DDL in the docs', () => { - expect(classify(serverError('60'), logger)).toBe('terminal'); - - expect(logger.error).toHaveBeenCalledWith( - expect.stringContaining('table DDL'), - { code: '60' }, + expect(adviceFor('60')).toContain('table DDL'); + expect(adviceFor('60')).toContain( + 'https://www.walkeros.io/docs/destinations/server/clickhouse#create-the-table', ); - expect(logger.error).toHaveBeenCalledWith( - expect.stringContaining( - 'https://www.walkeros.io/docs/destinations/server/clickhouse#create-the-table', - ), - { code: '60' }, - ); - }); - - it('stays quiet on a retryable failure', () => { - classify(serverError('159'), logger); - - expect(logger.error).not.toHaveBeenCalled(); }); - // The collector logs every failed push and holds the error in its dead letter - // queue, so a second line adds nothing where there is no advice to give. - it('stays quiet on a terminal code it has no advice for', () => { - classify(serverError('53'), logger); - - expect(logger.error).not.toHaveBeenCalled(); + it.each([ + ['a retryable code', '159'], + ['a terminal code that explains itself', '53'], + ['no code', undefined], + ])('gives no advice for %s', (_label, code) => { + expect(adviceFor(code)).toBeUndefined(); }); }); diff --git a/packages/server/destinations/clickhouse/src/__tests__/config.test.ts b/packages/server/destinations/clickhouse/src/__tests__/config.test.ts index 74dd4654e..249571ec6 100644 --- a/packages/server/destinations/clickhouse/src/__tests__/config.test.ts +++ b/packages/server/destinations/clickhouse/src/__tests__/config.test.ts @@ -332,7 +332,7 @@ describe('resolveSettings', () => { it('applies the documented defaults', () => { const config: PartialConfig = { settings: { url, client } }; - expect(resolveSettings(config, logger)).toEqual({ + expect(resolveSettings(config)).toEqual({ url, database: 'default', table: 'events', @@ -353,7 +353,7 @@ describe('resolveSettings', () => { }, }; - expect(resolveSettings(config, logger)).toMatchObject({ + expect(resolveSettings(config)).toMatchObject({ database: 'analytics', table: 'hits', maxRetries: 3, @@ -365,7 +365,7 @@ describe('resolveSettings', () => { settings: { url, maxRetries: 0, client }, }; - expect(resolveSettings(config, logger).maxRetries).toBe(0); + expect(resolveSettings(config).maxRetries).toBe(0); }); it('merges the flow settings under the insert defaults', () => { @@ -373,7 +373,7 @@ describe('resolveSettings', () => { settings: { url, clickhouseSettings: { async_insert: 1 }, client }, }; - expect(resolveSettings(config, logger).clickhouseSettings).toEqual({ + expect(resolveSettings(config).clickhouseSettings).toEqual({ async_insert: 1, input_format_skip_unknown_fields: 0, input_format_null_as_default: 0, @@ -381,20 +381,20 @@ describe('resolveSettings', () => { }); it('refuses a config without settings', () => { - expect(() => resolveSettings({}, logger)).toThrow( - 'Config settings missing, init() not run', + expect(() => resolveSettings({})).toThrow( + 'ClickHouse client missing, init() not run', ); }); it('refuses settings without a client', () => { - expect(() => resolveSettings({ settings: { url } }, logger)).toThrow( + expect(() => resolveSettings({ settings: { url } })).toThrow( 'ClickHouse client missing, init() not run', ); }); it('refuses settings without a url', () => { - expect(() => - resolveSettings({ settings: { url: '', client } }, logger), - ).toThrow('Config settings url missing'); + expect(() => resolveSettings({ settings: { url: '', client } })).toThrow( + 'Config settings url missing', + ); }); }); diff --git a/packages/server/destinations/clickhouse/src/__tests__/conformance.test.ts b/packages/server/destinations/clickhouse/src/__tests__/conformance.test.ts new file mode 100644 index 000000000..722bfa255 --- /dev/null +++ b/packages/server/destinations/clickhouse/src/__tests__/conformance.test.ts @@ -0,0 +1,22 @@ +import path from 'path'; +import { conformance } from '@walkeros/collector/dev'; +import { destinationClickHouse } from '..'; +import { examples, schemas } from '../dev'; + +describe('clickhouse conformance', () => { + beforeEach(() => { + jest.useRealTimers(); + }); + + it('conforms', async () => { + await conformance('destination', destinationClickHouse, { + profile: 'socket', + examples: examples.step, + schema: schemas.settings, + // No retry: the kit's failure answers are judged on the first attempt. + settings: { url: 'https://clickhouse.example.com:8443', maxRetries: 0 }, + env: { push: examples.env.push, simulation: examples.env.simulation }, + root: path.join(__dirname, '..', '..'), + }); + }); +}); diff --git a/packages/server/destinations/clickhouse/src/__tests__/init.test.ts b/packages/server/destinations/clickhouse/src/__tests__/init.test.ts index dfaebc879..f3af605ef 100644 --- a/packages/server/destinations/clickhouse/src/__tests__/init.test.ts +++ b/packages/server/destinations/clickhouse/src/__tests__/init.test.ts @@ -93,9 +93,11 @@ describe('init', () => { it('refuses a config without settings', async () => { const { env } = createStub(); - await expect(destination.init(initContext({}, env))).rejects.toThrow( - 'Config settings missing', - ); + await expect(destination.init(initContext({}, env))).rejects.toMatchObject({ + name: 'StepError', + code: 'config', + message: 'Config settings url missing', + }); }); it('queries nothing', async () => { diff --git a/packages/server/destinations/clickhouse/src/__tests__/insert.test.ts b/packages/server/destinations/clickhouse/src/__tests__/insert.test.ts index 1676a2b5b..ead2d54b0 100644 --- a/packages/server/destinations/clickhouse/src/__tests__/insert.test.ts +++ b/packages/server/destinations/clickhouse/src/__tests__/insert.test.ts @@ -386,17 +386,15 @@ describe('retry', () => { // The advice is what an operator acts on, so it has to reach them even where // there was never a retry to skip. - it('gives the operator guidance with no retry left to skip', async () => { + it('gives up on a terminal code without a retry or a line of its own', async () => { __failInsert(1, { code: '252' }); await settle( - insertRows(entriesFor('evt-a'), settingsFor({ maxRetries: 0 }), logger), + insertRows(entriesFor('evt-a'), settingsFor({ maxRetries: 2 }), logger), ); - expect(logger.error).toHaveBeenCalledWith( - expect.stringContaining('config.batch.size'), - { code: '252' }, - ); + expect(logger.warn).not.toHaveBeenCalled(); + expect(logger.error).not.toHaveBeenCalled(); }); }); diff --git a/packages/server/destinations/clickhouse/src/__tests__/push.test.ts b/packages/server/destinations/clickhouse/src/__tests__/push.test.ts index b974ee29b..b488e8559 100644 --- a/packages/server/destinations/clickhouse/src/__tests__/push.test.ts +++ b/packages/server/destinations/clickhouse/src/__tests__/push.test.ts @@ -46,7 +46,7 @@ describe('push', () => { // and the defaults unapplied, so a push that read them straight off the config // would reach the insert with an undefined client. it.each([ - ['init never ran', {}, 'Config settings missing'], + ['init never ran', {}, 'ClickHouse client missing'], ['init left no client', { settings: { url } }, 'ClickHouse client missing'], ])('refuses to insert when %s', async (_case, config, message) => { await expect( @@ -127,48 +127,62 @@ describe('deduplication token', () => { describe('failure', () => { // The collector reads a throw as the delivery failing: it dead-letters the - // event and records one transport failure against the breaker. A push that - // swallowed the rejection would report a lost event as delivered. - it('throws the failure the insert ended on, code included', async () => { + // event, logs one line and records one transport failure against the + // breaker. A push that swallowed the rejection would report a lost event as + // delivered. + it('throws a StepError naming the code the insert ended on', async () => { const config = await bootstrap(); __failInsert(1, { code: '16', message: 'no such column' }); await expect( destination.push(getEvent('page view'), pushContext(config)), - ).rejects.toMatchObject({ code: '16', message: 'no such column' }); + ).rejects.toMatchObject({ + name: 'StepError', + code: 'transport', + retryable: false, + message: 'ClickHouse insert failed (code 16)', + }); }); - it('logs the failure at error with the code and the row count', async () => { + // The server's own text can quote the rows it could not parse, so it stays + // the cause and never reaches the message or a log line. + it('writes no failure line of its own and keeps the server text out', async () => { const config = await bootstrap(); - __failInsert(1, { code: '16' }); + __failInsert(1, { code: '27', message: 'Cannot parse input: alice@x' }); await expect( destination.push(getEvent('page view'), pushContext(config)), - ).rejects.toBeDefined(); - - expect(logger.error).toHaveBeenCalledWith( - expect.any(String), - expect.objectContaining({ code: '16', rows: 1 }), - ); + ).rejects.toMatchObject({ message: 'ClickHouse insert failed (code 27)' }); + expect(logger.error).not.toHaveBeenCalled(); }); // A transport failure carries a Node errno, which is not a ClickHouse code // and is deliberately not read as one, so the code is absent for exactly the - // failure class the retry exists for. The message is what is left to grep. - it('names a codeless transport failure in the error line', async () => { + // failure class the retry exists for. The errno is what is left to grep. + it('names the errno of a codeless transport failure', async () => { // A transport failure is retryable, so the retries are turned off here to - // reach the error line without waiting out a delay. + // reach the failure without waiting out a delay. const config = await bootstrap({ settings: { url, maxRetries: 0 } }); __failInsert(1, { code: 'ECONNRESET', message: 'socket hang up' }); await expect( destination.push(getEvent('page view'), pushContext(config)), - ).rejects.toBeDefined(); + ).rejects.toMatchObject({ + message: 'ClickHouse insert failed (ECONNRESET)', + retryable: true, + }); + }); - expect(logger.error).toHaveBeenCalledWith( - expect.any(String), - expect.objectContaining({ code: undefined, message: 'socket hang up' }), - ); + it('carries the advice for a terminal code that needs one', async () => { + const config = await bootstrap(); + __failInsert(1, { code: '60' }); + + await expect( + destination.push(getEvent('page view'), pushContext(config)), + ).rejects.toMatchObject({ + message: expect.stringContaining('table DDL'), + retryable: false, + }); }); // One line per delivery at debug, and nothing at a level an operator watches: diff --git a/packages/server/destinations/clickhouse/src/__tests__/pushBatch.test.ts b/packages/server/destinations/clickhouse/src/__tests__/pushBatch.test.ts index 42c949c7d..a09264cf2 100644 --- a/packages/server/destinations/clickhouse/src/__tests__/pushBatch.test.ts +++ b/packages/server/destinations/clickhouse/src/__tests__/pushBatch.test.ts @@ -95,7 +95,7 @@ describe('pushBatch', () => { }); it.each([ - ['init never ran', {}, 'Config settings missing'], + ['init never ran', {}, 'ClickHouse client missing'], ['init left no client', { settings: { url } }, 'ClickHouse client missing'], ])('refuses to insert when %s', async (_case, config, message) => { await expect( @@ -140,7 +140,11 @@ describe('whole-batch failure', () => { await expect( destination.pushBatch(batchOf(...entries), batchContext(config)), - ).rejects.toMatchObject({ code: '16', message: 'no such column' }); + ).rejects.toMatchObject({ + name: 'StepError', + code: 'transport', + message: 'ClickHouse insert failed (code 16)', + }); }); it('resolves undefined when the insert lands', async () => { @@ -151,7 +155,7 @@ describe('whole-batch failure', () => { ).resolves.toBeUndefined(); }); - it('logs the failure at error with the code and the row count', async () => { + it('writes no failure line of its own: the collector logs it', async () => { const config = await bootstrap(); __failInsert(1, { code: '16' }); @@ -159,10 +163,7 @@ describe('whole-batch failure', () => { destination.pushBatch(batchOf(...entries), batchContext(config)), ).rejects.toBeDefined(); - expect(logger.error).toHaveBeenCalledWith( - expect.any(String), - expect.objectContaining({ code: '16', rows: entries.length }), - ); + expect(logger.error).not.toHaveBeenCalled(); }); it('stays at debug when the insert lands', async () => { diff --git a/packages/server/destinations/clickhouse/src/classify.ts b/packages/server/destinations/clickhouse/src/classify.ts index eef348b43..8637b3aa5 100644 --- a/packages/server/destinations/clickhouse/src/classify.ts +++ b/packages/server/destinations/clickhouse/src/classify.ts @@ -1,5 +1,3 @@ -import type { Logger } from '@walkeros/core'; - /** Whether another attempt at the same insert can plausibly succeed. */ export type Classification = 'retryable' | 'terminal'; @@ -36,9 +34,9 @@ const RETRYABLE: ReadonlySet = new Set([ * up with. Retrying inside a ten second window cannot help a merge backlog and * adds parts to it, and the setting that names the limit is not the lever. * - * Everything else terminal explains itself in the server's own message, and the - * collector already logs every failed push, so a second line would only repeat - * it. + * Everything else terminal is named by its code. The advice travels in the + * failure's message, which the collector logs once per cause; the package + * writes no failure line of its own. */ const ADVICE: Readonly> = { '252': @@ -78,8 +76,7 @@ export function errorCode(error: unknown): string | undefined { } /** - * Whether a failed insert is worth another attempt, and the one place an - * operator is told what to change when it is not. + * Whether a failed insert is worth another attempt. * * The code is the classification input, never the HTTP status. ClickHouse maps * a few dozen codes to a status of their own and sends everything else as a @@ -97,17 +94,28 @@ export function errorCode(error: unknown): string | undefined { * second, while a batch wrongly called terminal is lost for good, the collector * having no retry of its own and never draining its dead letter queue. */ -export function classify( - error: unknown, - logger: Logger.Instance, -): Classification { +export function classify(error: unknown): Classification { const code = errorCode(error); - if (code === undefined || RETRYABLE.has(code)) return 'retryable'; + return code === undefined || RETRYABLE.has(code) ? 'retryable' : 'terminal'; +} - const advice = ADVICE[code]; +/** What an operator has to change for a terminal code, when it is not obvious. */ +export function adviceFor(code: string | undefined): string | undefined { + return code === undefined ? undefined : ADVICE[code]; +} - if (advice) logger.error(advice, { code }); +/** + * The Node errno of a rejection that never reached the server + * (`ECONNRESET`, `ENOTFOUND`): the code a ClickHouse code is not. + */ +export function errnoOf(error: unknown): string | undefined { + if (typeof error !== 'object' || error === null) return undefined; + if (!('code' in error)) return undefined; + + const { code } = error; - return 'terminal'; + return typeof code === 'string' && /^E[A-Z0-9_]+$/.test(code) + ? code + : undefined; } diff --git a/packages/server/destinations/clickhouse/src/config.ts b/packages/server/destinations/clickhouse/src/config.ts index 5a7c32928..b34e1b0b1 100644 --- a/packages/server/destinations/clickhouse/src/config.ts +++ b/packages/server/destinations/clickhouse/src/config.ts @@ -3,6 +3,7 @@ import type { ClickHouseSettings, } from '@clickhouse/client'; import type { Logger } from '@walkeros/core'; +import { StepError } from '@walkeros/core'; import { createClient } from '@clickhouse/client'; import type { ClickHouseClientFactory, @@ -187,13 +188,11 @@ function endpointHost(url: string): string { /** The resolved settings before init attaches the live client. */ type ResolvedSettings = Omit; -function withDefaults( - settings: InitSettings, - logger: Logger.Instance, -): ResolvedSettings { - const { url, clickhouse } = settings; +function withDefaults(settings: InitSettings | undefined): ResolvedSettings { + const { url, clickhouse } = settings ?? {}; - if (!url) return logger.throw('Config settings url missing'); + if (!url || !settings) + throw new StepError('Config settings url missing', { code: 'config' }); return { url, @@ -260,11 +259,7 @@ export function getConfig( env: Env | undefined, logger: Logger.Instance, ): Config { - const { settings } = config; - - if (!settings) return logger.throw('Config settings missing'); - - const resolved = withDefaults(settings, logger); + const resolved = withDefaults(config.settings); const options = resolveClientOptions( resolved, config.credentials, @@ -296,17 +291,14 @@ export function getConfig( * so the insert path holds a live client and a complete configuration without * guarding field by field. */ -export function resolveSettings( - config: PartialConfig, - logger: Logger.Instance, -): Settings { +export function resolveSettings(config: PartialConfig): Settings { const { settings } = config; + const client = settings?.client; - if (!settings) return logger.throw('Config settings missing, init() not run'); - - const { client } = settings; - - if (!client) return logger.throw('ClickHouse client missing, init() not run'); + if (!settings || !client) + throw new StepError('ClickHouse client missing, init() not run', { + code: 'not-ready', + }); - return { ...withDefaults(settings, logger), client }; + return { ...withDefaults(settings), client }; } diff --git a/packages/server/destinations/clickhouse/src/deliver.ts b/packages/server/destinations/clickhouse/src/deliver.ts index 8be211413..ae58d5d47 100644 --- a/packages/server/destinations/clickhouse/src/deliver.ts +++ b/packages/server/destinations/clickhouse/src/deliver.ts @@ -1,6 +1,6 @@ import type { Destination, Logger } from '@walkeros/core'; -import { isObject } from '@walkeros/core'; -import { errorCode } from './classify'; +import { isObject, StepError } from '@walkeros/core'; +import { adviceFor, classify, errnoOf, errorCode } from './classify'; import { resolveSettings } from './config'; import { eventToRow } from './eventToRow'; import type { InsertEntry } from './insert'; @@ -43,36 +43,47 @@ function toInsertEntry({ event, data }: Entry): InsertEntry { * client optional and the defaults unapplied, so they have to be narrowed once * per delivery. * - * A failure is logged and rethrown as it arrived. The collector reads a throw - * as the whole delivery failing: every event goes to the dead letter queue and - * one transport failure is recorded against the destination's breaker. - * Swallowing the rejection would report lost events as delivered. + * A failure is thrown as the package's own StepError: the collector reads a + * throw as the whole delivery failing, every event goes to the dead letter + * queue, one line is logged and one transport failure is recorded against the + * destination's breaker. Swallowing the rejection would report lost events as + * delivered. */ export async function deliver( entries: readonly Entry[], config: PartialConfig, logger: Logger.Instance, ): Promise { - const settings = resolveSettings(config, logger); + const settings = resolveSettings(config); try { await insertRows(entries.map(toInsertEntry), settings, logger); } catch (error) { - // The message travels beside the code because the code is often absent: a - // transport failure carries a Node errno, which is not a ClickHouse code - // and is deliberately not read as one. `ECONNRESET` is the commonest - // retryable failure there is, and a line that named neither it nor the - // server's own text would leave an operator grepping for nothing. - logger.error('ClickHouse insert failed', { - code: errorCode(error), - message: error instanceof Error ? error.message : String(error), - rows: entries.length, - }); - - throw error; + throw insertError(error); } // A landed delivery stays at debug. Every event that arrives produces one of // these, so anything louder buries the line that matters. logger.debug('ClickHouse insert ok', { rows: entries.length }); } + +/** + * A failed insert as the package's StepError. The message names the ClickHouse + * code, or the Node errno of a rejection that never reached the server + * (`ECONNRESET` is the commonest retryable failure there is), plus the advice + * for the terminal codes that need one. Never the server's own text: it can + * quote the rows it could not parse. The original error stays the cause. + */ +function insertError(error: unknown): StepError { + const code = errorCode(error); + const reason = code ? `code ${code}` : errnoOf(error); + return new StepError( + adviceFor(code) ?? + `ClickHouse insert failed${reason ? ` (${reason})` : ''}`, + { + code: 'transport', + retryable: classify(error) === 'retryable', + cause: error, + }, + ); +} diff --git a/packages/server/destinations/clickhouse/src/examples/step.ts b/packages/server/destinations/clickhouse/src/examples/step.ts index 26248cc9a..e6b5863f9 100644 --- a/packages/server/destinations/clickhouse/src/examples/step.ts +++ b/packages/server/destinations/clickhouse/src/examples/step.ts @@ -188,3 +188,12 @@ export const purchase: Flow.StepExample = { mapping: undefined, out: expectedInsert(purchaseEvent), }; + +/** A silent rule inserts no row. */ +export const silentRule: Flow.StepExample = { + public: false, + description: 'A rule with silent: true inserts no row.', + in: getEvent('page view', { timestamp: 1700000000124 }), + mapping: { silent: true }, + out: [], +}; diff --git a/packages/server/destinations/clickhouse/src/insert.ts b/packages/server/destinations/clickhouse/src/insert.ts index 7148dfb85..88444d616 100644 --- a/packages/server/destinations/clickhouse/src/insert.ts +++ b/packages/server/destinations/clickhouse/src/insert.ts @@ -135,9 +135,7 @@ export async function insertRows( return; } catch (error) { - // Classified before the budget is read, so the operator hears about a - // terminal failure even where there was never a retry to skip. - const retryable = classify(error, logger) === 'retryable'; + const retryable = classify(error) === 'retryable'; if (!retryable || attempt >= retries) throw error; diff --git a/packages/server/destinations/clickhouse/src/push.ts b/packages/server/destinations/clickhouse/src/push.ts index 87881658a..79a21c5d0 100644 --- a/packages/server/destinations/clickhouse/src/push.ts +++ b/packages/server/destinations/clickhouse/src/push.ts @@ -8,6 +8,7 @@ import type { PushFn } from './types'; * a flushed batch, so a mapped event takes the same path whether batching is on * or off. */ -export const push: PushFn = async (event, { config, data, logger }) => { - await deliver([{ event, data }], config, logger); +export const push: PushFn = async (event, { config, data, rule, logger }) => { + // A silent rule has no insert of its own: nothing to send. + if (!rule?.silent) await deliver([{ event, data }], config, logger); }; diff --git a/packages/server/destinations/clickhouse/src/pushBatch.ts b/packages/server/destinations/clickhouse/src/pushBatch.ts index 718f9ef68..3276b112a 100644 --- a/packages/server/destinations/clickhouse/src/pushBatch.ts +++ b/packages/server/destinations/clickhouse/src/pushBatch.ts @@ -15,9 +15,9 @@ import type { PushBatchFn } from './types'; * would pass the breaker unseen. This destination never returns one. */ export const pushBatch: PushBatchFn = async (batch, { config, logger }) => { - // Nothing to send and nothing to report. An insert of no rows would spend a - // round trip, and a deduplication token, on an empty block. - if (batch.entries.length === 0) return; - - await deliver(batch.entries, config, logger); + // A silent rule has no insert of its own. With nothing left to send there is + // nothing to report: an insert of no rows would spend a round trip, and a + // deduplication token, on an empty block. + const entries = batch.entries.filter((entry) => !entry.rule?.silent); + if (entries.length) await deliver(entries, config, logger); }; diff --git a/packages/server/destinations/clickhouse/src/schemas/settings.ts b/packages/server/destinations/clickhouse/src/schemas/settings.ts index 0d439c054..a2f20d429 100644 --- a/packages/server/destinations/clickhouse/src/schemas/settings.ts +++ b/packages/server/destinations/clickhouse/src/schemas/settings.ts @@ -1,42 +1,44 @@ import { z } from '@walkeros/core/dev'; -export const SettingsSchema = z.object({ - url: z - .string() - .min(1) - .describe( - 'ClickHouse HTTP endpoint (like https://host.clickhouse.cloud:8443)', - ), - database: z - .string() - .min(1) - .default('default') - .describe('Database holding the events table (like analytics)'), - table: z - .string() - .min(1) - .default('events') - .describe('Table every event is inserted into (like events)'), - maxRetries: z - .number() - .int() - .min(0) - .default(1) - .describe( - 'Retries added on top of the first attempt, so total attempts = 1 + maxRetries (like 1 for two attempts)', - ), - clickhouse: z - .record(z.string(), z.unknown()) - .optional() - .describe( - 'Raw @clickhouse/client options merged into the created client (like { request_timeout: 60000 })', - ), - clickhouseSettings: z - .record(z.string(), z.unknown()) - .optional() - .describe( - 'ClickHouse settings applied to every insert (like { async_insert: 1 })', - ), -}); +export const SettingsSchema = z + .object({ + url: z + .string() + .min(1) + .describe( + 'ClickHouse HTTP endpoint (like https://host.clickhouse.cloud:8443)', + ), + database: z + .string() + .min(1) + .default('default') + .describe('Database holding the events table (like analytics)'), + table: z + .string() + .min(1) + .default('events') + .describe('Table every event is inserted into (like events)'), + maxRetries: z + .number() + .int() + .min(0) + .default(1) + .describe( + 'Retries added on top of the first attempt, so total attempts = 1 + maxRetries (like 1 for two attempts)', + ), + clickhouse: z + .record(z.string(), z.unknown()) + .optional() + .describe( + 'Raw @clickhouse/client options merged into the created client (like { request_timeout: 60000 })', + ), + clickhouseSettings: z + .record(z.string(), z.unknown()) + .optional() + .describe( + 'ClickHouse settings applied to every insert (like { async_insert: 1 })', + ), + }) + .strict(); export type Settings = z.infer; diff --git a/packages/server/destinations/file/src/__tests__/conformance.test.ts b/packages/server/destinations/file/src/__tests__/conformance.test.ts new file mode 100644 index 000000000..fd75e9e17 --- /dev/null +++ b/packages/server/destinations/file/src/__tests__/conformance.test.ts @@ -0,0 +1,31 @@ +import { mkdtempSync, rmSync } from 'fs'; +import { tmpdir } from 'os'; +import path from 'path'; +import { conformance } from '@walkeros/collector/dev'; +import { destinationFile } from '..'; +import { examples, schemas } from '../dev'; + +describe('file conformance', () => { + // The runs without env write to the real disk: keep them in a temp dir. + let dir: string; + + beforeEach(() => { + jest.useRealTimers(); + dir = mkdtempSync(path.join(tmpdir(), 'walkeros-file-kit-')); + }); + + afterEach(() => { + rmSync(dir, { recursive: true, force: true }); + }); + + it('conforms', async () => { + await conformance('destination', destinationFile, { + profile: 'socket', + examples: examples.step, + schema: schemas.settings, + settings: { filename: path.join(dir, 'events.jsonl') }, + env: { push: examples.env.push, simulation: examples.env.simulation }, + root: path.join(__dirname, '..', '..'), + }); + }); +}); diff --git a/packages/server/destinations/file/src/__tests__/failure.test.ts b/packages/server/destinations/file/src/__tests__/failure.test.ts new file mode 100644 index 000000000..d060ca88a --- /dev/null +++ b/packages/server/destinations/file/src/__tests__/failure.test.ts @@ -0,0 +1,47 @@ +import { mkdtempSync, rmSync } from 'fs'; +import { tmpdir } from 'os'; +import path from 'path'; +import { startFlow } from '@walkeros/collector'; +import { getEvent } from '@walkeros/core'; +import { destinationFile } from '..'; + +describe('file destination on a failing disk', () => { + let dir: string; + + beforeEach(() => { + dir = mkdtempSync(path.join(tmpdir(), 'walkeros-file-fail-')); + }); + + afterEach(() => { + rmSync(dir, { recursive: true, force: true }); + }); + + it('fails the push when the file cannot be written, without crashing', async () => { + // A directory as the filename: opening it for append fails (EISDIR). + const { collector } = await startFlow({ + logger: { level: 'ERROR', handler: () => undefined }, + destinations: { + file: { + code: destinationFile, + config: { settings: { filename: dir } }, + }, + }, + }); + + const result = await collector.push(getEvent('page view')); + + expect(result.ok).toBe(false); + expect(result.failed?.file?.reason).toMatchObject({ + code: 'transport', + message: expect.stringContaining('file destination: write failed'), + }); + expect(collector.status.destinations.file.failed).toBe(1); + expect(collector.destinations.file.dlq).toHaveLength(1); + // The stream's error event was heard (a crash otherwise). + expect( + collector.status.connectionErrors['destination.file'], + ).toBeGreaterThanOrEqual(1); + + await collector.command('shutdown'); + }); +}); diff --git a/packages/server/destinations/file/src/__tests__/stepExamples.test.ts b/packages/server/destinations/file/src/__tests__/stepExamples.test.ts index 3cbf87188..74c21c982 100644 --- a/packages/server/destinations/file/src/__tests__/stepExamples.test.ts +++ b/packages/server/destinations/file/src/__tests__/stepExamples.test.ts @@ -35,8 +35,9 @@ function makeSpyEnv(): SpyEnv { }; if (!existing) state.captured.set(path, file); const stream: FileWriteStream = { - write(chunk) { + write(chunk, callback) { file.lines.push(chunk); + callback(); return true; }, end() { @@ -62,7 +63,12 @@ function flatten(out: unknown): CallRecord[] { } describe('@walkeros/server-destination-file step examples', () => { - it.each(Object.entries(examples.step))('%s', async (_name, rawExample) => { + // Non-public examples (silent) run in the conformance test. + const published = Object.entries(examples.step).filter( + ([, example]) => example.public !== false, + ); + + it.each(published)('%s', async (_name, rawExample) => { const example = rawExample as FileStepExample; const inEvent = example.in as WalkerOS.Event; diff --git a/packages/server/destinations/file/src/examples/env.ts b/packages/server/destinations/file/src/examples/env.ts index b3abb143d..8ea93afac 100644 --- a/packages/server/destinations/file/src/examples/env.ts +++ b/packages/server/destinations/file/src/examples/env.ts @@ -40,8 +40,9 @@ function makeSpyEnv(): SpyEnv { }; if (!existing) state.captured.set(path, file); const stream: FileWriteStream = { - write(chunk) { + write(chunk, callback) { file.lines.push(chunk); + callback(); return true; }, end() { diff --git a/packages/server/destinations/file/src/examples/step.ts b/packages/server/destinations/file/src/examples/step.ts index d564735a5..6031d0c33 100644 --- a/packages/server/destinations/file/src/examples/step.ts +++ b/packages/server/destinations/file/src/examples/step.ts @@ -157,3 +157,13 @@ export const csvObjectCell: FileStepExample = { ], ], }; + +/** A silent rule writes no line. */ +export const silentRule: FileStepExample = { + public: false, + description: 'A rule with silent: true writes no line.', + in: getEvent('page view', { timestamp: 1700000000000 }), + settings: { filename: 'events.jsonl' }, + mapping: { silent: true }, + out: [], +}; diff --git a/packages/server/destinations/file/src/helpers.ts b/packages/server/destinations/file/src/helpers.ts index 461cc1b0b..a5093ec1b 100644 --- a/packages/server/destinations/file/src/helpers.ts +++ b/packages/server/destinations/file/src/helpers.ts @@ -1,10 +1,7 @@ -import { - createWriteStream as nodeCreateWriteStream, - type WriteStream, -} from 'node:fs'; +import { createWriteStream as nodeCreateWriteStream } from 'node:fs'; import { mkdir as nodeMkdir } from 'node:fs/promises'; import { dirname } from 'node:path'; -import { getByPath, isObject } from '@walkeros/core'; +import { getByPath, isObject, isThenable, StepError } from '@walkeros/core'; import type { WalkerOS } from '@walkeros/core'; import type { Env, FileWriteStream, Format } from './types'; @@ -16,9 +13,30 @@ import type { Env, FileWriteStream, Format } from './types'; export interface State { handles: Map; fs: NonNullable; + /** Receives a stream's `error` event (a failed open or write). */ + onError?: (error: unknown) => void; } -export function createState(env: Env | undefined): State { +/** The code of a failed file operation (`EACCES`, `ENOSPC`), duck-typed. */ +function errorCode(error: unknown): string | undefined { + if (typeof error !== 'object' || error === null || !('code' in error)) + return undefined; + return typeof error.code === 'string' ? error.code : undefined; +} + +/** A failed file operation as a StepError; the path stays in the cause. */ +export function fileError(operation: string, error: unknown): StepError { + const code = errorCode(error); + return new StepError( + `file destination: ${operation} failed${code ? ` (${code})` : ''}`, + { code: 'transport', cause: error }, + ); +} + +export function createState( + env: Env | undefined, + onError?: (error: unknown) => void, +): State { const fsOverride = env?.fs; const defaultFs: NonNullable = { createWriteStream: (path: string, options: { flags: string }) => @@ -30,6 +48,7 @@ export function createState(env: Env | undefined): State { return { handles: new Map(), fs: fsOverride ?? defaultFs, + onError, }; } @@ -47,11 +66,34 @@ export async function ensureHandle( await state.fs.mkdir(dirname(filename) || '.', { recursive: true }); const handle = state.fs.createWriteStream(filename, { flags: 'a', - }) as FileWriteStream | WriteStream; + }); + // An unheard `error` event would crash the process. A broken stream is + // dropped, so the next event opens the file again. + handle.on?.('error', (error) => { + if (state.handles.get(filename) === handle) state.handles.delete(filename); + state.onError?.(fileError('stream', error)); + }); state.handles.set(filename, handle); return handle; } +/** + * Write one line and wait until it is written; a failed write throws. A + * node stream answers through the callback; an injected writer (tests, + * simulation) may answer with a promise instead. + */ +export function writeLine( + handle: FileWriteStream, + line: string, +): Promise { + return new Promise((resolve, reject) => { + const answer: unknown = handle.write(line, (error) => + error ? reject(error) : resolve(), + ); + if (isThenable(answer)) answer.then(() => resolve(), reject); + }); +} + /** * Format an event into a single output line (with trailing newline). * One implementation, used by push() for all three formats. diff --git a/packages/server/destinations/file/src/index.ts b/packages/server/destinations/file/src/index.ts index cb4a3bb65..47d8f447a 100644 --- a/packages/server/destinations/file/src/index.ts +++ b/packages/server/destinations/file/src/index.ts @@ -1,44 +1,48 @@ -import { getMappingValue, isString, isObject } from '@walkeros/core'; -import type { Destination, InitFn, PushFn, Settings } from './types'; +import { getMappingValue, isString, isObject, StepError } from '@walkeros/core'; +import type { + Destination, + FileWriteStream, + InitFn, + PushFn, + Settings, +} from './types'; import { createState, ensureHandle, + fileError, getState, registerState, removeState, serialize, + writeLine, } from './helpers'; export * as DestinationFile from './types'; -const init: InitFn = async ({ id, config, env, logger }) => { - const partial = (config?.settings ?? {}) as Partial; - if (partial.filename === undefined || partial.filename === null) { - logger.throw( - "file destination: 'filename' is required (string or Mapping.Value)", - ); - return false; - } +const init: InitFn = async ({ id, config, env, reportError }) => { + const partial: Partial = config?.settings ?? {}; + if (partial.filename === undefined || partial.filename === null) + throw new StepError('Config settings filename missing', { + code: 'config', + }); if ( !isString(partial.filename) && !(isObject(partial.filename) || Array.isArray(partial.filename)) - ) { - logger.throw( - "file destination: 'filename' must be a string or a Mapping.Value", + ) + throw new StepError( + 'Config settings filename must be a string or a Mapping.Value', + { code: 'config' }, ); - return false; - } const format = partial.format ?? 'jsonl'; if ( (format === 'tsv' || format === 'csv') && (!partial.fields || partial.fields.length === 0) - ) { - logger.throw( - `file destination: format '${format}' requires non-empty 'fields'`, + ) + throw new StepError( + `Config settings format '${format}' requires non-empty fields`, + { code: 'config' }, ); - return false; - } const settings: Settings = { filename: partial.filename, @@ -46,19 +50,17 @@ const init: InitFn = async ({ id, config, env, logger }) => { fields: partial.fields, }; - const state = createState(env); + const state = createState(env, reportError); - // Eager open for static filename — fail-fast on bad path. + // Eager open for static filename: a bad path shows at startup. if (isString(settings.filename)) { try { await ensureHandle(state, settings.filename); } catch (err) { - logger.throw( - `file destination: failed to open '${settings.filename}': ${ - err instanceof Error ? err.message : String(err) - }`, - ); - return false; + throw new StepError(`file destination: failed to open the file`, { + code: 'config', + cause: err, + }); } } @@ -70,17 +72,18 @@ const init: InitFn = async ({ id, config, env, logger }) => { }; }; -const push: PushFn = async (event, { id, config, collector, logger }) => { - const settings = config.settings as Settings | undefined; - if (!settings) { - logger.warn('file destination: settings missing'); - return; - } +// A silent rule has no line of its own: nothing to write. +const push: PushFn = async (event, context) => { + if (!context.rule?.silent) await write(event, context); +}; + +const write: PushFn = async (event, { id, config, collector }) => { + const settings = config.settings; const state = getState(id); - if (!state) { - logger.warn('file destination: state missing, init may have failed'); - return; - } + if (!settings || !state) + throw new StepError('file destination: not initialized', { + code: 'not-ready', + }); let filename: string; if (isString(settings.filename)) { @@ -89,24 +92,27 @@ const push: PushFn = async (event, { id, config, collector, logger }) => { const resolved = await getMappingValue(event, settings.filename, { collector, }); - if (!isString(resolved) || resolved.length === 0) { - logger.warn( - 'file destination: dynamic filename resolved to empty or non-string, dropping event', + if (!isString(resolved) || resolved.length === 0) + throw new StepError( + 'file destination: the filename mapping resolved to no name', + { code: 'missing-input' }, ); - return; - } filename = resolved; } + let handle: FileWriteStream; try { - const handle = await ensureHandle(state, filename); - handle.write(serialize(event, settings.format ?? 'jsonl', settings.fields)); + handle = await ensureHandle(state, filename); } catch (err) { - logger.warn( - `file destination: write failed for ${filename}: ${ - err instanceof Error ? err.message : String(err) - }`, + throw fileError('open', err); + } + try { + await writeLine( + handle, + serialize(event, settings.format ?? 'jsonl', settings.fields), ); + } catch (err) { + throw fileError('write', err); } }; diff --git a/packages/server/destinations/file/src/schemas/settings.ts b/packages/server/destinations/file/src/schemas/settings.ts index f2d034e50..aab65a7d2 100644 --- a/packages/server/destinations/file/src/schemas/settings.ts +++ b/packages/server/destinations/file/src/schemas/settings.ts @@ -1,28 +1,30 @@ import { z } from '@walkeros/core/dev'; -export const SettingsSchema = z.object({ - filename: z - .union([ - z.string().describe('Static output filename.'), - z - .record(z.string(), z.unknown()) - .describe( - 'Mapping.Value resolved per event (e.g. { key: "data.tenant" } or { fn: "$code:..." }).', - ), - ]) - .describe( - 'Output filename. Static string or Mapping.Value (e.g. { fn: "$code:..." } for daily rotation, { key: "data.tenant" } for sharding).', - ), - format: z - .enum(['jsonl', 'tsv', 'csv']) - .optional() - .describe('Serialisation format. Defaults to jsonl.'), - fields: z - .array(z.string()) - .optional() - .describe( - 'Event paths used as columns for tsv/csv formats. Object values are JSON-stringified. Required when format is tsv or csv.', - ), -}); +export const SettingsSchema = z + .object({ + filename: z + .union([ + z.string().describe('Static output filename.'), + z + .record(z.string(), z.unknown()) + .describe( + 'Mapping.Value resolved per event (e.g. { key: "data.tenant" } or { fn: "$code:..." }).', + ), + ]) + .describe( + 'Output filename. Static string or Mapping.Value (e.g. { fn: "$code:..." } for daily rotation, { key: "data.tenant" } for sharding).', + ), + format: z + .enum(['jsonl', 'tsv', 'csv']) + .optional() + .describe('Serialisation format. Defaults to jsonl.'), + fields: z + .array(z.string()) + .optional() + .describe( + 'Event paths used as columns for tsv/csv formats. Object values are JSON-stringified. Required when format is tsv or csv.', + ), + }) + .strict(); export type Settings = z.infer; diff --git a/packages/server/destinations/file/src/types/index.ts b/packages/server/destinations/file/src/types/index.ts index a790b3ac4..4d90c1b74 100644 --- a/packages/server/destinations/file/src/types/index.ts +++ b/packages/server/destinations/file/src/types/index.ts @@ -3,7 +3,6 @@ import type { Mapping as CoreMapping, } from '@walkeros/core'; import type { DestinationServer } from '@walkeros/server-core'; -import type { WriteStream } from 'node:fs'; export type Format = 'jsonl' | 'tsv' | 'csv'; @@ -37,8 +36,18 @@ export interface Mapping {} * inject a fake implementation via env.fs. */ export interface FileWriteStream { - write: (chunk: string) => boolean; + /** + * Calls `callback` once the chunk is written, with the error if it failed + * (node streams answer `boolean`). An injected writer may answer with a + * promise instead. + */ + write: ( + chunk: string, + callback: (error?: Error | null) => void, + ) => boolean | PromiseLike; end: () => void; + /** A stream reports a failed open or write as an `error` event. */ + on?: (event: 'error', listener: (error: Error) => void) => unknown; } export interface Env extends DestinationServer.Env { @@ -50,7 +59,7 @@ export interface Env extends DestinationServer.Env { createWriteStream: ( path: string, options: { flags: string }, - ) => FileWriteStream | WriteStream; + ) => FileWriteStream; mkdir: (path: string, options: { recursive: boolean }) => Promise; }; } diff --git a/packages/server/destinations/gcp/src/__tests__/conformance.test.ts b/packages/server/destinations/gcp/src/__tests__/conformance.test.ts new file mode 100644 index 000000000..460312d74 --- /dev/null +++ b/packages/server/destinations/gcp/src/__tests__/conformance.test.ts @@ -0,0 +1,43 @@ +import path from 'path'; +import { conformance } from '@walkeros/collector/dev'; +import { destinationBigQuery, destinationPubSub } from '..'; +import { examples, pubsubExamples, schemas } from '../dev'; + +// The runs without env construct the SDK clients: keep them on the package's +// own SDK mocks, never a real Google endpoint. +jest.mock('@google-cloud/bigquery'); +jest.mock('@google-cloud/bigquery-storage'); +jest.mock('@google-cloud/pubsub'); + +const root = path.join(__dirname, '..', '..'); + +describe('gcp conformance', () => { + beforeEach(() => { + jest.useRealTimers(); + }); + + it('bigquery conforms', async () => { + await conformance('destination', destinationBigQuery, { + profile: 'socket', + examples: examples.step, + schema: schemas.settings, + settings: { projectId: 'test-project' }, + env: { push: examples.env.push, simulation: examples.env.simulation }, + root, + }); + }); + + it('pubsub conforms', async () => { + await conformance('destination', destinationPubSub, { + profile: 'socket', + examples: pubsubExamples.step, + schema: schemas.pubsubSettings, + settings: { projectId: 'test-project', topic: 'events' }, + env: { + push: pubsubExamples.env.push, + simulation: pubsubExamples.env.simulation, + }, + root, + }); + }); +}); diff --git a/packages/server/destinations/gcp/src/bigquery/__tests__/index.test.ts b/packages/server/destinations/gcp/src/bigquery/__tests__/index.test.ts index c982bce08..eb038a5d5 100644 --- a/packages/server/destinations/gcp/src/bigquery/__tests__/index.test.ts +++ b/packages/server/destinations/gcp/src/bigquery/__tests__/index.test.ts @@ -26,7 +26,7 @@ import { import type { MockLogger } from '@walkeros/core'; import * as examples from '../examples'; import { eventToRow } from '../eventToRow'; -import { openWriter } from '../writer'; +import { ensureWriter, openWriter } from '../writer'; jest.mock('@google-cloud/bigquery'); jest.mock('@google-cloud/bigquery-storage'); @@ -64,6 +64,19 @@ describe('Server Destination BigQuery', () => { }); } + /** Init, then the writer open the first push makes (init stays offline). */ + async function initOpen( + initSettings: InitSettings, + logger?: MockLogger, + timeout?: number, + reportError?: (err: unknown, event?: WalkerOS.Event) => void, + ) { + const config = await callInit(initSettings, logger, timeout, reportError); + if (config && config.settings) + await ensureWriter(config.settings, createMockLogger()); + return config; + } + beforeEach(() => { jest.clearAllMocks(); __resetMockCalls(); @@ -74,11 +87,14 @@ describe('Server Destination BigQuery', () => { testEnv = clone(env.push); }); - test('init opens the JSONWriter through the 5-step sequence', async () => { + test('init stays offline; the first push opens the JSONWriter through the 5-step sequence', async () => { const result = await callInit({ projectId, datasetId, tableId }); expect(result).toBeTruthy(); - if (!result) return; + if (!result || !result.settings) return; + expect(__getMockCalls()).toEqual([]); + + await ensureWriter(result.settings, createMockLogger()); const methods = __getMockCalls().map((c) => c.method); expect(methods).toEqual([ @@ -94,7 +110,7 @@ describe('Server Destination BigQuery', () => { }); test('init forwards settings.bigquery auth to the Storage Write client', async () => { - await callInit({ + await initOpen({ projectId, datasetId, tableId, @@ -124,6 +140,7 @@ describe('Server Destination BigQuery', () => { reportError: () => undefined, }); if (!config || !config.settings) throw new Error('init returned no config'); + await ensureWriter(config.settings, createMockLogger()); // Query client (BigQuery) carries the credentials on its constructor options. // The example mock records its ctor options on `this.options`. @@ -168,83 +185,53 @@ describe('Server Destination BigQuery', () => { ); }); - test('init logs error and rethrows on non-NotFound openWriter failure', async () => { - // Simulate the INVALID_ARGUMENT case (e.g., TYPE_UNSPECIFIED). Code 3 is - // gRPC INVALID_ARGUMENT, which is NOT in the isNotFound (5/404) set. + test('the first push fails with a transport StepError on a non-NotFound open failure', async () => { + // Code 3 is gRPC INVALID_ARGUMENT, which is NOT in the isNotFound (5/404) + // set. The SDK error stays the cause, so its routing code is kept. const underlyingError: Error & { code?: number } = Object.assign( new Error('TYPE_UNSPECIFIED: bad write stream type'), { code: 3 }, ); - __setNextOpenWriterError(underlyingError); - const logger = createMockLogger(); + const config = await callInit({ projectId, datasetId, tableId }, logger); + if (!config) throw new Error('init returned void'); + __setNextOpenWriterError(underlyingError); await expect( - callInit({ projectId, datasetId, tableId }, logger), - ).rejects.toThrow('TYPE_UNSPECIFIED: bad write stream type'); - - expect(logger.error).toHaveBeenCalledWith( - expect.stringContaining('BigQuery init failed'), - expect.objectContaining({ - error: 'TYPE_UNSPECIFIED: bad write stream type', - }), - ); - const errorCall = logger.error.mock.calls.find( - (call: unknown[]) => - typeof call[0] === 'string' && call[0].includes('BigQuery init failed'), - ); - expect(errorCall).toBeDefined(); - if (!errorCall) return; - const errorContext = errorCall[1]; - expect( - typeof errorContext === 'object' && - errorContext !== null && - 'error' in errorContext, - ).toBe(true); - if ( - typeof errorContext !== 'object' || - errorContext === null || - !('error' in errorContext) - ) - return; - expect(errorContext.error).toBe('TYPE_UNSPECIFIED: bad write stream type'); + destination.push( + event, + createMockContext({ config, env: testEnv, logger, id: 'test-bq' }), + ), + ).rejects.toMatchObject({ + name: 'StepError', + code: 'transport', + retryable: true, + message: 'BigQuery writer could not be opened', + cause: underlyingError, + }); + // The collector writes the one line. + expect(logger.error).not.toHaveBeenCalled(); }); - test('init logs and throws an informative, DLQ-routable error on a non-NotFound failure', async () => { - // The destination no longer scrubs error messages itself: secret redaction - // is standardized at the CLI logger handler (covers stderr + heartbeat ring) - // and the thrown error is scrubbed on output there. Here the destination's - // job is to log the failure and rethrow the RAW error so it stays - // informative and DLQ-routable, keeping its routing `code`. - const underlyingError: Error & { code?: number } = Object.assign( - new Error('streams/_default" contains illegal characters'), - { code: 3 }, // INVALID_ARGUMENT — not NotFound, so it hits the catch-all + test('the first push fails with the setup hint when the table is missing', async () => { + const config = await callInit({ projectId, datasetId, tableId }); + if (!config) throw new Error('init returned void'); + __setNextOpenWriterError( + Object.assign(new Error('Not found: Table'), { code: 5 }), ); - __setNextOpenWriterError(underlyingError); - const logger = createMockLogger(); - - let thrown: unknown; - try { - await callInit({ projectId, datasetId, tableId }, logger); - } catch (err) { - thrown = err; - } - - // The thrown error is the raw error: informative and DLQ-routable, with its - // routing code intact (read without a cast). - expect(thrown).toBe(underlyingError); - expect(thrown).toBeInstanceOf(Error); - if (!(thrown instanceof Error)) return; - expect(thrown.message).toContain('contains illegal characters'); - expect('code' in thrown).toBe(true); - if (!('code' in thrown)) return; - const withCode: { code?: unknown } = thrown; - expect(withCode.code).toBe(3); - - // Still informative for the operator. - const serialized = JSON.stringify(logger.error.mock.calls); - expect(serialized).toContain('BigQuery init failed'); + await expect( + destination.push( + event, + createMockContext({ config, env: testEnv, id: 'test-bq' }), + ), + ).rejects.toMatchObject({ + code: 'config', + retryable: false, + message: expect.stringContaining( + 'Run "walkeros setup destination.test-bq" to create them.', + ), + }); }); test('push appends one row through JSONWriter', async () => { @@ -351,7 +338,7 @@ describe('Server Destination BigQuery', () => { }); test('destroy closes writer and writeClient', async () => { - const result = await callInit({ projectId, datasetId, tableId }); + const result = await initOpen({ projectId, datasetId, tableId }); expect(result).toBeTruthy(); if (!result) return; __resetMockCalls(); @@ -486,33 +473,23 @@ describe('Server Destination BigQuery', () => { }), ); + // The row's own message can quote the row: only its code is named. expect(outcome).toEqual({ failed: [ { index: 0, error: expect.objectContaining({ - code: 3, - message: 'invalid row', + name: 'StepError', + code: 'transport', + retryable: false, + message: 'BigQuery row append failed (code 3)', }), }, ], }); - // INFO summary with ok/failed counts for operator visibility. - expect(logger.info).toHaveBeenCalledWith( - expect.any(String), - expect.objectContaining({ ok: 2, failed: 1 }), - ); - - // ERROR per row error. - expect(logger.error).toHaveBeenCalledWith( - expect.any(String), - expect.objectContaining({ - index: 0, - code: 3, - message: 'invalid row', - }), - ); + // The collector writes the failure lines. + expect(logger.error).not.toHaveBeenCalled(); }); test('rejects when appendRows throws', async () => { @@ -538,7 +515,11 @@ describe('Server Destination BigQuery', () => { id: 'test-bq', }), ), - ).rejects.toThrow('append boom'); + ).rejects.toMatchObject({ + name: 'StepError', + message: 'BigQuery batch append threw', + cause: expect.objectContaining({ message: 'append boom' }), + }); }); test('rejects when init() has not run (writer missing)', async () => { @@ -569,7 +550,10 @@ describe('Server Destination BigQuery', () => { id: 'test-bq', }), ), - ).rejects.toThrow('writer is missing'); + ).rejects.toMatchObject({ + code: 'not-ready', + message: 'BigQuery writer is not initialized', + }); }); }); @@ -581,7 +565,7 @@ describe('Server Destination BigQuery', () => { // healthy connection on expiry. The stream keeps the SDK's own default. test('init forwards config.timeout to the unary schema fetch but never to the appendRows stream', async () => { - await callInit({ projectId, datasetId, tableId }, undefined, 5000); + await initOpen({ projectId, datasetId, tableId }, undefined, 5000); const streamCall = __getMockCalls().find( (c) => c.method === 'createStreamConnection', @@ -595,7 +579,7 @@ describe('Server Destination BigQuery', () => { }); test('init applies the default unary deadline when config.timeout is unset', async () => { - await callInit({ projectId, datasetId, tableId }); + await initOpen({ projectId, datasetId, tableId }); const streamCall = __getMockCalls().find( (c) => c.method === 'createStreamConnection', @@ -613,7 +597,7 @@ describe('Server Destination BigQuery', () => { // sentinel. Resolution falls back to the default so the unary call // always gets a positive deadline, mirroring the collector's // resolveDestinationTimeout. - await callInit({ projectId, datasetId, tableId }, undefined, 0); + await initOpen({ projectId, datasetId, tableId }, undefined, 0); const streamCall = __getMockCalls().find( (c) => c.method === 'createStreamConnection', @@ -662,14 +646,16 @@ describe('Server Destination BigQuery', () => { id: 'test-bq', }), ), - ).rejects.toThrow('Deadline exceeded'); + ).rejects.toMatchObject({ + message: 'BigQuery row append threw', + cause: deadlineError, + }); }); - test('a getResult rejection surfaces the raw, informative, DLQ-routable error from push', async () => { - // Single-event path (batching off): a getResult() rejection must surface - // to the collector/DLQ as the RAW error, keeping its message and routing - // `code`. Secret redaction is standardized at the CLI logger handler, so - // the destination no longer scrubs here. + test('a getResult rejection fails push with its error as the cause, logged by the collector only', async () => { + // Single-event path (batching off): a getResult() rejection surfaces to + // the collector/DLQ as a StepError whose cause is the raw error, its + // routing `code` intact. The raw message stays out of the reported one. const logger = createMockLogger(); const { writer, writeClient } = await openWriter( { projectId, datasetId, tableId }, @@ -694,9 +680,8 @@ describe('Server Destination BigQuery', () => { ); __setNextGetResultReject(appendError); - let thrown: unknown; - try { - await destination.push( + await expect( + destination.push( event, createMockContext({ config, @@ -706,25 +691,14 @@ describe('Server Destination BigQuery', () => { logger, id: 'test-bq', }), - ); - } catch (err) { - thrown = err; - } - - // The raw error surfaces: an Error, informative, with the routing code - // intact (read without a cast). - expect(thrown).toBe(appendError); - expect(thrown).toBeInstanceOf(Error); - if (!(thrown instanceof Error)) return; - expect(thrown.message).toContain('contains illegal characters'); - expect('code' in thrown).toBe(true); - if (!('code' in thrown)) return; - const withCode: { code?: unknown } = thrown; - expect(withCode.code).toBe(3); - - // The destination still logs the failure for the operator. - const serialized = JSON.stringify(logger.error.mock.calls); - expect(serialized).toContain('BigQuery row append threw'); + ), + ).rejects.toMatchObject({ + name: 'StepError', + code: 'transport', + message: 'BigQuery row append threw', + cause: appendError, + }); + expect(logger.error).not.toHaveBeenCalled(); }); test('a deadline-exceeded getResult rejects pushBatch (whole batch DLQ)', async () => { @@ -768,14 +742,17 @@ describe('Server Destination BigQuery', () => { id: 'test-bq', }), ), - ).rejects.toThrow('Deadline exceeded'); + ).rejects.toMatchObject({ + message: 'BigQuery batch append threw', + cause: deadlineError, + }); }); }); describe("connection 'error' containment + self-heal", () => { test('a non-retryable stream error sets writerBroken, calls reportError (orphan), and DLQ-routes the next push', async () => { const reportError = jest.fn(); - const config = await callInit( + const config = await initOpen( { projectId, datasetId, tableId }, undefined, undefined, @@ -821,14 +798,17 @@ describe('Server Destination BigQuery', () => { id: 'test-bq', }), ), - ).rejects.toThrow('re-open failed'); + ).rejects.toMatchObject({ + message: 'BigQuery writer could not be opened', + cause: expect.objectContaining({ message: 're-open failed' }), + }); // Stayed broken after a failed re-open. expect(settings.writerBroken).toBe(true); }); test('the next push self-heals (re-opens) and delivers when writerBroken', async () => { - const config = await callInit({ projectId, datasetId, tableId }); + const config = await initOpen({ projectId, datasetId, tableId }); if (!config) throw new Error('init returned void'); const { settings } = config; if (!settings) throw new Error('settings missing after init'); @@ -880,6 +860,7 @@ describe('Server Destination BigQuery', () => { if (!config || !config.settings) throw new Error('init returned no config'); const { settings } = config; + await ensureWriter(settings, createMockLogger()); __getLastConnection().__emitConnectionError(new Error('stream gone')); expect(settings.writerBroken).toBe(true); @@ -910,7 +891,7 @@ describe('Server Destination BigQuery', () => { test('a broken writer DLQ-routes the whole batch when re-open fails (batch path)', async () => { if (!destination.pushBatch) throw new Error('pushBatch missing'); - const config = await callInit({ projectId, datasetId, tableId }); + const config = await initOpen({ projectId, datasetId, tableId }); if (!config) throw new Error('init returned void'); const { settings } = config; if (!settings) throw new Error('settings missing after init'); @@ -936,13 +917,16 @@ describe('Server Destination BigQuery', () => { id: 'test-bq', }), ), - ).rejects.toThrow('re-open failed (batch)'); + ).rejects.toMatchObject({ + message: 'BigQuery writer could not be opened', + cause: expect.objectContaining({ message: 're-open failed (batch)' }), + }); expect(settings.writerBroken).toBe(true); }); test('concurrent pushes on a broken writer trigger exactly ONE re-open (no orphaned connection or listener)', async () => { - const config = await callInit({ projectId, datasetId, tableId }); + const config = await initOpen({ projectId, datasetId, tableId }); if (!config) throw new Error('init returned void'); const { settings } = config; if (!settings) throw new Error('settings missing after init'); @@ -994,7 +978,7 @@ describe('Server Destination BigQuery', () => { }); test('a re-opened appendRows stream carries no gax deadline (push path)', async () => { - const config = await callInit( + const config = await initOpen( { projectId, datasetId, tableId }, undefined, 5000, @@ -1038,7 +1022,7 @@ describe('Server Destination BigQuery', () => { test('a re-opened appendRows stream carries no gax deadline (batch path)', async () => { if (!destination.pushBatch) throw new Error('pushBatch missing'); - const config = await callInit( + const config = await initOpen( { projectId, datasetId, tableId }, undefined, 5000, diff --git a/packages/server/destinations/gcp/src/bigquery/__tests__/stepExamples.test.ts b/packages/server/destinations/gcp/src/bigquery/__tests__/stepExamples.test.ts index 3bfdc3007..e334605b6 100644 --- a/packages/server/destinations/gcp/src/bigquery/__tests__/stepExamples.test.ts +++ b/packages/server/destinations/gcp/src/bigquery/__tests__/stepExamples.test.ts @@ -29,7 +29,12 @@ describe('Step Examples', () => { __resetMockCalls(); }); - it.each(Object.entries(examples.step))('%s', async (_name, example) => { + // Non-public examples (silent) run in the conformance test. + const published = Object.entries(examples.step).filter( + ([, example]) => example.public !== false, + ); + + it.each(published)('%s', async (_name, example) => { const destination: Destination = jest.requireActual('../').default; destination.config = {}; diff --git a/packages/server/destinations/gcp/src/bigquery/config.ts b/packages/server/destinations/gcp/src/bigquery/config.ts index 5560b586e..b962653da 100644 --- a/packages/server/destinations/gcp/src/bigquery/config.ts +++ b/packages/server/destinations/gcp/src/bigquery/config.ts @@ -1,6 +1,7 @@ import type { Config, Env, PartialConfig, Settings } from './types'; import type { BigQueryOptions } from '@google-cloud/bigquery'; import type { Logger } from '@walkeros/core'; +import { StepError } from '@walkeros/core'; import { BigQuery } from '@google-cloud/bigquery'; import { parseCredentials } from '../pubsub/config'; @@ -13,7 +14,10 @@ export function getConfig( const { projectId, bigquery } = settings; let { client, location, datasetId, tableId } = settings; - if (!projectId) logger.throw('Config settings projectId missing'); + if (!projectId) + throw new StepError('Config settings projectId missing', { + code: 'config', + }); location = location || 'EU'; datasetId = datasetId || 'walkerOS'; @@ -24,9 +28,9 @@ export function getConfig( // `config.credentials` (parse-if-string) merges into the client options. // `settings.bigquery` stays the raw passthrough escape hatch. - const parsed = parseCredentials(partialConfig.credentials, logger); - // parseCredentials returns the object form (or undefined); a string can only - // remain on the invalid-JSON path, which logger.throw already handles. + const parsed = parseCredentials(partialConfig.credentials); + // parseCredentials returns the object form (or undefined); an invalid JSON + // string throws a config StepError. const credentials = parsed !== undefined && typeof parsed !== 'string' ? parsed : undefined; if (credentials !== undefined) { diff --git a/packages/server/destinations/gcp/src/bigquery/examples/step.ts b/packages/server/destinations/gcp/src/bigquery/examples/step.ts index 5c297d13a..33843fe6a 100644 --- a/packages/server/destinations/gcp/src/bigquery/examples/step.ts +++ b/packages/server/destinations/gcp/src/bigquery/examples/step.ts @@ -53,3 +53,12 @@ export const purchase: Flow.StepExample = { mapping: undefined, out: [['appendRows', [expectedRow(purchaseEvent)]]], }; + +/** A silent rule appends no row. */ +export const silentRule: Flow.StepExample = { + public: false, + description: 'A rule with silent: true appends no row.', + in: getEvent('page view', { timestamp: 1700000000500 }), + mapping: { silent: true }, + out: [], +}; diff --git a/packages/server/destinations/gcp/src/bigquery/index.ts b/packages/server/destinations/gcp/src/bigquery/index.ts index a1b257efa..908bc01d8 100644 --- a/packages/server/destinations/gcp/src/bigquery/index.ts +++ b/packages/server/destinations/gcp/src/bigquery/index.ts @@ -1,4 +1,5 @@ -import type { Destination } from './types'; +import type { Destination, Settings } from './types'; +import { StepError } from '@walkeros/core'; import { getConfig } from './config'; import { push } from './push'; import { pushBatch } from './pushBatch'; @@ -62,6 +63,10 @@ export const destinationBigQuery: Destination = { // (projectId/datasetId/tableId/bigquery/timeout) are immutable post-init, so // a re-open targets the same table with the same auth and unary-call // deadline. + // + // The same hook opens the writer at the first push, so init stays + // offline: a missing dataset or table, or a failed connection, fails that + // push with a StepError the collector reports. settings.reopenWriter = () => openWriter( { @@ -75,59 +80,9 @@ export const destinationBigQuery: Destination = { env, }, logger, - ); - - // Open the long-lived JSONWriter on the _default stream. - // Hard-fail when the dataset/table is missing. - try { - const { writer, writeClient, connection, connectionErrorListener } = - await openWriter( - { - projectId: settings.projectId, - datasetId: settings.datasetId, - tableId: settings.tableId, - credentials: settings.credentials, - bigquery: settings.bigquery, - timeout, - onConnectionError, - env, - }, - logger, - ); - settings.writer = writer; - settings.writeClient = writeClient; - settings.connection = connection; - settings.connectionErrorListener = connectionErrorListener; - } catch (err) { - // Log the failure and rethrow the raw error. Secret redaction is - // standardized at the CLI logger handler, which scrubs every line before - // both stderr and the heartbeat ring, so the destination logs the message - // as-is. The raw error keeps its `code` naturally, so it stays - // NotFound-classifiable and DLQ-routable. - const message = err instanceof Error ? err.message : String(err); - if (isNotFound(err)) { - const target = `${config.settings.datasetId}.${config.settings.tableId}`; - const project = config.settings.projectId; - logger.error( - `BigQuery dataset or table not found: project="${project}", target="${target}". ` + - `Run "walkeros setup destination.${id}" to create them.`, - { - project, - target, - error: message, - }, - ); - } else { - // Catch-all so init failures are never silent. The destination is the - // layer with the most context about what was attempted (open writer, - // resolve stream, build proto descriptor), so it logs here before - // re-throwing. - logger.error('BigQuery init failed', { - error: message, - }); - } - throw err; - } + ).catch((err: unknown) => { + throw openError(err, settings, id); + }); return config; }, @@ -151,6 +106,25 @@ export const destinationBigQuery: Destination = { }, }; +/** + * A failed writer open as a StepError: a missing dataset or table is a setup + * step the operator has to run (not retryable), anything else a connection + * failure. The SDK's own error stays the cause. + */ +function openError(err: unknown, settings: Settings, id: string): StepError { + if (isNotFound(err)) + return new StepError( + `BigQuery dataset or table not found: project="${settings.projectId}", ` + + `target="${settings.datasetId}.${settings.tableId}". ` + + `Run "walkeros setup destination.${id}" to create them.`, + { code: 'config', cause: err }, + ); + return new StepError('BigQuery writer could not be opened', { + code: 'transport', + cause: err, + }); +} + // Type predicate for NOT_FOUND-class errors. The Storage Write API surfaces // missing datasets/tables as gRPC code 5 (NOT_FOUND) or HTTP 404. function isNotFound(err: unknown): boolean { diff --git a/packages/server/destinations/gcp/src/bigquery/push.ts b/packages/server/destinations/gcp/src/bigquery/push.ts index b17ec236b..fe2b419e6 100644 --- a/packages/server/destinations/gcp/src/bigquery/push.ts +++ b/packages/server/destinations/gcp/src/bigquery/push.ts @@ -1,31 +1,32 @@ import type { PushFn } from './types'; -import { isObject } from '@walkeros/core'; +import { isObject, StepError } from '@walkeros/core'; import { eventToRow } from './eventToRow'; import { ensureWriter } from './writer'; -export const push: PushFn = async function ( - event, - { config, rule: _rule, data, logger }, -) { - const settings = config.settings; - if (!settings) return logger.throw('settings missing, init() not run'); +// A silent rule has no row of its own: nothing to append. +export const push: PushFn = async function (event, context) { + if (!context.rule?.silent) await append(event, context); +}; - if (!settings.writer) - return logger.throw('writer is missing, init() not run'); +const append: PushFn = async function (event, { config, data, logger }) { + const settings = config.settings; + if (!settings) + throw new StepError('BigQuery settings missing, init() not run', { + code: 'not-ready', + }); - // Self-heal a writer that the connection's `'error'` handler flagged broken: - // exactly one re-open attempt before failing. On a failed re-open this throws, - // routing the event to the DLQ in-band (instead of an out-of-band crash) and - // feeding the collector's per-destination breaker. Breaker-gated: the - // collector does not call push while the breaker is OPEN, so the re-open runs - // at most once per admitted push, never in a loop. - if (settings.writerBroken) await ensureWriter(settings, logger); + // Open the writer at the first push, or self-heal one that the connection's + // `'error'` handler flagged broken: exactly one (re-)open attempt before + // failing. A failed open throws, routing the event to the DLQ in-band + // (instead of an out-of-band crash) and feeding the collector's + // per-destination breaker. Breaker-gated: the collector does not call push + // while the breaker is OPEN, so the open runs at most once per admitted + // push, never in a loop. + await ensureWriter(settings, logger); const { writer, datasetId, tableId } = settings; - - if (!writer) return logger.throw('writer is missing, init() not run'); - if (!datasetId) return logger.throw('datasetId is missing'); - if (!tableId) return logger.throw('tableId is missing'); + if (!writer) + throw new StepError('BigQuery writer is missing', { code: 'not-ready' }); const row = isObject(data) ? data : eventToRow(event); const rows = [row]; @@ -38,25 +39,26 @@ export const push: PushFn = async function ( let result; try { - const pending = writer.appendRows(rows); + // `appendRows` answers synchronously; an injected writer (tests, + // simulation) may answer with a promise, so its answer is awaited. + const pending = await writer.appendRows(rows); result = await pending.getResult(); } catch (err) { - // Log the failure and rethrow the raw error to the collector/DLQ. Secret - // redaction is standardized at the CLI logger handler; the raw error keeps - // its `code` so the row stays DLQ-routable. - logger.error('BigQuery row append threw', { - error: err instanceof Error ? err.message : String(err), + // The collector reports it and DLQ-routes the row; the SDK error (with + // its `code`) stays the cause. + throw new StepError('BigQuery row append threw', { + code: 'transport', + cause: err, }); - throw err; } - if (result.rowErrors && result.rowErrors.length > 0) { - // Single-event push path: throw with row context so the caller sees the failure. - const first = result.rowErrors[0]; - return logger.throw( - `BigQuery row append failed: code=${first.code} message=${first.message}`, + if (result.rowErrors && result.rowErrors.length > 0) + // The server rejected the row itself: a retry sends the same row. Its + // message can quote the row, so only the code is named. + throw new StepError( + `BigQuery row append failed (code ${result.rowErrors[0].code})`, + { code: 'transport', retryable: false }, ); - } logger.debug('BigQuery Storage Write API response', { ok: true, diff --git a/packages/server/destinations/gcp/src/bigquery/pushBatch.ts b/packages/server/destinations/gcp/src/bigquery/pushBatch.ts index 04fe2676d..aa176e613 100644 --- a/packages/server/destinations/gcp/src/bigquery/pushBatch.ts +++ b/packages/server/destinations/gcp/src/bigquery/pushBatch.ts @@ -1,5 +1,5 @@ import type { PushBatchFn } from './types'; -import { isObject } from '@walkeros/core'; +import { isObject, StepError } from '@walkeros/core'; import { eventToRow } from './eventToRow'; import { ensureWriter } from './writer'; @@ -10,88 +10,81 @@ import { ensureWriter } from './writer'; * as a BatchOutcome identifying the failed entries by index. The collector then * DLQs only those rows and counts the rest as delivered, so a later DLQ retry * does not re-write the already-succeeded rows (no duplicates). A whole-append - * failure (the appendRows call itself rejects, or the writer is missing) still - * throws, routing the entire batch to the DLQ. + * failure (the appendRows call itself rejects, or the writer cannot be opened) + * still throws, routing the entire batch to the DLQ. The collector writes the + * failure lines; this package writes none of its own. */ export const pushBatch: PushBatchFn = async (batch, { config, logger }) => { const settings = config.settings; - if (!settings) return logger.throw('settings missing, init() not run'); + if (!settings) + throw new StepError('BigQuery settings missing, init() not run', { + code: 'not-ready', + }); - if (!settings.writer) - return logger.throw('writer is missing, init() not run'); + // A silent rule has no row of its own. Entry indexes stay those of the + // flushed batch, which the collector maps failures back to. + const sent = batch.entries.flatMap((entry, index) => + entry.rule?.silent ? [] : [{ entry, index }], + ); + if (sent.length === 0) return; - // Self-heal a broken writer (one re-open attempt) before failing. On a failed - // re-open this throws, routing the WHOLE batch to the DLQ in-band and feeding - // the collector's per-destination breaker. Breaker-gated by the collector, so - // the re-open runs at most once per admitted batch. - if (settings.writerBroken) await ensureWriter(settings, logger); + // Open the writer at the first push, or self-heal a broken one (one + // re-open attempt) before failing. On a failed open this throws, routing the + // WHOLE batch to the DLQ in-band and feeding the collector's + // per-destination breaker. Breaker-gated by the collector, so the open runs + // at most once per admitted batch. + await ensureWriter(settings, logger); const { writer, datasetId, tableId } = settings; + if (!writer) + throw new StepError('BigQuery writer is missing', { code: 'not-ready' }); - if (!writer) return logger.throw('writer is missing, init() not run'); - if (!datasetId) return logger.throw('datasetId is missing'); - if (!tableId) return logger.throw('tableId is missing'); - - const rows = batch.entries.map((e) => - isObject(e.data) ? e.data : eventToRow(e.event), + const rows = sent.map(({ entry }) => + isObject(entry.data) ? entry.data : eventToRow(entry.event), ); - if (rows.length === 0) return; + logger.debug('Calling BigQuery Storage Write API (batch)', { + dataset: datasetId, + table: tableId, + rowCount: rows.length, + }); + let result; try { - logger.debug('Calling BigQuery Storage Write API (batch)', { - dataset: datasetId, - table: tableId, - rowCount: rows.length, + // `appendRows` answers synchronously; an injected writer (tests, + // simulation) may answer with a promise, so its answer is awaited. + const pending = await writer.appendRows(rows); + result = await pending.getResult(); + } catch (err) { + // The whole batch routes to the DLQ; the collector reports it once. + throw new StepError('BigQuery batch append threw', { + code: 'transport', + cause: err, }); - const pending = writer.appendRows(rows); - const result = await pending.getResult(); - - if (result.rowErrors && result.rowErrors.length > 0) { - const failed = result.rowErrors.length; - // Operator-visible summary at INFO, each row error at ERROR. - logger.info('BigQuery batch had partial failures', { - ok: rows.length - failed, - failed, - rowCount: rows.length, - }); - for (const re of result.rowErrors) { - logger.error('BigQuery row append failed', { - index: re.index, - code: re.code, - message: re.message, - }); - } - // Report per-row failures so the collector DLQs only the failed rows and - // counts the succeeded rows as delivered. `re.index` is the row's - // position in `rows`, which is built 1:1 from `batch.entries`, so it maps - // directly to the collector's entry index. The per-row error preserves - // the original BigQuery `code` and `message` for debugging from the DLQ. - // The Storage Write SDK types `index` as a protobuf int64 (number, string, - // or Long) and `message` as a nullable string, so both are normalized to - // the strict BatchFailure shape here. - const failures = result.rowErrors.map((re) => ({ - index: Number(re.index), - error: Object.assign(new Error(re.message ?? 'BigQuery row error'), { - code: re.code, - }), - })); + } - return { failed: failures }; - } + if (result.rowErrors && result.rowErrors.length > 0) { + // Report per-row failures so the collector DLQs only the failed rows and + // counts the succeeded rows as delivered. `re.index` is the row's position + // in `rows`, mapped back to the entry's index in the flushed batch. The + // Storage Write SDK types `index` as a protobuf int64 (number, string, or + // Long), normalized here. A row error's message can quote the row, so the + // error names only its code. + const failures = result.rowErrors.map((re) => ({ + index: sent[Number(re.index)]?.index ?? Number(re.index), + error: new StepError(`BigQuery row append failed (code ${re.code})`, { + code: 'transport', + retryable: false, + }), + })); - // Full success: keep at DEBUG to avoid noise on every batch. - logger.debug('BigQuery batch append ok', { - ok: rows.length, - failed: 0, - offset: result.appendResult?.offset?.value, - }); - } catch (err) { - // Log the failure and rethrow the raw error so the whole batch routes to the - // DLQ. Secret redaction is standardized at the CLI logger handler. - logger.error('BigQuery batch append threw', { - error: err instanceof Error ? err.message : String(err), - }); - throw err; + return { failed: failures }; } + + // Full success: keep at DEBUG to avoid noise on every batch. + logger.debug('BigQuery batch append ok', { + ok: rows.length, + failed: 0, + offset: result.appendResult?.offset?.value, + }); }; diff --git a/packages/server/destinations/gcp/src/bigquery/schemas/settings.ts b/packages/server/destinations/gcp/src/bigquery/schemas/settings.ts index 74f422726..609a95838 100644 --- a/packages/server/destinations/gcp/src/bigquery/schemas/settings.ts +++ b/packages/server/destinations/gcp/src/bigquery/schemas/settings.ts @@ -1,38 +1,40 @@ import { z } from '@walkeros/core/dev'; -export const SettingsSchema = z.object({ - client: z - .any() - .describe( - 'Google Cloud BigQuery client instance (like new BigQuery({ projectId, keyFilename }))', - ) - .optional(), - projectId: z - .string() - .min(1) - .describe('Google Cloud Project ID (like my-gcp-project)'), - datasetId: z - .string() - .min(1) - .default('walkerOS') - .describe( - 'BigQuery dataset ID where events will be stored (like walker_events)', - ), - tableId: z - .string() - .min(1) - .default('events') - .describe('BigQuery table ID for event storage (like events)'), - location: z - .string() - .default('EU') - .describe('Geographic location for the BigQuery dataset (like US)'), - bigquery: z - .any() - .describe( - 'Additional BigQuery client configuration options (like { keyFilename: "path/to/key.json" })', - ) - .optional(), -}); +export const SettingsSchema = z + .object({ + client: z + .any() + .describe( + 'Google Cloud BigQuery client instance (like new BigQuery({ projectId, keyFilename }))', + ) + .optional(), + projectId: z + .string() + .min(1) + .describe('Google Cloud Project ID (like my-gcp-project)'), + datasetId: z + .string() + .min(1) + .default('walkerOS') + .describe( + 'BigQuery dataset ID where events will be stored (like walker_events)', + ), + tableId: z + .string() + .min(1) + .default('events') + .describe('BigQuery table ID for event storage (like events)'), + location: z + .string() + .default('EU') + .describe('Geographic location for the BigQuery dataset (like US)'), + bigquery: z + .any() + .describe( + 'Additional BigQuery client configuration options (like { keyFilename: "path/to/key.json" })', + ) + .optional(), + }) + .strict(); export type Settings = z.infer; diff --git a/packages/server/destinations/gcp/src/bigquery/writer.ts b/packages/server/destinations/gcp/src/bigquery/writer.ts index c5c2cde5d..9526f6819 100644 --- a/packages/server/destinations/gcp/src/bigquery/writer.ts +++ b/packages/server/destinations/gcp/src/bigquery/writer.ts @@ -1,4 +1,5 @@ import type { Logger, ServiceAccount } from '@walkeros/core'; +import { StepError } from '@walkeros/core'; import type { BigQueryOptions } from '@google-cloud/bigquery'; import { managedwriter, adapt, protos } from '@google-cloud/bigquery-storage'; @@ -319,9 +320,9 @@ export interface EnsureWriterSettings { } /** - * Single owner of the writer lifecycle on the push path. When the connection's - * `'error'` handler has flagged the writer broken, attempt a lazy re-open - * before failing: + * Single owner of the writer lifecycle on the push path. The first push opens + * the writer (init stays offline). When the connection's `'error'` handler has + * flagged the writer broken, attempt a lazy re-open before failing: * - success: swap in the fresh handles, clear the broken flag, proceed. * - failure: stay broken, throw so the event is DLQ-routed. * @@ -353,27 +354,26 @@ export function ensureWriter( settings: EnsureWriterSettings, logger: Logger.Instance, ): Promise { - if (!settings.writerBroken) return Promise.resolve(); + const opening = !settings.writer; + if (!opening && !settings.writerBroken) return Promise.resolve(); - // A concurrent caller already started the re-open: join it instead of opening - // a second connection (which would orphan one of them). + // A concurrent caller already started the (re-)open: join it instead of + // opening a second connection (which would orphan one of them). if (settings.reopenInFlight) return settings.reopenInFlight; - const lastError = settings.lastStreamError; - logger.info('BigQuery writer broken; attempting one re-open before failing', { - error: lastError ? lastError.message : 'unknown stream error', - }); - const reopen = settings.reopenWriter; - if (!reopen) { - // No re-open hook (init never wired it): cannot self-heal, stay broken. + if (!reopen) + // No open hook (init never wired it): cannot open or self-heal. return Promise.reject( - new Error( - 'BigQuery writer is broken and no re-open hook is configured: ' + - (lastError ? lastError.message : 'unknown stream error'), - ), + new StepError('BigQuery writer is not initialized', { + code: 'not-ready', + }), + ); + + if (!opening) + logger.info( + 'BigQuery writer broken; attempting one re-open before failing', ); - } const inFlight = (async () => { try { @@ -397,7 +397,8 @@ export function ensureWriter( settings.connectionErrorListener = handles.connectionErrorListener; settings.writerBroken = false; settings.lastStreamError = undefined; - logger.info('BigQuery writer re-opened after a stream error'); + if (!opening) + logger.info('BigQuery writer re-opened after a stream error'); } finally { // Clear the memo so a later push after a FAILED re-open retries (on a // successful re-open writerBroken is already false, so the next push skips diff --git a/packages/server/destinations/gcp/src/pubsub/config.ts b/packages/server/destinations/gcp/src/pubsub/config.ts index c39e6d38d..024cd1a61 100644 --- a/packages/server/destinations/gcp/src/pubsub/config.ts +++ b/packages/server/destinations/gcp/src/pubsub/config.ts @@ -7,7 +7,7 @@ import type { Settings, } from './types'; import type { Logger } from '@walkeros/core'; -import { isObject } from '@walkeros/core'; +import { isObject, StepError } from '@walkeros/core'; import { PubSub } from '@google-cloud/pubsub'; interface ResolvedClientConfig { @@ -48,12 +48,18 @@ export function getConfig( typeof settings.projectId === 'string' ? settings.projectId : ''; const topic = typeof settings.topic === 'string' ? settings.topic : ''; - if (!projectId) logger.throw('Config settings projectId missing'); - if (!topic) logger.throw('Config settings topic missing'); + if (!projectId || !topic) { + const missing = [ + ...(projectId ? [] : ['projectId']), + ...(topic ? [] : ['topic']), + ]; + throw new StepError(`Config settings ${missing.join(', ')} missing`, { + code: 'config', + }); + } const credentials = parseCredentials( resolveCredentials(partialConfig, logger), - logger, ); // Build the client once: prefer pre-supplied settings.client, then env- @@ -103,23 +109,21 @@ export function resolveCredentials( export function parseCredentials( raw: Settings['credentials'], - logger: Logger.Instance, ): Settings['credentials'] { if (raw === undefined) return undefined; - if (typeof raw === 'string') { - try { - const parsed: unknown = JSON.parse(raw); - if (!isServiceAccountCredentials(parsed)) { - logger.throw('Invalid credentials JSON'); - return undefined; - } - return parsed; - } catch { - logger.throw('Invalid credentials JSON'); - return undefined; - } + if (typeof raw !== 'string') return raw; + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch (err) { + throw new StepError('Invalid credentials JSON', { + code: 'config', + cause: err, + }); } - return raw; + if (!isServiceAccountCredentials(parsed)) + throw new StepError('Invalid credentials JSON', { code: 'config' }); + return parsed; } function isServiceAccountCredentials( diff --git a/packages/server/destinations/gcp/src/pubsub/examples/step.ts b/packages/server/destinations/gcp/src/pubsub/examples/step.ts index 411d17af6..f900e42bc 100644 --- a/packages/server/destinations/gcp/src/pubsub/examples/step.ts +++ b/packages/server/destinations/gcp/src/pubsub/examples/step.ts @@ -201,3 +201,12 @@ export const pageView: Flow.StepExample = { ['publishMessage', 'events', { data: expectedPayload(pageEvent) }], ], }; + +/** A silent rule publishes no message. */ +export const silentRule: Flow.StepExample = { + public: false, + description: 'A rule with silent: true publishes no message.', + in: getEvent('page view', { timestamp: 1700000000500 }), + mapping: { silent: true }, + out: [], +}; diff --git a/packages/server/destinations/gcp/src/pubsub/push.ts b/packages/server/destinations/gcp/src/pubsub/push.ts index 49e419c3e..729e9955e 100644 --- a/packages/server/destinations/gcp/src/pubsub/push.ts +++ b/packages/server/destinations/gcp/src/pubsub/push.ts @@ -1,6 +1,6 @@ import type { PushFn, Mapping, Settings } from './types'; import type { Mapping as WalkerOSMapping, Collector } from '@walkeros/core'; -import { getMappingValue, isObject } from '@walkeros/core'; +import { getMappingValue, isObject, StepError } from '@walkeros/core'; const NOT_FOUND_GRPC = 5; const NOT_FOUND_HTTP = 404; @@ -105,14 +105,23 @@ function isMapping(value: unknown): value is Mapping { return typeof value === 'object' && value !== null; } -export const push: PushFn = async function ( +// A silent rule has no message of its own: nothing to publish. +export const push: PushFn = async function (event, context) { + if (!context.rule?.silent) await publish(event, context); +}; + +const publish: PushFn = async function ( event, { config, rule, data, collector, logger, id }, ) { const settings = config.settings; - if (!isResolvedSettings(settings)) { - return logger.throw('settings missing or incomplete, init() not run'); - } + if (!isResolvedSettings(settings)) + throw new StepError( + 'Pub/Sub settings missing or incomplete, init() not run', + { + code: 'not-ready', + }, + ); const { client, projectId, topic: defaultTopic } = settings; @@ -170,7 +179,6 @@ export const push: PushFn = async function ( logger.debug('Pub/Sub publish', { topic: topicName, projectId, - orderingKey, event: event.name, }); @@ -192,34 +200,24 @@ export const push: PushFn = async function ( } } - if (isNotFound(err)) { - logger.error( + // The collector reports it; the SDK error (with its `code`) stays the + // cause. A missing topic and a denied publish are setup steps the + // operator has to take, not retried. + if (isNotFound(err)) + throw new StepError( `Pub/Sub topic "${topicName}" not found in project "${projectId}". ` + `Run "walkeros setup destination.${id}" to create it.`, - { - topic: topicName, - projectId, - error: err instanceof Error ? err.message : String(err), - }, + { code: 'config', cause: err }, ); - } else if (isPermissionError(err)) { - logger.error( + if (isPermissionError(err)) + throw new StepError( `Pub/Sub publish denied for topic "${topicName}". Grant the runtime ` + `service account roles/pubsub.publisher on this topic.`, - { - topic: topicName, - projectId, - error: err instanceof Error ? err.message : String(err), - }, + { code: 'config', cause: err }, ); - } else { - logger.error('Pub/Sub publish failed', { - topic: topicName, - projectId, - event: event.name, - error: err instanceof Error ? err.message : String(err), - }); - } - throw err; + throw new StepError('Pub/Sub publish failed', { + code: 'transport', + cause: err, + }); } }; diff --git a/packages/server/destinations/gcp/src/pubsub/schemas/settings.ts b/packages/server/destinations/gcp/src/pubsub/schemas/settings.ts index a8ed1bdbc..1e67d90cf 100644 --- a/packages/server/destinations/gcp/src/pubsub/schemas/settings.ts +++ b/packages/server/destinations/gcp/src/pubsub/schemas/settings.ts @@ -1,46 +1,48 @@ import { z } from '@walkeros/core/dev'; -export const SettingsSchema = z.object({ - client: z - .any() - .describe( - 'Google Cloud Pub/Sub client instance (like new PubSub({ projectId, keyFilename }))', - ) - .optional(), - projectId: z - .string() - .min(1) - .describe('Google Cloud Project ID (like my-gcp-project)'), - topic: z - .string() - .min(1) - .describe( - 'Pub/Sub topic short name (like events). The full resource name projects//topics/ is built by the SDK.', - ), - credentials: z - .any() - .describe( - 'Service account credentials as a JSON string or an object with client_email and private_key. Default: Application Default Credentials (ADC). (deprecated: use config.credentials)', - ) - .optional(), - apiEndpoint: z - .string() - .describe( - 'Override Pub/Sub API endpoint. Useful for the local emulator (like localhost:8085).', - ) - .optional(), - orderingKey: z - .any() - .describe( - 'Mapping value resolved per-event. Truthy enables per-key ordering for the publish.', - ) - .optional(), - attributes: z - .any() - .describe( - 'Default per-event attribute map merged into every published message. Mapping.Map shape.', - ) - .optional(), -}); +export const SettingsSchema = z + .object({ + client: z + .any() + .describe( + 'Google Cloud Pub/Sub client instance (like new PubSub({ projectId, keyFilename }))', + ) + .optional(), + projectId: z + .string() + .min(1) + .describe('Google Cloud Project ID (like my-gcp-project)'), + topic: z + .string() + .min(1) + .describe( + 'Pub/Sub topic short name (like events). The full resource name projects//topics/ is built by the SDK.', + ), + credentials: z + .any() + .describe( + 'Service account credentials as a JSON string or an object with client_email and private_key. Default: Application Default Credentials (ADC). (deprecated: use config.credentials)', + ) + .optional(), + apiEndpoint: z + .string() + .describe( + 'Override Pub/Sub API endpoint. Useful for the local emulator (like localhost:8085).', + ) + .optional(), + orderingKey: z + .any() + .describe( + 'Mapping value resolved per-event. Truthy enables per-key ordering for the publish.', + ) + .optional(), + attributes: z + .any() + .describe( + 'Default per-event attribute map merged into every published message. Mapping.Map shape.', + ) + .optional(), + }) + .strict(); export type Settings = z.infer; diff --git a/packages/server/destinations/gcp/src/pubsub/setup.ts b/packages/server/destinations/gcp/src/pubsub/setup.ts index 94ed4d289..4d8a6ef0a 100644 --- a/packages/server/destinations/gcp/src/pubsub/setup.ts +++ b/packages/server/destinations/gcp/src/pubsub/setup.ts @@ -96,10 +96,7 @@ export async function setup( ); } if (!client) { - const credentials = parseCredentials( - resolveCredentials(config, logger), - logger, - ); + const credentials = parseCredentials(resolveCredentials(config, logger)); const clientOptions = { projectId, ...(credentials !== undefined && typeof credentials !== 'string' diff --git a/packages/server/destinations/hubspot/src/__tests__/batch.test.ts b/packages/server/destinations/hubspot/src/__tests__/batch.test.ts new file mode 100644 index 000000000..c88389c8e --- /dev/null +++ b/packages/server/destinations/hubspot/src/__tests__/batch.test.ts @@ -0,0 +1,56 @@ +import { startFlow } from '@walkeros/collector'; +import { createEvent } from '@walkeros/core'; +import { destinationHubspot } from '..'; +import type { HubSpotClientMock, HubSpotEventRequest } from '../types'; + +function clientRecording(batches: HubSpotEventRequest[][]): HubSpotClientMock { + return { + events: { + send: { + basicApi: { send: () => Promise.resolve() }, + batchApi: { + send: async ({ inputs }) => { + batches.push(inputs); + }, + }, + }, + }, + crm: { contacts: { basicApi: { update: () => Promise.resolve() } } }, + }; +} + +describe('hubspot config.batch', () => { + it('sends a flushed batch through the batch API and fails only the entry it cannot send', async () => { + const batches: HubSpotEventRequest[][] = []; + const { collector } = await startFlow({ + logger: { level: 'ERROR', handler: () => undefined }, + destinations: { + hubspot: { + code: destinationHubspot, + config: { + settings: { accessToken: 'pat', eventNamePrefix: 'pe1_' }, + batch: { size: 10, wait: 60_000 }, + }, + env: { client: clientRecording(batches) }, + }, + }, + }); + + for (const user of [{ email: 'a@x.com' }, {}, { email: 'b@x.com' }]) + await collector.push(createEvent({ name: 'page view', user })); + const flushed = await collector.command('flush'); + + expect(batches).toHaveLength(1); + expect(batches[0].map(({ email }) => email)).toEqual([ + 'a@x.com', + 'b@x.com', + ]); + expect(flushed.ok).toBe(false); + const status = collector.status.destinations.hubspot; + expect(status.count).toBe(2); + expect(status.failed).toBe(1); + expect(collector.destinations.hubspot.dlq).toHaveLength(1); + + await collector.command('shutdown'); + }); +}); diff --git a/packages/server/destinations/hubspot/src/__tests__/conformance.test.ts b/packages/server/destinations/hubspot/src/__tests__/conformance.test.ts new file mode 100644 index 000000000..2749b4152 --- /dev/null +++ b/packages/server/destinations/hubspot/src/__tests__/conformance.test.ts @@ -0,0 +1,51 @@ +import path from 'path'; +import { conformance, runExamples } from '@walkeros/collector/dev'; +import { destinationHubspot } from '..'; +import { examples, schemas } from '../dev'; + +// The runs without env load the SDK: keep it a local stand-in. +jest.mock('@hubspot/api-client', () => ({ + __esModule: true, + Client: class { + events = { + send: { + basicApi: { send: () => Promise.resolve() }, + batchApi: { send: () => Promise.resolve() }, + }, + }; + crm = { contacts: { basicApi: { update: () => Promise.resolve() } } }; + }, +})); + +describe('hubspot conformance', () => { + beforeEach(() => { + jest.useRealTimers(); + }); + + it('conforms', async () => { + await conformance('destination', destinationHubspot, { + profile: 'sdk-env', + examples: examples.step, + schema: schemas.settings, + settings: { accessToken: 'pat-test-xxx', eventNamePrefix: 'pe12345678_' }, + env: { push: examples.env.push, simulation: examples.env.simulation }, + root: path.join(__dirname, '..', '..'), + }); + }); + + it('sends nothing for a silent identify rule that finds no email', async () => { + const { userLoginWithoutEmail } = examples.step; + await runExamples( + 'destination', + destinationHubspot, + { userLoginWithoutEmail }, + { + settings: { + accessToken: 'pat-test-xxx', + eventNamePrefix: 'pe12345678_', + }, + env: { push: examples.env.push, simulation: examples.env.simulation }, + }, + ); + }); +}); diff --git a/packages/server/destinations/hubspot/src/__tests__/init.test.ts b/packages/server/destinations/hubspot/src/__tests__/init.test.ts index a8e7e0ec8..b4361851b 100644 --- a/packages/server/destinations/hubspot/src/__tests__/init.test.ts +++ b/packages/server/destinations/hubspot/src/__tests__/init.test.ts @@ -1,3 +1,4 @@ +import { createMockContext } from '@walkeros/core'; import { destinationHubspot } from '..'; describe('init', () => { @@ -12,6 +13,20 @@ describe('init', () => { }), }; + it('throws a config StepError naming every missing setting', () => { + expect(() => + destinationHubspot.init( + createMockContext({ config: { settings: {} }, id: 'test' }), + ), + ).toThrow( + expect.objectContaining({ + name: 'StepError', + code: 'config', + message: 'Config settings accessToken, eventNamePrefix missing', + }), + ); + }); + it('throws when accessToken is missing', () => { expect(() => destinationHubspot.init!({ @@ -32,55 +47,3 @@ describe('init', () => { ).toThrow('eventNamePrefix'); }); }); - -describe('destroy', () => { - it('flushes remaining event queue on destroy', async () => { - const batchSend = jest.fn().mockResolvedValue(undefined); - const settings = { - accessToken: 'pat-xxx', - eventNamePrefix: 'pe12345678_', - batch: true, - batchSize: 50, - _client: { - events: { - send: { - basicApi: { send: jest.fn() }, - batchApi: { send: batchSend }, - }, - }, - crm: { contacts: { basicApi: { update: jest.fn() } } }, - }, - _eventQueue: [ - { - eventName: 'pe12345678_test', - email: 'user@example.com', - properties: {}, - }, - ], - _state: {}, - }; - - await destinationHubspot.destroy!({ config: { settings } } as never); - - expect(batchSend).toHaveBeenCalledWith({ - inputs: [ - { - eventName: 'pe12345678_test', - email: 'user@example.com', - properties: {}, - }, - ], - }); - }); - - it('does nothing when no event queue exists', async () => { - const settings = { - accessToken: 'pat-xxx', - eventNamePrefix: 'pe12345678_', - _state: {}, - }; - - // Should not throw - await destinationHubspot.destroy!({ config: { settings } } as never); - }); -}); diff --git a/packages/server/destinations/hubspot/src/config.ts b/packages/server/destinations/hubspot/src/config.ts index 1c30bd1a4..a9d1a8ab8 100644 --- a/packages/server/destinations/hubspot/src/config.ts +++ b/packages/server/destinations/hubspot/src/config.ts @@ -1,15 +1,19 @@ import type { Config, Settings, PartialConfig } from './types'; -import type { Logger } from '@walkeros/core'; +import { StepError } from '@walkeros/core'; -export function getConfig( - partialConfig: PartialConfig = {}, - logger: Logger.Instance, -): Config { - const settings = (partialConfig.settings || {}) as Partial; +export function getConfig(partialConfig: PartialConfig = {}): Config { + const settings: Partial = partialConfig.settings || {}; const { accessToken, eventNamePrefix } = settings; - if (!accessToken) logger.throw('Config settings accessToken missing'); - if (!eventNamePrefix) logger.throw('Config settings eventNamePrefix missing'); + if (!accessToken || !eventNamePrefix) { + const missing = [ + ...(accessToken ? [] : ['accessToken']), + ...(eventNamePrefix ? [] : ['eventNamePrefix']), + ]; + throw new StepError(`Config settings ${missing.join(', ')} missing`, { + code: 'config', + }); + } const settingsConfig: Settings = { ...settings, @@ -17,9 +21,6 @@ export function getConfig( eventNamePrefix, // Default identity resolution path email: settings.email ?? 'user.email', - // Batch defaults - batch: settings.batch ?? false, - batchSize: Math.min(settings.batchSize ?? 50, 500), }; return { ...partialConfig, settings: settingsConfig }; diff --git a/packages/server/destinations/hubspot/src/examples/step.ts b/packages/server/destinations/hubspot/src/examples/step.ts index 30f3fa18a..460c24297 100644 --- a/packages/server/destinations/hubspot/src/examples/step.ts +++ b/packages/server/destinations/hubspot/src/examples/step.ts @@ -253,15 +253,40 @@ export const objectIdAssociation: HubSpotStepExample = { }; /** - * No identity resolved -- event is skipped with a warning. Neither email - * nor objectId can be resolved from the event. + * No identity resolved -- neither email nor objectId resolves from the + * event, so it cannot be associated with a contact: the push fails with + * missing-input instead of counting as delivered. One per event name above, + * so every example's event without identity has its declared outcome. */ -export const noIdentity: HubSpotStepExample = { +function withoutIdentity(name: string, timestamp: number): HubSpotStepExample { + return { + public: false, + description: `A ${name} event without email or objectId cannot be associated with a HubSpot contact: the push fails with missing-input instead of counting as delivered.`, + in: { name, timestamp }, + out: [], + outcome: { status: 'failed', code: 'missing-input', retryable: false }, + }; +} + +export const noIdentity = withoutIdentity('product view', 1700000106); +export const noIdentityOrder = withoutIdentity('order complete', 1700000108); +export const noIdentityPage = withoutIdentity('page view', 1700000109); +export const noIdentityLogin = withoutIdentity('user login', 1700000110); + +/** + * Silent rule -- no custom event is sent; without an identify mapping there + * is no contact upsert either. + */ +export const silentRule: HubSpotStepExample = { public: false, - in: getEvent('product view', { - timestamp: 1700000106, - user: {}, + description: + 'A rule with silent: true sends no HubSpot event; without an identify mapping nothing is called.', + in: createEvent({ + name: 'page view', + timestamp: 1700000111, + user: { email: 'user@example.com' }, }), + mapping: { silent: true }, out: [], }; @@ -279,3 +304,23 @@ export const wildcardIgnored: HubSpotStepExample = { mapping: { ignore: true }, out: [], }; + +/** + * The silent identify rule of userLoginIdentify on a login without the email + * it maps: no contact can be upserted and the rule suppresses the event, so + * nothing is sent. + */ +export const userLoginWithoutEmail: HubSpotStepExample = { + public: false, + description: + 'A silent identify rule whose email finds no data sends nothing: an unmet mapping, not a failure, so the push returns normally.', + in: { + name: 'user login', + entity: 'user', + action: 'login', + timestamp: 1700000112, + user: { email: 'user@example.com' }, + }, + mapping: userLoginIdentify.mapping, + out: [], +}; diff --git a/packages/server/destinations/hubspot/src/index.ts b/packages/server/destinations/hubspot/src/index.ts index 6767172f4..cf08fa5d3 100644 --- a/packages/server/destinations/hubspot/src/index.ts +++ b/packages/server/destinations/hubspot/src/index.ts @@ -1,60 +1,47 @@ -import type { Destination, Settings, HubSpotClientMock } from './types'; +import type { Destination } from './types'; +import { StepError } from '@walkeros/core'; import { getConfig } from './config'; -import { push, flushBatch } from './push'; +import { push, pushBatch } from './push'; // Types export * as DestinationHubspot from './types'; +/** + * Batching is the collector's `config.batch`: a flushed batch goes out + * through the batch API in `pushBatch`, with each entry's own outcome. + */ export const destinationHubspot: Destination = { type: 'hubspot', config: {}, - init({ config: partialConfig, logger, env }) { - const config = getConfig(partialConfig, logger); - const settings = config.settings as Settings; + init({ config: partialConfig, env }) { + const config = getConfig(partialConfig); + const { settings } = config; // Use env.client mock if provided (testing), otherwise create real SDK - const envClient = (env as { client?: HubSpotClientMock } | undefined) - ?.client; - - if (!envClient) { - // Production path: create real HubSpot Client instance + if (!env?.client) { try { const { Client } = require('@hubspot/api-client'); settings._client = new Client({ accessToken: settings.accessToken, }); } catch (err) { - logger.throw(`Failed to initialize HubSpot SDK: ${err}`); + throw new StepError('Failed to initialize the HubSpot SDK', { + code: 'not-ready', + cause: err, + }); } } settings._state = {}; - if (settings.batch) settings._eventQueue = []; return config; }, - async push(event, context) { - return await push(event, context); - }, + push, - async destroy({ config }) { - const settings = (config?.settings || {}) as Settings; - - // Flush remaining queued events in batch mode - if ( - settings.batch && - settings._eventQueue && - settings._eventQueue.length > 0 - ) { - const sdk = settings._client; - if (sdk) { - await flushBatch(sdk, settings); - } - } - }, + pushBatch, }; export default destinationHubspot; diff --git a/packages/server/destinations/hubspot/src/push.ts b/packages/server/destinations/hubspot/src/push.ts index 07a0bf6b6..a4ad2f501 100644 --- a/packages/server/destinations/hubspot/src/push.ts +++ b/packages/server/destinations/hubspot/src/push.ts @@ -1,27 +1,71 @@ import type { PushFn, + PushBatchFn, Settings, RuntimeState, HubSpotClientMock, HubSpotEventRequest, + Env, + Rule, } from './types'; -import type { Logger } from '@walkeros/core'; -import { getMappingValue, isObject, isString } from '@walkeros/core'; +import type { Collector, Destination, Logger, WalkerOS } from '@walkeros/core'; +import { getMappingValue, isObject, isString, StepError } from '@walkeros/core'; -export const push: PushFn = async function ( - event, - { config, rule, collector, env, logger }, -) { - const settings = config.settings as Settings; - const envClient = (env as { client?: HubSpotClientMock } | undefined)?.client; - const sdk = envClient || settings._client; +/** The HubSpot batch API takes at most this many events per request. */ +const MAX_BATCH = 500; - if (!sdk) { - logger.warn('HubSpot client not initialized'); - return; +interface Prepared { + sdk: HubSpotClientMock; + /** The event to send; none for a silent rule. */ + request?: HubSpotEventRequest; +} + +/** + * A failed HubSpot call as a StepError. The SDK's ApiException carries the + * HTTP status as `code` and the response body in its message, so only the + * status travels; the original error stays the cause. + */ +function hubspotError(message: string, error: unknown): StepError { + const status = + typeof error === 'object' && + error !== null && + 'code' in error && + typeof error.code === 'number' + ? error.code + : undefined; + return new StepError(message, { code: 'transport', status, cause: error }); +} + +/** Run one HubSpot call; a failure throws a StepError. */ +async function call(message: string, send: () => Promise) { + try { + await send(); + } catch (error) { + throw hubspotError(message, error); } +} + +/** + * Resolve one event: its identity, the contact upsert, and the event request + * (none for a silent rule). An event without email or objectId cannot be + * sent and fails with missing-input. + */ +async function prepare( + event: WalkerOS.Event, + settings: Partial | undefined, + env: Env | undefined, + rule: Rule | undefined, + collector: Collector.Instance, + logger: Logger.Instance, +): Promise { + const sdk = env?.client || settings?._client; + if (!isReady(settings) || !sdk) + throw new StepError('HubSpot client not initialized', { + code: 'not-ready', + }); const state: RuntimeState = settings._state || {}; + settings._state = state; const mappingSettings = rule?.settings || {}; // 1. Resolve identity from event @@ -34,12 +78,11 @@ export const push: PushFn = async function ( ) : undefined; - if (!email && !objectId) { - logger.warn('HubSpot requires email or objectId; skipping event', { - event: event.name, - }); - return; - } + if (!email && !objectId) + throw new StepError( + 'HubSpot needs an email or objectId to associate the event', + { code: 'missing-input' }, + ); // 2. Contact upsert -- rule-level overrides destination-level const identifyMapping = mappingSettings.identify ?? settings.identify; @@ -47,71 +90,122 @@ export const push: PushFn = async function ( const resolved = await getMappingValue(event, identifyMapping, { collector, }); - if (isObject(resolved)) { - await applyIdentify( - sdk, - resolved as Record, - state, - logger, - ); - } + if (isObject(resolved)) await applyIdentify(sdk, resolved, state, logger); } - // 3. Send event (unless silent: true) - if (rule?.silent !== true) { - const eventName = buildEventName( - settings.eventNamePrefix, - mappingSettings.eventName, - isString(rule?.name) ? rule.name : event.name, - ); + // 3. The event (unless silent: true) + if (rule?.silent === true) return { sdk }; - // Build properties: defaultProperties + mapped properties, all strings - let properties: Record = { - ...(settings.defaultProperties || {}), - }; + const eventName = buildEventName( + settings.eventNamePrefix, + mappingSettings.eventName, + isString(rule?.name) ? rule.name : event.name, + ); - if (mappingSettings.properties !== undefined) { - const resolved = await getMappingValue( - event, - mappingSettings.properties, - { collector }, - ); - if (isObject(resolved)) { - properties = { - ...properties, - ...toStringProperties(resolved as Record), - }; - } - } + // Build properties: defaultProperties + mapped properties, all strings + let properties: Record = { + ...(settings.defaultProperties || {}), + }; - const occurredAt = new Date(event.timestamp || Date.now()); + if (mappingSettings.properties !== undefined) { + const resolved = await getMappingValue(event, mappingSettings.properties, { + collector, + }); + if (isObject(resolved)) + properties = { ...properties, ...toStringProperties(resolved) }; + } + + const request: HubSpotEventRequest = { + eventName, + occurredAt: new Date(event.timestamp || Date.now()), + properties, + }; - const eventRequest: HubSpotEventRequest = { - eventName, - occurredAt, - properties, - }; + // Attach identity + if (email) request.email = email; + if (objectId) request.objectId = objectId; - // Attach identity - if (email) eventRequest.email = email; - if (objectId) eventRequest.objectId = objectId; + return { sdk, request }; +} + +export const push: PushFn = async function ( + event, + { config, rule, collector, env, logger }, +) { + const { sdk, request } = await prepare( + event, + config.settings, + env, + rule, + collector, + logger, + ); + if (request) + await call('HubSpot event send failed', () => + sdk.events.send.basicApi.send(request), + ); +}; + +/** + * A flushed `config.batch` batch: each entry resolved in order (its contact + * upsert included), the events sent through the batch API in chunks of 500. + * A failed chunk fails its own entries, and an entry that cannot be resolved + * fails alone; the rest are delivered. When nothing landed, the batch throws. + */ +export const pushBatch: PushBatchFn = async function ( + batch, + { config, collector, env, logger }, +) { + const failed: Destination.BatchFailure[] = []; + const sends: Array<{ index: number; request: HubSpotEventRequest }> = []; + let sdk: HubSpotClientMock | undefined; - if (settings.batch) { - // Queue for batch flush - if (!settings._eventQueue) settings._eventQueue = []; - settings._eventQueue.push(eventRequest); + for (const [index, entry] of batch.entries.entries()) { + try { + const prepared = await prepare( + entry.event, + config.settings, + env, + entry.rule, + collector, + logger, + ); + sdk = prepared.sdk; + if (prepared.request) sends.push({ index, request: prepared.request }); + } catch (error) { + failed.push({ index, error }); + } + } - if (settings._eventQueue.length >= (settings.batchSize || 50)) { - await flushBatch(sdk, settings); - } - } else { - await sdk.events.send.basicApi.send(eventRequest); + let sendError: unknown; + for (let start = 0; sdk && start < sends.length; start += MAX_BATCH) { + const chunk = sends.slice(start, start + MAX_BATCH); + const client = sdk; + try { + await call('HubSpot batch send failed', () => + client.events.send.batchApi.send({ + inputs: chunk.map(({ request }) => request), + }), + ); + } catch (error) { + sendError = sendError ?? error; + for (const { index } of chunk) failed.push({ index, error }); } } - settings._state = state; + // Nothing landed: a whole-batch failure, which the breaker counts. + if (failed.length === batch.entries.length && failed.length) + throw sendError ?? failed[0].error; + if (failed.length) return { failed }; }; +/** Settings as init resolved them, the required ones present. */ +function isReady( + settings: Partial | undefined, +): settings is Settings { + return Boolean(settings?.accessToken && settings.eventNamePrefix); +} + function resolveString(value: unknown): string | undefined { if (isString(value) && value.length > 0) return value; return undefined; @@ -154,7 +248,7 @@ function hashProperties( async function applyIdentify( sdk: HubSpotClientMock, - resolved: Record, + resolved: WalkerOS.AnyObject, state: RuntimeState, logger: Logger.Instance, ): Promise { @@ -165,7 +259,7 @@ async function applyIdentify( } const rawProperties = isObject(resolved.properties) - ? toStringProperties(resolved.properties as Record) + ? toStringProperties(resolved.properties) : undefined; const propertiesHash = hashProperties(rawProperties); @@ -176,31 +270,17 @@ async function applyIdentify( if (!emailChanged && !propertiesChanged) return; - if (rawProperties) { - await sdk.crm.contacts.basicApi.update( - identifyEmail, - { properties: rawProperties }, - 'email', + if (rawProperties) + await call('HubSpot contact update failed', () => + sdk.crm.contacts.basicApi.update( + identifyEmail, + { properties: rawProperties }, + 'email', + ), ); - } state.lastIdentity = { email: identifyEmail, propertiesHash, }; } - -export async function flushBatch( - sdk: HubSpotClientMock, - settings: Settings, -): Promise { - const queue = settings._eventQueue; - if (!queue || queue.length === 0) return; - - // Flush in chunks of batchSize (max 500) - const batchSize = Math.min(settings.batchSize || 50, 500); - while (queue.length > 0) { - const batch = queue.splice(0, batchSize); - await sdk.events.send.batchApi.send({ inputs: batch }); - } -} diff --git a/packages/server/destinations/hubspot/src/schemas/settings.ts b/packages/server/destinations/hubspot/src/schemas/settings.ts index be5c81862..a864f9ea4 100644 --- a/packages/server/destinations/hubspot/src/schemas/settings.ts +++ b/packages/server/destinations/hubspot/src/schemas/settings.ts @@ -1,57 +1,44 @@ import { z } from '@walkeros/core/dev'; -export const SettingsSchema = z.object({ - accessToken: z - .string() - .min(1) - .describe( - 'HubSpot private app access token. Create one in HubSpot Settings > Integrations > Private Apps. Requires analytics.behavioral_events.send scope.', - ), - eventNamePrefix: z - .string() - .min(1) - .describe( - 'Fully qualified event name prefix: pe{HubID}_ (e.g. pe12345678_). Find it in HubSpot under Data Management > Custom Events.', - ), - email: z - .string() - .describe( - 'walkerOS mapping value path to resolve contact email from events (like user.email). Required for contact association.', - ) - .optional(), - objectId: z - .string() - .describe( - 'walkerOS mapping value path to resolve HubSpot CRM objectId from events. Alternative to email for contact association.', - ) - .optional(), - identify: z - .unknown() - .describe( - 'Destination-level contact upsert mapping. Resolves to { email, properties }. Fires contact update on first push and re-fires when values change.', - ) - .optional(), - defaultProperties: z - .record(z.string(), z.string()) - .describe( - 'Static event properties added to every event occurrence. Useful for hs_touchpoint_source, hs_page_content_type, etc.', - ) - .optional(), - batch: z - .boolean() - .describe( - 'Use batch API for events (accumulate and flush). Default: false.', - ) - .optional(), - batchSize: z - .number() - .int() - .positive() - .max(500) - .describe( - 'Batch size before auto-flush. Only used when batch: true. Default: 50. Max: 500.', - ) - .optional(), -}); +export const SettingsSchema = z + .object({ + accessToken: z + .string() + .min(1) + .describe( + 'HubSpot private app access token. Create one in HubSpot Settings > Integrations > Private Apps. Requires analytics.behavioral_events.send scope.', + ), + eventNamePrefix: z + .string() + .min(1) + .describe( + 'Fully qualified event name prefix: pe{HubID}_ (e.g. pe12345678_). Find it in HubSpot under Data Management > Custom Events.', + ), + email: z + .string() + .describe( + 'walkerOS mapping value path to resolve contact email from events (like user.email). Required for contact association.', + ) + .optional(), + objectId: z + .string() + .describe( + 'walkerOS mapping value path to resolve HubSpot CRM objectId from events. Alternative to email for contact association.', + ) + .optional(), + identify: z + .unknown() + .describe( + 'Destination-level contact upsert mapping. Resolves to { email, properties }. Fires contact update on first push and re-fires when values change.', + ) + .optional(), + defaultProperties: z + .record(z.string(), z.string()) + .describe( + 'Static event properties added to every event occurrence. Useful for hs_touchpoint_source, hs_page_content_type, etc.', + ) + .optional(), + }) + .strict(); export type Settings = z.infer; diff --git a/packages/server/destinations/hubspot/src/types/index.ts b/packages/server/destinations/hubspot/src/types/index.ts index 0549a3bd8..27f70826c 100644 --- a/packages/server/destinations/hubspot/src/types/index.ts +++ b/packages/server/destinations/hubspot/src/types/index.ts @@ -79,22 +79,9 @@ export interface Settings { */ defaultProperties?: Record; - /** - * Whether to use batch API for events (accumulate and flush). - * Default: false (single event sends). - */ - batch?: boolean; - - /** - * Batch size before auto-flush. Only used when batch: true. - * Default: 50. Max: 500. - */ - batchSize?: number; - /** Runtime state -- not user-facing. Mutated by init/push. */ _client?: HubSpotClientMock; _state?: RuntimeState; - _eventQueue?: HubSpotEventRequest[]; } export interface RuntimeState { @@ -151,6 +138,7 @@ export type Config = { export type InitFn = DestinationServer.InitFn; export type PushFn = DestinationServer.PushFn; +export type PushBatchFn = CoreDestination.PushBatchFn; export type PartialConfig = DestinationServer.PartialConfig; export type PushEvents = DestinationServer.PushEvents; export type Rule = WalkerOSMapping.Rule; diff --git a/packages/server/destinations/kafka/src/__tests__/conformance.test.ts b/packages/server/destinations/kafka/src/__tests__/conformance.test.ts new file mode 100644 index 000000000..8892a54e9 --- /dev/null +++ b/packages/server/destinations/kafka/src/__tests__/conformance.test.ts @@ -0,0 +1,30 @@ +import path from 'path'; +import { conformance } from '@walkeros/collector/dev'; +import { destinationKafka } from '..'; +import { examples, schemas } from '../dev'; + +describe('kafka conformance', () => { + beforeEach(() => { + jest.useRealTimers(); + }); + + it('conforms', async () => { + await conformance('destination', destinationKafka, { + profile: 'socket', + examples: examples.step, + schema: schemas.settings, + // No broker runs here: the runs without env fail their first connect + // fast instead of retrying. + settings: { + kafka: { + brokers: ['localhost:9092'], + topic: 'walkeros-events', + connectionTimeout: 100, + retry: { retries: 0 }, + }, + }, + env: { push: examples.env.push, simulation: examples.env.simulation }, + root: path.join(__dirname, '..', '..'), + }); + }); +}); diff --git a/packages/server/destinations/kafka/src/__tests__/init.test.ts b/packages/server/destinations/kafka/src/__tests__/init.test.ts index 754ae4454..db56ffbd2 100644 --- a/packages/server/destinations/kafka/src/__tests__/init.test.ts +++ b/packages/server/destinations/kafka/src/__tests__/init.test.ts @@ -53,7 +53,7 @@ describe('init', () => { ).rejects.toThrow('topic'); }); - it('creates and connects a producer on valid config', async () => { + it('creates a producer on valid config, without connecting it', async () => { const logger = makeLogger(); const result = await destinationKafka.init!({ config: { diff --git a/packages/server/destinations/kafka/src/__tests__/push-missing-topic.test.ts b/packages/server/destinations/kafka/src/__tests__/push-missing-topic.test.ts index 0c05b9bfb..e87ae87a1 100644 --- a/packages/server/destinations/kafka/src/__tests__/push-missing-topic.test.ts +++ b/packages/server/destinations/kafka/src/__tests__/push-missing-topic.test.ts @@ -1,7 +1,7 @@ // Tests that push() surfaces an actionable error when the broker rejects // the produce call because the topic does not exist (kafkajs throws // KafkaJSProtocolError with code 3 / type UNKNOWN_TOPIC_OR_PARTITION). -// The destination must log a message pointing the operator at +// The push must fail with a message pointing the operator at // `walkeros setup destination.` instead of swallowing the failure. import { createMockContext, createMockLogger, getEvent } from '@walkeros/core'; @@ -48,7 +48,7 @@ function makeConfig(producer: KafkaProducerMock): Config { } describe('push: actionable error when topic missing', () => { - it('logs walkeros setup hint on UNKNOWN_TOPIC_OR_PARTITION', async () => { + it('fails with the walkeros setup hint on UNKNOWN_TOPIC_OR_PARTITION', async () => { const send = jest.fn().mockRejectedValue(makeUnknownTopicError()); const producer = makeProducer(send); const logger = createMockLogger(); @@ -58,23 +58,20 @@ describe('push: actionable error when topic missing', () => { id: 'kafka', }); - await push(getEvent(), context); - - expect(send).toHaveBeenCalledTimes(1); - expect(logger.error).toHaveBeenCalledTimes(1); - - const [message, meta] = logger.error.mock.calls[0]; - expect(typeof message).toBe('string'); - expect(message).toContain('walkeros-events'); - expect(message).toContain('walkeros setup destination.kafka'); - - expect(meta).toMatchObject({ - topic: 'walkeros-events', - brokers: 'kafka1:9092,kafka2:9092', + await expect(push(getEvent(), context)).rejects.toMatchObject({ + name: 'StepError', + code: 'config', + retryable: false, + message: expect.stringContaining( + 'Kafka topic "walkeros-events" not found on cluster kafka1:9092,kafka2:9092', + ), }); + expect(send).toHaveBeenCalledTimes(1); + // The collector writes the one line. + expect(logger.error).not.toHaveBeenCalled(); }); - it('logs the original kafka push failure on unrelated errors', async () => { + it('fails with transport on unrelated errors', async () => { const unrelated = new Error('socket reset'); const send = jest.fn().mockRejectedValue(unrelated); const producer = makeProducer(send); @@ -85,11 +82,46 @@ describe('push: actionable error when topic missing', () => { id: 'kafka', }); + await expect(push(getEvent(), context)).rejects.toMatchObject({ + code: 'transport', + message: 'Kafka push failed', + cause: unrelated, + }); + expect(logger.error).not.toHaveBeenCalled(); + }); + + it('connects once, at the first push', async () => { + const connect = jest.fn().mockResolvedValue(undefined); + const producer = { ...makeProducer(() => Promise.resolve([])), connect }; + const context = createMockContext({ + config: makeConfig(producer), + logger: createMockLogger(), + id: 'kafka', + }); + await push(getEvent(), context); + await push(getEvent(), context); + + expect(connect).toHaveBeenCalledTimes(1); + }); - expect(logger.error).toHaveBeenCalledTimes(1); - const [message, meta] = logger.error.mock.calls[0]; - expect(message).toBe('Kafka push failed'); - expect(meta).toMatchObject({ error: 'socket reset' }); + it('fails a push whose connect fails, and connects again at the next', async () => { + const connect = jest + .fn() + .mockRejectedValueOnce(new Error('ECONNREFUSED')) + .mockResolvedValue(undefined); + const producer = { ...makeProducer(() => Promise.resolve([])), connect }; + const context = createMockContext({ + config: makeConfig(producer), + logger: createMockLogger(), + id: 'kafka', + }); + + await expect(push(getEvent(), context)).rejects.toMatchObject({ + code: 'transport', + message: 'Kafka producer connect failed', + }); + await expect(push(getEvent(), context)).resolves.toBeUndefined(); + expect(connect).toHaveBeenCalledTimes(2); }); }); diff --git a/packages/server/destinations/kafka/src/config.ts b/packages/server/destinations/kafka/src/config.ts index 6ee876187..a7ddb4300 100644 --- a/packages/server/destinations/kafka/src/config.ts +++ b/packages/server/destinations/kafka/src/config.ts @@ -7,8 +7,7 @@ import type { CompressionType, CompressionTypesMap, } from './types'; -import type { Logger } from '@walkeros/core'; -import { isObject } from '@walkeros/core'; +import { isObject, StepError } from '@walkeros/core'; /** Fallback compression codec map when env is not provided. */ const COMPRESSION_FALLBACK: Record = { @@ -19,25 +18,26 @@ const COMPRESSION_FALLBACK: Record = { zstd: 4, }; -export function getConfig( - partialConfig: PartialConfig = {}, - logger: Logger.Instance, -): Config { +export function getConfig(partialConfig: PartialConfig = {}): Config { const raw = (partialConfig.settings ?? {}) as Partial; const kafka: Partial = raw.kafka ?? ({} as Partial); - if (!kafka.brokers || kafka.brokers.length === 0) { - logger.throw('Config settings kafka.brokers missing'); - } - if (!kafka.topic) { - logger.throw('Config settings kafka.topic missing'); + const { brokers, topic } = kafka; + if (!brokers || brokers.length === 0 || !topic) { + const missing = [ + ...(brokers && brokers.length ? [] : ['kafka.brokers']), + ...(topic ? [] : ['kafka.topic']), + ]; + throw new StepError(`Config settings ${missing.join(', ')} missing`, { + code: 'config', + }); } const kafkaSettings: KafkaSettings = { ...kafka, - brokers: kafka.brokers as string[], - topic: kafka.topic as string, + brokers, + topic, clientId: kafka.clientId ?? 'walkeros', acks: kafka.acks ?? -1, compression: kafka.compression ?? 'gzip', diff --git a/packages/server/destinations/kafka/src/examples/step.ts b/packages/server/destinations/kafka/src/examples/step.ts index 97de76084..61a1ab079 100644 --- a/packages/server/destinations/kafka/src/examples/step.ts +++ b/packages/server/destinations/kafka/src/examples/step.ts @@ -203,3 +203,14 @@ export const ignoredEvent: KafkaStepExample = { mapping: { ignore: true }, out: [], }; + +/** + * Silent rule -- mapping.silent: true sends no message. + */ +export const silentRule: KafkaStepExample = { + public: false, + description: 'A rule with silent: true sends no message.', + in: getEvent('page view', { timestamp: 1700000106 }), + mapping: { silent: true }, + out: [], +}; diff --git a/packages/server/destinations/kafka/src/index.ts b/packages/server/destinations/kafka/src/index.ts index dc2e6c5f9..7bb0876ff 100644 --- a/packages/server/destinations/kafka/src/index.ts +++ b/packages/server/destinations/kafka/src/index.ts @@ -7,6 +7,7 @@ import type { KafkaClientConstructor, ProducerConfig, } from './types'; +import { StepError } from '@walkeros/core'; import { getConfig, isKafkaEnv } from './config'; import { push } from './push'; import { setup } from './setup'; @@ -21,8 +22,8 @@ export const destinationKafka: Destination = { setup, - async init({ config: partialConfig, logger, env }) { - const config = getConfig(partialConfig, logger); + async init({ config: partialConfig, env }) { + const config = getConfig(partialConfig); const settings = config.settings as Settings; const kafka = settings.kafka; @@ -46,8 +47,10 @@ export const destinationKafka: Destination = { }; Constructor = kafkajs.Kafka; } catch (err) { - logger.throw(`Failed to load kafkajs: ${String(err)}`); - return config; + throw new StepError('Failed to load kafkajs', { + code: 'not-ready', + cause: err, + }); } } @@ -68,17 +71,8 @@ export const destinationKafka: Destination = { idempotent: kafka.idempotent, }; - const producer = client.producer(producerConfig); - - try { - await producer.connect(); - } catch (err) { - logger.error('Kafka producer connect failed', { error: String(err) }); - logger.throw(`Kafka producer connect failed: ${String(err)}`); - return config; - } - - kafka._producer = producer; + // The producer connects at the first push, so init stays offline. + kafka._producer = client.producer(producerConfig); return config; }, @@ -95,6 +89,7 @@ export const destinationKafka: Destination = { await producer.disconnect(); } finally { settings.kafka._producer = undefined; + settings.kafka._connected = undefined; } } }, diff --git a/packages/server/destinations/kafka/src/push.ts b/packages/server/destinations/kafka/src/push.ts index c0aa5aa2b..3d8e87cdf 100644 --- a/packages/server/destinations/kafka/src/push.ts +++ b/packages/server/destinations/kafka/src/push.ts @@ -7,7 +7,7 @@ import type { KafkaProducerMock, } from './types'; import type { Collector } from '@walkeros/core'; -import { getMappingValue, isObject, isString } from '@walkeros/core'; +import { getMappingValue, isObject, isString, StepError } from '@walkeros/core'; import { getCompressionType } from './config'; const UNKNOWN_TOPIC_CODE = 3; @@ -26,23 +26,44 @@ function isUnknownTopicError(err: unknown): boolean { return err.code === UNKNOWN_TOPIC_CODE || err.type === UNKNOWN_TOPIC_TYPE; } -export const push: PushFn = async function ( +/** + * The producer's connection, opened once at the first push; a failed + * connect fails that push and is tried again at the next one. + */ +function connected( + kafka: KafkaSettings, + producer: KafkaProducerMock, +): Promise { + if (!kafka._connected) { + const connecting = producer.connect().catch((error: unknown) => { + throw new StepError('Kafka producer connect failed', { + code: 'transport', + cause: error, + }); + }); + connecting.catch(() => { + if (kafka._connected === connecting) kafka._connected = undefined; + }); + kafka._connected = connecting; + } + return kafka._connected; +} + +// A silent rule has no message of its own: nothing to send. +export const push: PushFn = async function (event, context) { + if (!context.rule?.silent) await send(event, context); +}; + +const send: PushFn = async function ( event, { config, rule, data, collector, env, logger, id }, ) { - const settings = config.settings as { kafka?: KafkaSettings } | undefined; - const kafka: KafkaSettings | undefined = settings?.kafka; - - if (!kafka) { - logger.warn('Kafka settings missing'); - return; - } - - const producer = kafka._producer; - if (!producer) { - logger.warn('Kafka producer not initialized'); - return; - } + const kafka = config.settings?.kafka; + const producer = kafka?._producer; + if (!kafka || !producer) + throw new StepError('Kafka producer not initialized', { + code: 'not-ready', + }); // Derive event name (rule.name overrides) const eventName = isString(rule?.name) ? rule.name : event.name; @@ -87,31 +108,24 @@ export const push: PushFn = async function ( if (kafka.timeout !== undefined) record.timeout = kafka.timeout; - logger.debug('Kafka push', { topic, key, event: eventName }); + logger.debug('Kafka push', { topic, event: eventName }); + await connected(kafka, producer); try { - await (producer as KafkaProducerMock).send(record); + await producer.send(record); } catch (error) { - if (isUnknownTopicError(error)) { - const brokers = (kafka.brokers ?? []).join(','); - logger.error( - `Kafka topic "${topic}" not found on cluster ${brokers}. ` + + // The collector reports it; the client's own message stays the cause. + if (isUnknownTopicError(error)) + throw new StepError( + `Kafka topic "${topic}" not found on cluster ${kafka.brokers.join(',')}. ` + `Run "walkeros setup destination.${id}" with explicit ` + `{ numPartitions, replicationFactor } to create it.`, - { - topic, - brokers, - event: eventName, - originalError: error instanceof Error ? error.message : String(error), - }, + { code: 'config', cause: error }, ); - } else { - logger.error('Kafka push failed', { - topic, - error: error instanceof Error ? error.message : String(error), - event: eventName, - }); - } + throw new StepError('Kafka push failed', { + code: 'transport', + cause: error, + }); } }; diff --git a/packages/server/destinations/kafka/src/schemas/settings.ts b/packages/server/destinations/kafka/src/schemas/settings.ts index d207d3b7d..8282a2df7 100644 --- a/packages/server/destinations/kafka/src/schemas/settings.ts +++ b/packages/server/destinations/kafka/src/schemas/settings.ts @@ -1,137 +1,147 @@ import { z } from '@walkeros/core/dev'; -const SASLSchema = z.object({ - mechanism: z - .enum(['plain', 'scram-sha-256', 'scram-sha-512', 'aws', 'oauthbearer']) - .describe('SASL authentication mechanism.'), - username: z - .string() - .optional() - .describe('Username for plain/scram mechanisms.'), - password: z - .string() - .optional() - .describe('Password for plain/scram mechanisms.'), - accessKeyId: z - .string() - .optional() - .describe('AWS access key ID for IAM auth (mechanism: aws).'), - secretAccessKey: z - .string() - .optional() - .describe('AWS secret access key for IAM auth (mechanism: aws).'), - sessionToken: z - .string() - .optional() - .describe('AWS session token for temporary credentials (mechanism: aws).'), - authorizationIdentity: z - .string() - .optional() - .describe('AWS authorization identity (mechanism: aws).'), -}); +const SASLSchema = z + .object({ + mechanism: z + .enum(['plain', 'scram-sha-256', 'scram-sha-512', 'aws', 'oauthbearer']) + .describe('SASL authentication mechanism.'), + username: z + .string() + .optional() + .describe('Username for plain/scram mechanisms.'), + password: z + .string() + .optional() + .describe('Password for plain/scram mechanisms.'), + accessKeyId: z + .string() + .optional() + .describe('AWS access key ID for IAM auth (mechanism: aws).'), + secretAccessKey: z + .string() + .optional() + .describe('AWS secret access key for IAM auth (mechanism: aws).'), + sessionToken: z + .string() + .optional() + .describe( + 'AWS session token for temporary credentials (mechanism: aws).', + ), + authorizationIdentity: z + .string() + .optional() + .describe('AWS authorization identity (mechanism: aws).'), + }) + .strict(); -const RetrySchema = z.object({ - maxRetryTime: z - .number() - .int() - .positive() - .optional() - .describe('Max total retry wait in ms. Default: 30000.'), - initialRetryTime: z - .number() - .int() - .positive() - .optional() - .describe('First retry delay in ms. Default: 300.'), - retries: z - .number() - .int() - .min(0) - .optional() - .describe('Max retry count. Default: 5.'), -}); +const RetrySchema = z + .object({ + maxRetryTime: z + .number() + .int() + .positive() + .optional() + .describe('Max total retry wait in ms. Default: 30000.'), + initialRetryTime: z + .number() + .int() + .positive() + .optional() + .describe('First retry delay in ms. Default: 300.'), + retries: z + .number() + .int() + .min(0) + .optional() + .describe('Max retry count. Default: 5.'), + }) + .strict(); -export const KafkaSettingsSchema = z.object({ - brokers: z - .array(z.string().min(1)) - .min(1) - .describe('Kafka broker addresses (host:port). At least one required.'), - clientId: z - .string() - .optional() - .describe('Kafka client ID. Default: walkeros.'), - ssl: z - .union([z.boolean(), z.record(z.string(), z.unknown())]) - .optional() - .describe( - 'TLS configuration. Set true for default TLS, or provide a tls.ConnectionOptions object for mTLS.', +export const KafkaSettingsSchema = z + .object({ + brokers: z + .array(z.string().min(1)) + .min(1) + .describe('Kafka broker addresses (host:port). At least one required.'), + clientId: z + .string() + .optional() + .describe('Kafka client ID. Default: walkeros.'), + ssl: z + .union([z.boolean(), z.record(z.string(), z.unknown())]) + .optional() + .describe( + 'TLS configuration. Set true for default TLS, or provide a tls.ConnectionOptions object for mTLS.', + ), + sasl: SASLSchema.optional().describe( + 'SASL authentication config. Required for Confluent Cloud, AWS MSK with IAM, etc.', ), - sasl: SASLSchema.optional().describe( - 'SASL authentication config. Required for Confluent Cloud, AWS MSK with IAM, etc.', - ), - connectionTimeout: z - .number() - .int() - .positive() - .optional() - .describe('Connection timeout in ms. Default: 1000.'), - requestTimeout: z - .number() - .int() - .positive() - .optional() - .describe('Request timeout in ms. Default: 30000.'), - topic: z.string().min(1).describe('Target Kafka topic name.'), - acks: z - .number() - .int() - .min(-1) - .max(1) - .optional() - .describe( - 'Acknowledgement level. -1 = all replicas, 0 = fire-and-forget, 1 = leader only. Default: -1.', + connectionTimeout: z + .number() + .int() + .positive() + .optional() + .describe('Connection timeout in ms. Default: 1000.'), + requestTimeout: z + .number() + .int() + .positive() + .optional() + .describe('Request timeout in ms. Default: 30000.'), + topic: z.string().min(1).describe('Target Kafka topic name.'), + acks: z + .number() + .int() + .min(-1) + .max(1) + .optional() + .describe( + 'Acknowledgement level. -1 = all replicas, 0 = fire-and-forget, 1 = leader only. Default: -1.', + ), + timeout: z + .number() + .int() + .positive() + .optional() + .describe('Broker response timeout in ms. Default: 30000.'), + compression: z + .enum(['none', 'gzip', 'snappy', 'lz4', 'zstd']) + .optional() + .describe( + 'Message compression codec. Default: gzip. Snappy/LZ4/ZSTD require additional npm packages.', + ), + idempotent: z + .boolean() + .optional() + .describe( + 'Enable idempotent producer for exactly-once delivery. Default: false.', + ), + allowAutoTopicCreation: z + .boolean() + .optional() + .describe('Allow auto-creation of topics on the broker. Default: false.'), + key: z + .string() + .optional() + .describe( + 'Mapping value path for message key derivation (e.g. user.id, data.userId). Default: entity_action.', + ), + headers: z + .record(z.string(), z.string()) + .optional() + .describe('Static headers added to every message.'), + retry: RetrySchema.optional().describe( + 'Retry configuration for transient failures.', ), - timeout: z - .number() - .int() - .positive() - .optional() - .describe('Broker response timeout in ms. Default: 30000.'), - compression: z - .enum(['none', 'gzip', 'snappy', 'lz4', 'zstd']) - .optional() - .describe( - 'Message compression codec. Default: gzip. Snappy/LZ4/ZSTD require additional npm packages.', - ), - idempotent: z - .boolean() - .optional() - .describe( - 'Enable idempotent producer for exactly-once delivery. Default: false.', - ), - allowAutoTopicCreation: z - .boolean() - .optional() - .describe('Allow auto-creation of topics on the broker. Default: false.'), - key: z - .string() - .optional() - .describe( - 'Mapping value path for message key derivation (e.g. user.id, data.userId). Default: entity_action.', - ), - headers: z - .record(z.string(), z.string()) - .optional() - .describe('Static headers added to every message.'), - retry: RetrySchema.optional().describe( - 'Retry configuration for transient failures.', - ), -}); + }) + .strict(); -export const SettingsSchema = z.object({ - kafka: KafkaSettingsSchema.describe( - 'Kafka connection and producer settings.', - ), -}); +export const SettingsSchema = z + .object({ + kafka: KafkaSettingsSchema.describe( + 'Kafka connection and producer settings.', + ), + }) + .strict(); export type Settings = z.infer; diff --git a/packages/server/destinations/kafka/src/types/index.ts b/packages/server/destinations/kafka/src/types/index.ts index c539c9e96..a96851834 100644 --- a/packages/server/destinations/kafka/src/types/index.ts +++ b/packages/server/destinations/kafka/src/types/index.ts @@ -160,6 +160,8 @@ export interface KafkaSettings { // Runtime -- set during init, not user-facing _producer?: KafkaProducerMock; + /** The producer's connection, opened at the first push. */ + _connected?: Promise; } export interface Settings { diff --git a/packages/server/destinations/pinterest/src/__tests__/conformance.test.ts b/packages/server/destinations/pinterest/src/__tests__/conformance.test.ts new file mode 100644 index 000000000..b1b088741 --- /dev/null +++ b/packages/server/destinations/pinterest/src/__tests__/conformance.test.ts @@ -0,0 +1,32 @@ +import path from 'path'; +import { conformance, runExamples } from '@walkeros/collector/dev'; +import { destinationPinterest } from '..'; +import { examples, schemas } from '../dev'; + +const settings = { accessToken: 's3cr3t', adAccountId: '123456789' }; + +describe('pinterest conformance', () => { + beforeEach(() => { + jest.useRealTimers(); + }); + + it('conforms', async () => { + await conformance('destination', destinationPinterest, { + profile: 'http', + examples: examples.step, + schema: schemas.settings, + settings, + root: path.join(__dirname, '..', '..'), + }); + }); + + it('fails an event Pinterest rejects in its HTTP 200 body', async () => { + const { rejectedEvent } = examples.step; + await runExamples( + 'destination', + destinationPinterest, + { rejectedEvent }, + { settings }, + ); + }); +}); diff --git a/packages/server/destinations/pinterest/src/__tests__/index.test.ts b/packages/server/destinations/pinterest/src/__tests__/index.test.ts index cd1cee2ef..fce6c66a6 100644 --- a/packages/server/destinations/pinterest/src/__tests__/index.test.ts +++ b/packages/server/destinations/pinterest/src/__tests__/index.test.ts @@ -61,7 +61,7 @@ describe('Server Destination Pinterest', () => { logger: createMockLogger(), id: 'test-pinterest', }), - ).rejects.toThrow('Config settings accessToken missing'); + ).rejects.toThrow('Config settings accessToken, adAccountId missing'); await expect( destination.init({ config: { settings: { accessToken, adAccountId: '' } }, diff --git a/packages/server/destinations/pinterest/src/__tests__/stepExamples.test.ts b/packages/server/destinations/pinterest/src/__tests__/stepExamples.test.ts index ab3a4f950..6d9d2d97c 100644 --- a/packages/server/destinations/pinterest/src/__tests__/stepExamples.test.ts +++ b/packages/server/destinations/pinterest/src/__tests__/stepExamples.test.ts @@ -25,7 +25,12 @@ describe('Step Examples', () => { }); }); - it.each(Object.entries(examples.step))('%s', async (_name, example) => { + // Non-public examples (silent, failures) run in the conformance test. + const published = Object.entries(examples.step).filter( + ([, example]) => example.public !== false, + ); + + it.each(published)('%s', async (_name, example) => { const event = example.in as WalkerOS.Event; const mapping = example.mapping; diff --git a/packages/server/destinations/pinterest/src/config.ts b/packages/server/destinations/pinterest/src/config.ts index 5b0c27080..a59985f41 100644 --- a/packages/server/destinations/pinterest/src/config.ts +++ b/packages/server/destinations/pinterest/src/config.ts @@ -1,15 +1,19 @@ import type { Config, Settings, PartialConfig } from './types'; -import type { Logger } from '@walkeros/core'; +import { StepError } from '@walkeros/core'; -export function getConfig( - partialConfig: PartialConfig = {}, - logger: Logger.Instance, -): Config { - const settings = (partialConfig.settings || {}) as Partial; +export function getConfig(partialConfig: PartialConfig = {}): Config { + const settings: Partial = partialConfig.settings || {}; const { accessToken, adAccountId } = settings; - if (!accessToken) logger.throw('Config settings accessToken missing'); - if (!adAccountId) logger.throw('Config settings adAccountId missing'); + if (!accessToken || !adAccountId) { + const missing = [ + ...(accessToken ? [] : ['accessToken']), + ...(adAccountId ? [] : ['adAccountId']), + ]; + throw new StepError(`Config settings ${missing.join(', ')} missing`, { + code: 'config', + }); + } const settingsConfig: Settings = { ...settings, diff --git a/packages/server/destinations/pinterest/src/examples/step.ts b/packages/server/destinations/pinterest/src/examples/step.ts index 80b1f72eb..17f10f4e5 100644 --- a/packages/server/destinations/pinterest/src/examples/step.ts +++ b/packages/server/destinations/pinterest/src/examples/step.ts @@ -347,3 +347,35 @@ export const signup: Flow.StepExample = { ], ], }; + +/** A silent rule sends nothing to Pinterest. */ +export const silentRule: Flow.StepExample = { + public: false, + description: 'A rule with silent: true sends no request to Pinterest.', + in: getEvent('page view', { + id: 'a1b2c3d4e5f60006', + timestamp: 1700000905, + }), + mapping: { silent: true }, + out: [], +}; + +/** Pinterest rejects the event in an HTTP 200 body. */ +export const rejectedEvent: Flow.StepExample = { + public: false, + description: + 'Pinterest answers HTTP 200 and reports the event as failed: the push fails with transport, not retryable, and the event goes to the DLQ.', + in: getEvent('page view', { + id: 'a1b2c3d4e5f60007', + timestamp: 1700000906, + }), + respond: { + status: 200, + body: { + num_events_received: 1, + num_events_processed: 0, + events: [{ status: 'failed', error_message: 'Invalid event_time' }], + }, + }, + outcome: { status: 'failed', code: 'transport', retryable: false }, +}; diff --git a/packages/server/destinations/pinterest/src/index.ts b/packages/server/destinations/pinterest/src/index.ts index f5a7b94bd..08a3aebf6 100644 --- a/packages/server/destinations/pinterest/src/index.ts +++ b/packages/server/destinations/pinterest/src/index.ts @@ -10,9 +10,8 @@ export const destinationPinterest: Destination = { config: {}, - async init({ config: partialConfig, logger }) { - const config = getConfig(partialConfig, logger); - return config; + async init({ config: partialConfig }) { + return getConfig(partialConfig); }, async push(event, context) { diff --git a/packages/server/destinations/pinterest/src/push.ts b/packages/server/destinations/pinterest/src/push.ts index ee36d503c..d35566828 100644 --- a/packages/server/destinations/pinterest/src/push.ts +++ b/packages/server/destinations/pinterest/src/push.ts @@ -1,10 +1,20 @@ import type { ConversionEvent, PushFn, RequestBody, UserData } from './types'; import type { Collector, Mapping as WalkerOSMapping } from '@walkeros/core'; -import { createMappingRoot, getMappingValue, isObject } from '@walkeros/core'; +import { + createMappingRoot, + getMappingValue, + isObject, + StepError, +} from '@walkeros/core'; import { sendServer } from '@walkeros/server-core'; import { hashEvent } from './hash'; -export const push: PushFn = async function ( +// A silent rule has no Pinterest call of its own: nothing to send. +export const push: PushFn = async function (event, context) { + if (!context.rule?.silent) await send(event, context); +}; + +const send: PushFn = async function ( event, { config, data, ingest, env, logger, collector }, ) { @@ -102,11 +112,30 @@ export const push: PushFn = async function ( }); const sendServerFn = env?.sendServer || sendServer; - await sendServerFn(endpoint, JSON.stringify(body), { + const response = await sendServerFn(endpoint, JSON.stringify(body), { headers: { Authorization: `Bearer ${accessToken}` }, }); + checkResponse(response.data); }; +/** + * Pinterest answers HTTP 200 with a per-event `status` and reports an event + * it did not process as `failed` (with its `error_message`). A rejected + * event fails the push, not retryable; the body stays on the error. + */ +function checkResponse(data: unknown): void { + if (!isObject(data) || !Array.isArray(data.events)) return; + const failed = data.events.filter( + (result) => isObject(result) && result.status === 'failed', + ).length; + if (!failed) return; + throw new StepError('Pinterest Conversions API rejected the event', { + code: 'transport', + retryable: false, + body: data, + }); +} + const DEFAULT_IP: WalkerOSMapping.Value = ['ingest.ip', 'event.user.ip']; const DEFAULT_USER_AGENT: WalkerOSMapping.Value = [ 'ingest.userAgent', diff --git a/packages/server/destinations/pinterest/src/schemas/settings.ts b/packages/server/destinations/pinterest/src/schemas/settings.ts index c48507512..08273501a 100644 --- a/packages/server/destinations/pinterest/src/schemas/settings.ts +++ b/packages/server/destinations/pinterest/src/schemas/settings.ts @@ -6,55 +6,57 @@ const InputSchema = z.union([ z.literal(false), ]); -export const SettingsSchema = z.object({ - accessToken: z - .string() - .min(1) - .describe( - 'Pinterest conversion access token for Bearer authentication (like pina_ABC123...)', - ), - adAccountId: z - .string() - .regex(/^[0-9]+$/, 'Ad account ID must contain only digits') - .describe('Pinterest ad account ID from Ads Manager (like 123456789)'), - action_source: ActionSourceSchema.describe( - 'Source of the event (web, app_android, app_ios, offline) (like web)', - ).optional(), - doNotHash: z - .array(z.string()) - .describe( - "Array of user_data fields that should not be hashed (like ['external_id'])", - ) - .optional(), - test: z - .boolean() - .describe( - 'Enable test mode by appending ?test=true to the API URL (like true)', - ) - .optional(), - url: z - .string() - .url() - .describe( - 'Custom URL for Pinterest Conversions API endpoint (like https://api.pinterest.com/v5/)', - ) - .optional(), - user_data: z - .record(z.string(), z.string()) - .describe( - "Mapping configuration for user data fields (like { em: 'user.email', ph: 'user.phone' })", - ) - .optional(), - partner_name: z - .string() - .describe('Third-party partner name for attribution (like ss-walkeros)') - .optional(), - ip: InputSchema.describe( - 'Client IP, sent as user_data.client_ip_address. Resolves against { ingest, event }. Default: ["ingest.ip", "event.user.ip"]. false switches it off. A mapped user_data.client_ip_address wins.', - ).optional(), - userAgent: InputSchema.describe( - 'Client user agent, sent as user_data.client_user_agent. Resolves against { ingest, event }. Default: ["ingest.userAgent", "event.user.userAgent"]. false switches it off. A mapped user_data.client_user_agent wins.', - ).optional(), -}); +export const SettingsSchema = z + .object({ + accessToken: z + .string() + .min(1) + .describe( + 'Pinterest conversion access token for Bearer authentication (like pina_ABC123...)', + ), + adAccountId: z + .string() + .regex(/^[0-9]+$/, 'Ad account ID must contain only digits') + .describe('Pinterest ad account ID from Ads Manager (like 123456789)'), + action_source: ActionSourceSchema.describe( + 'Source of the event (web, app_android, app_ios, offline) (like web)', + ).optional(), + doNotHash: z + .array(z.string()) + .describe( + "Array of user_data fields that should not be hashed (like ['external_id'])", + ) + .optional(), + test: z + .boolean() + .describe( + 'Enable test mode by appending ?test=true to the API URL (like true)', + ) + .optional(), + url: z + .string() + .url() + .describe( + 'Custom URL for Pinterest Conversions API endpoint (like https://api.pinterest.com/v5/)', + ) + .optional(), + user_data: z + .record(z.string(), z.string()) + .describe( + "Mapping configuration for user data fields (like { em: 'user.email', ph: 'user.phone' })", + ) + .optional(), + partner_name: z + .string() + .describe('Third-party partner name for attribution (like ss-walkeros)') + .optional(), + ip: InputSchema.describe( + 'Client IP, sent as user_data.client_ip_address. Resolves against { ingest, event }. Default: ["ingest.ip", "event.user.ip"]. false switches it off. A mapped user_data.client_ip_address wins.', + ).optional(), + userAgent: InputSchema.describe( + 'Client user agent, sent as user_data.client_user_agent. Resolves against { ingest, event }. Default: ["ingest.userAgent", "event.user.userAgent"]. false switches it off. A mapped user_data.client_user_agent wins.', + ).optional(), + }) + .strict(); export type Settings = z.infer; diff --git a/packages/server/destinations/posthog/src/__tests__/conformance.test.ts b/packages/server/destinations/posthog/src/__tests__/conformance.test.ts new file mode 100644 index 000000000..d1052c2d3 --- /dev/null +++ b/packages/server/destinations/posthog/src/__tests__/conformance.test.ts @@ -0,0 +1,35 @@ +import path from 'path'; +import { conformance } from '@walkeros/collector/dev'; +import { destinationPostHog } from '..'; +import { examples, schemas } from '../dev'; + +// posthog-node is ESM-first and Jest cannot parse it: the runs without env +// get a stand-in client whose calls all land. +jest.mock('posthog-node', () => ({ + PostHog: class { + capture = () => undefined; + identify = () => undefined; + groupIdentify = () => undefined; + flush = () => Promise.resolve(); + shutdown = () => Promise.resolve(); + enable = () => undefined; + disable = () => undefined; + }, +})); + +describe('posthog conformance', () => { + beforeEach(() => { + jest.useRealTimers(); + }); + + it('conforms', async () => { + await conformance('destination', destinationPostHog, { + profile: 'sdk-env', + examples: examples.step, + schema: schemas.settings, + settings: { apiKey: 'phc_test' }, + env: { push: examples.env.push, simulation: examples.env.simulation }, + root: path.join(__dirname, '..', '..'), + }); + }); +}); diff --git a/packages/server/destinations/posthog/src/__tests__/failure.test.ts b/packages/server/destinations/posthog/src/__tests__/failure.test.ts new file mode 100644 index 000000000..10a875574 --- /dev/null +++ b/packages/server/destinations/posthog/src/__tests__/failure.test.ts @@ -0,0 +1,121 @@ +jest.mock('posthog-node', () => ({ PostHog: class {} })); + +import { startFlow } from '@walkeros/collector'; +import { createEvent, getEvent } from '@walkeros/core'; +import { destinationPostHog } from '..'; +import type { Env } from '../types'; + +interface Failing { + /** Captures the client refuses to queue (an invalid call). */ + capture?: (distinctId: unknown) => boolean; + /** Fails every flush the way posthog-node does. */ + flush?: boolean; +} + +/** + * A client that fails the way posthog-node does: a failed request is an + * `error` event plus a rejected flush. + */ +function failingEnv(failing: Failing): Env { + return { + PostHog: class { + private listeners: Array<(error: unknown) => void> = []; + on(_event: 'error', listener: (error: unknown) => void) { + this.listeners.push(listener); + return () => { + this.listeners = this.listeners.filter((l) => l !== listener); + }; + } + capture({ distinctId }: { distinctId?: unknown }) { + if (failing.capture?.(distinctId)) + throw new Error('The capture call was refused'); + } + identify() {} + groupIdentify() {} + async flush() { + if (!failing.flush) return; + const error = new Error('HTTP error while fetching PostHog: 503'); + for (const listener of this.listeners) listener(error); + throw error; + } + async shutdown() {} + enable() {} + disable() {} + }, + }; +} + +describe('posthog send failures', () => { + it('fails a single push (batching off) when its flush fails', async () => { + const { collector } = await startFlow({ + logger: { level: 'ERROR', handler: () => undefined }, + destinations: { + posthog: { + code: destinationPostHog, + config: { settings: { apiKey: 'phc_test' }, batch: undefined }, + env: failingEnv({ flush: true }), + }, + }, + }); + + const result = await collector.push(getEvent('page view')); + + expect(result.ok).toBe(false); + expect(result.failed?.posthog?.reason).toMatchObject({ + code: 'transport', + message: 'PostHog send failed', + }); + expect(collector.status.destinations.posthog.failed).toBe(1); + expect(collector.destinations.posthog.dlq).toHaveLength(1); + + await collector.command('shutdown'); + }); + + it('gives each entry of a batch its own outcome', async () => { + const { collector } = await startFlow({ + logger: { level: 'ERROR', handler: () => undefined }, + destinations: { + posthog: { + code: destinationPostHog, + config: { settings: { apiKey: 'phc_test' } }, + env: failingEnv({ capture: (distinctId) => distinctId === 'bad' }), + }, + }, + }); + + for (const id of ['a', 'bad', 'b']) + await collector.push(createEvent({ name: 'page view', user: { id } })); + await collector.command('flush'); + + const status = collector.status.destinations.posthog; + expect(status.count).toBe(2); + expect(status.failed).toBe(1); + expect(collector.destinations.posthog.dlq).toHaveLength(1); + + await collector.command('shutdown'); + }); + + it('fails the whole batch when its flush fails', async () => { + const { collector } = await startFlow({ + logger: { level: 'ERROR', handler: () => undefined }, + destinations: { + posthog: { + code: destinationPostHog, + config: { settings: { apiKey: 'phc_test' } }, + env: failingEnv({ flush: true }), + }, + }, + }); + + for (const id of ['a', 'b', 'c']) + await collector.push(createEvent({ name: 'page view', user: { id } })); + await collector.command('flush'); + + const status = collector.status.destinations.posthog; + expect(status.count).toBe(0); + expect(status.failed).toBe(3); + expect(collector.destinations.posthog.dlq).toHaveLength(3); + + await collector.command('shutdown'); + }); +}); diff --git a/packages/server/destinations/posthog/src/__tests__/stepExamples.test.ts b/packages/server/destinations/posthog/src/__tests__/stepExamples.test.ts index 40b1da9ef..416455e07 100644 --- a/packages/server/destinations/posthog/src/__tests__/stepExamples.test.ts +++ b/packages/server/destinations/posthog/src/__tests__/stepExamples.test.ts @@ -107,6 +107,8 @@ describe('posthog server destination — step examples', () => { }, }); await elb(event); + // The destination batches by default: send what it holds. + await elb('walker flush'); } expect(collected()).toEqual(example.out); diff --git a/packages/server/destinations/posthog/src/config.ts b/packages/server/destinations/posthog/src/config.ts index 553fc6beb..81e0d9bd5 100644 --- a/packages/server/destinations/posthog/src/config.ts +++ b/packages/server/destinations/posthog/src/config.ts @@ -1,16 +1,16 @@ import type { Config, Env, PartialConfig, Settings } from './types'; -import type { Logger } from '@walkeros/core'; +import { StepError } from '@walkeros/core'; import { PostHog } from 'posthog-node'; export function getConfig( partialConfig: PartialConfig = {}, env: Env | undefined, - logger: Logger.Instance, ): Config { - const settings = (partialConfig.settings || {}) as Partial; + const settings: Partial = partialConfig.settings || {}; const { apiKey } = settings; - if (!apiKey) logger.throw('Config settings apiKey missing'); + if (!apiKey) + throw new StepError('Config settings apiKey missing', { code: 'config' }); // Extract PostHog SDK options from settings const options: Record = {}; diff --git a/packages/server/destinations/posthog/src/examples/step.ts b/packages/server/destinations/posthog/src/examples/step.ts index 43876b1e0..390760858 100644 --- a/packages/server/destinations/posthog/src/examples/step.ts +++ b/packages/server/destinations/posthog/src/examples/step.ts @@ -237,3 +237,32 @@ export const consentGrant: PostHogStepExample = { settings: {} as Partial, out: [['client.enable']], }; + +/** A silent rule without identify or group mappings calls nothing. */ +export const silentRule: PostHogStepExample = { + public: false, + description: + 'A rule with silent: true and no identify or group mapping sends nothing to PostHog.', + in: getEvent('page view', { timestamp: 1700000105 }), + mapping: { silent: true }, + out: [], +}; + +/** + * The silent group identify rule of groupIdentifyWithProperties on an event + * without data: no group key resolves and the rule suppresses the capture, + * so nothing is sent. + */ +export const groupIdentifyWithoutData: PostHogStepExample = { + public: false, + description: + 'A silent group rule whose key finds no data sends nothing: an unmet mapping, not a failure, so the push returns normally.', + in: { + name: 'company update', + entity: 'company', + action: 'update', + timestamp: 1700000106, + }, + mapping: groupIdentifyWithProperties.mapping, + out: [], +}; diff --git a/packages/server/destinations/posthog/src/index.ts b/packages/server/destinations/posthog/src/index.ts index 515358672..81c01f3e7 100644 --- a/packages/server/destinations/posthog/src/index.ts +++ b/packages/server/destinations/posthog/src/index.ts @@ -1,7 +1,7 @@ import type { WalkerOS } from '@walkeros/core'; import type { Destination, Env, Settings } from './types'; import { getConfig } from './config'; -import { push } from './push'; +import { push, pushBatch } from './push'; // Types export * as DestinationPostHog from './types'; @@ -9,16 +9,16 @@ export * as DestinationPostHog from './types'; export const destinationPostHog: Destination = { type: 'posthog', - config: {}, + // Batched by default, at the SDK's own former defaults (20 events, 10 s). + config: { batch: { size: 20, wait: 10_000 } }, - async init({ config: partialConfig, env, logger }) { - const config = getConfig(partialConfig, env as Env | undefined, logger); - return config; + async init({ config: partialConfig, env }) { + return getConfig(partialConfig, env); }, - async push(event, context) { - return await push(event, context); - }, + push, + + pushBatch, async destroy({ config }) { const client = (config?.settings as Settings | undefined)?.client; diff --git a/packages/server/destinations/posthog/src/push.ts b/packages/server/destinations/posthog/src/push.ts index 412fd6528..b37f1e64d 100644 --- a/packages/server/destinations/posthog/src/push.ts +++ b/packages/server/destinations/posthog/src/push.ts @@ -1,14 +1,128 @@ -import type { PushFn } from './types'; -import { getMappingValue, isObject, isString } from '@walkeros/core'; +import type { Collector, Destination, WalkerOS } from '@walkeros/core'; +import type { + PostHogClient, + PushBatchFn, + PushFn, + Rule, + Settings, +} from './types'; +import { getMappingValue, isObject, isString, StepError } from '@walkeros/core'; +/** + * Run one flush to its end. The SDK reports a failed send as an `error` + * event and a rejected flush; the call listens for the event while it runs. + * Sends never overlap (`exclusive`), so the event belongs to this flush. + */ +async function call( + client: PostHogClient, + message: string, + send: () => unknown, +): Promise { + let failure: unknown; + const off = client.on?.('error', (error) => { + failure = failure ?? error; + }); + try { + await send(); + } catch (error) { + failure = failure ?? error; + } finally { + off?.(); + } + if (failure !== undefined) + throw new StepError(message, { code: 'transport', cause: failure }); +} + +/** Run one queueing call; an injected client may answer with a promise. */ +async function queue(message: string, send: () => unknown): Promise { + try { + await send(); + } catch (error) { + throw new StepError(message, { code: 'transport', cause: error }); + } +} + +/** + * One send at a time per client: its calls queued, then one flush. A flush + * sends the whole queue, so a send that overlapped another could be carried, + * and failed, by the other's request without hearing of it. + */ +function exclusive( + settings: Partial, + task: () => Promise, +): Promise { + const running = (settings._sending ?? Promise.resolve()).then(task); + settings._sending = running.catch(() => undefined); + return running; +} + +/** The settings and client init resolved, or a StepError. */ +function clientOf(settings: Partial | undefined): { + settings: Partial; + client: PostHogClient; +} { + const client = settings?.client; + if (!settings || !client) + throw new StepError('PostHog client not initialized', { + code: 'not-ready', + }); + return { settings, client }; +} + +/** One event: its calls queued and sent by one flush. */ export const push: PushFn = async function ( event, { config, rule, data, collector }, ) { - const settings = config.settings!; - const client = settings.client; - if (!client) return; + const { settings, client } = clientOf(config.settings); + await exclusive(settings, async () => { + await send(event, rule, data, settings, client, collector); + await call(client, 'PostHog send failed', () => client.flush()); + }); +}; +/** + * A flushed `config.batch` batch: every entry's calls queued, then one flush + * sends them. An entry whose calls could not be queued fails alone; a failed + * flush fails the batch, since PostHog answers a request as a whole. + */ +export const pushBatch: PushBatchFn = async function ( + batch, + { config, collector }, +) { + const { settings, client } = clientOf(config.settings); + const failed: Destination.BatchFailure[] = []; + + await exclusive(settings, async () => { + for (const [index, entry] of batch.entries.entries()) + await send( + entry.event, + entry.rule, + entry.data, + settings, + client, + collector, + ).catch((error: unknown) => failed.push({ index, error })); + + if (failed.length < batch.entries.length) + await call(client, 'PostHog send failed', () => client.flush()); + }); + + // Nothing landed: a whole-batch failure, which the breaker counts. + if (failed.length && failed.length === batch.entries.length) + throw failed[0].error; + if (failed.length) return { failed }; +}; + +/** The calls of one event (identify, group identify, capture). */ +async function send( + event: WalkerOS.Event, + rule: Rule | undefined, + data: Destination.Data, + settings: Partial, + client: PostHogClient, + collector: Collector.Instance, +): Promise { const mappingSettings = (rule?.settings || {}) as Record; // Resolve distinctId: rule-level identify wins over destination-level @@ -50,7 +164,9 @@ export const push: PushFn = async function ( const properties: Record = {}; if ($set) properties.$set = $set; if ($setOnce) properties.$set_once = $setOnce; - client.identify({ distinctId, properties }); + await queue('PostHog identify failed', () => + client.identify({ distinctId, properties }), + ); } } @@ -72,13 +188,15 @@ export const push: PushFn = async function ( if (isString(type) && (isString(key) || typeof key === 'number')) { // If properties present, call groupIdentify - if (properties) { - client.groupIdentify({ - groupType: type, - groupKey: String(key), - properties, - }); - } + // No immediate variant: queued, then flushed in this push. + if (properties) + await queue('PostHog group identify failed', () => + client.groupIdentify({ + groupType: type, + groupKey: String(key), + properties, + }), + ); // Build groups object for capture groups = { [type]: key as string | number }; } @@ -97,6 +215,8 @@ export const push: PushFn = async function ( }; if (groups) captureArgs.groups = groups; - client.capture(captureArgs as Parameters[0]); + await queue('PostHog capture failed', () => + client.capture(captureArgs as Parameters[0]), + ); } -}; +} diff --git a/packages/server/destinations/posthog/src/schemas/settings.ts b/packages/server/destinations/posthog/src/schemas/settings.ts index d5b048e30..799b54698 100644 --- a/packages/server/destinations/posthog/src/schemas/settings.ts +++ b/packages/server/destinations/posthog/src/schemas/settings.ts @@ -1,62 +1,86 @@ import { z } from '@walkeros/core/dev'; -export const SettingsSchema = z.object({ - apiKey: z - .string() - .min(1) - .describe( - 'PostHog project API key (starts with "phc_"). Find it in PostHog project settings.', - ), - host: z - .string() - .describe( - 'PostHog API host. Defaults to https://us.i.posthog.com. Use https://eu.i.posthog.com for EU or your self-hosted URL.', - ) - .optional(), - flushAt: z - .number() - .describe('Number of events queued before auto-flush. Default: 20.') - .optional(), - flushInterval: z - .number() - .describe('Milliseconds between periodic flushes. Default: 10000.') - .optional(), - personalApiKey: z - .string() - .describe('Personal API key (phx_...) for local feature flag evaluation.') - .optional(), - featureFlagsPollingInterval: z - .number() - .describe( - 'Milliseconds between feature flag definition polls. Default: 30000.', - ) - .optional(), - disableGeoip: z - .boolean() - .describe('Disable GeoIP lookups globally. Useful for GDPR compliance.') - .optional(), - debug: z - .boolean() - .describe('Enable PostHog SDK debug logging. Default: false.') - .optional(), - identify: z - .unknown() - .describe( - 'walkerOS mapping value resolving to an identity object. Keys: distinctId, $set, $set_once. Resolved on every push (server is stateless).', - ) - .optional(), - group: z - .unknown() - .describe( - 'walkerOS mapping value resolving to a group object. Keys: type, key, properties. Resolved on every push.', - ) - .optional(), - include: z - .array(z.string()) - .describe( - 'Event sections to flatten into capture() properties (e.g. ["data", "globals"]).', - ) - .optional(), -}); +export const SettingsSchema = z + .object({ + apiKey: z + .string() + .min(1) + .describe( + 'PostHog project API key (starts with "phc_"). Find it in PostHog project settings.', + ), + host: z + .string() + .describe( + 'PostHog API host. Defaults to https://us.i.posthog.com. Use https://eu.i.posthog.com for EU or your self-hosted URL.', + ) + .optional(), + flushAt: z + .number() + .describe( + 'Number of queued group identifies before auto-flush. Captures and identifies are sent at once, each push awaiting its own call. Default: 20.', + ) + .optional(), + flushInterval: z + .number() + .describe( + 'Milliseconds between periodic flushes of queued group identifies. Default: 10000.', + ) + .optional(), + personalApiKey: z + .string() + .describe('Personal API key (phx_...) for local feature flag evaluation.') + .optional(), + featureFlagsPollingInterval: z + .number() + .describe( + 'Milliseconds between feature flag definition polls. Default: 30000.', + ) + .optional(), + disableGeoip: z + .boolean() + .describe('Disable GeoIP lookups globally. Useful for GDPR compliance.') + .optional(), + disableCompression: z + .boolean() + .describe('Send requests uncompressed (no gzip). Default: false.') + .optional(), + requestTimeout: z + .number() + .describe( + 'Milliseconds before a request to PostHog times out. Default: 10000.', + ) + .optional(), + fetchRetryCount: z + .number() + .describe('Retries of a failed request to PostHog. Default: 3.') + .optional(), + fetchRetryDelay: z + .number() + .describe( + 'Milliseconds between retries of a failed request. Default: 3000.', + ) + .optional(), + debug: z + .boolean() + .describe('Enable PostHog SDK debug logging. Default: false.') + .optional(), + disabled: z + .boolean() + .describe('Disable the PostHog client: nothing is sent. Default: false.') + .optional(), + identify: z + .unknown() + .describe( + 'walkerOS mapping value resolving to an identity object. Keys: distinctId, $set, $set_once. Resolved on every push (server is stateless).', + ) + .optional(), + group: z + .unknown() + .describe( + 'walkerOS mapping value resolving to a group object. Keys: type, key, properties. Resolved on every push.', + ) + .optional(), + }) + .strict(); export type Settings = z.infer; diff --git a/packages/server/destinations/posthog/src/types/index.ts b/packages/server/destinations/posthog/src/types/index.ts index f2e674b75..924d0c817 100644 --- a/packages/server/destinations/posthog/src/types/index.ts +++ b/packages/server/destinations/posthog/src/types/index.ts @@ -14,8 +14,6 @@ export interface Settings { identify?: WalkerOSMapping.Value; /** Destination-level group mapping */ group?: WalkerOSMapping.Value; - /** Event sections to flatten into capture properties */ - include?: string[]; // PostHog SDK passthrough options host?: string; flushAt?: number; @@ -29,6 +27,8 @@ export interface Settings { fetchRetryDelay?: number; debug?: boolean; disabled?: boolean; + /** The running send; the next one waits for it. Runtime only. */ + _sending?: Promise; } export type InitSettings = Partial; @@ -47,6 +47,10 @@ export interface PostHogClient { capture(props: Parameters[0]): void; identify(props: Parameters[0]): void; groupIdentify(props: Parameters[0]): void; + /** Sends the queued calls; rejects when the request failed. */ + flush(): Promise; + /** The SDK reports a failed send as an `error` event; returns its remover. */ + on?(event: 'error', listener: (error: unknown) => void): () => void; shutdown(): Promise | void; enable(): Promise | void; disable(): Promise | void; @@ -71,6 +75,7 @@ export type Config = { export type InitFn = DestinationServer.InitFn; export type PushFn = DestinationServer.PushFn; +export type PushBatchFn = CoreDestination.PushBatchFn; export type PartialConfig = DestinationServer.PartialConfig; diff --git a/packages/server/destinations/redis/src/__tests__/conformance.test.ts b/packages/server/destinations/redis/src/__tests__/conformance.test.ts new file mode 100644 index 000000000..e2be26bd3 --- /dev/null +++ b/packages/server/destinations/redis/src/__tests__/conformance.test.ts @@ -0,0 +1,21 @@ +import path from 'path'; +import { conformance } from '@walkeros/collector/dev'; +import { destinationRedis } from '..'; +import { examples, schemas } from '../dev'; + +describe('redis conformance', () => { + beforeEach(() => { + jest.useRealTimers(); + }); + + it('conforms', async () => { + await conformance('destination', destinationRedis, { + profile: 'socket', + examples: examples.step, + schema: schemas.settings, + settings: { redis: { streamKey: 'walkeros:events' } }, + env: { push: examples.env.push, simulation: examples.env.simulation }, + root: path.join(__dirname, '..', '..'), + }); + }); +}); diff --git a/packages/server/destinations/redis/src/config.ts b/packages/server/destinations/redis/src/config.ts index 32415795d..6a40f6f34 100644 --- a/packages/server/destinations/redis/src/config.ts +++ b/packages/server/destinations/redis/src/config.ts @@ -5,24 +5,21 @@ import type { PartialConfig, Env, } from './types'; -import type { Logger } from '@walkeros/core'; -import { isObject } from '@walkeros/core'; +import { isObject, StepError } from '@walkeros/core'; -export function getConfig( - partialConfig: PartialConfig = {}, - logger: Logger.Instance, -): Config { +export function getConfig(partialConfig: PartialConfig = {}): Config { const raw = (partialConfig.settings ?? {}) as Partial; const redis: Partial = raw.redis ?? ({} as Partial); - if (!redis.streamKey) { - logger.throw('Config settings redis.streamKey missing'); - } + if (!redis.streamKey) + throw new StepError('Config settings redis.streamKey missing', { + code: 'config', + }); const redisSettings: RedisSettings = { ...redis, - streamKey: redis.streamKey as string, + streamKey: redis.streamKey, serialization: redis.serialization ?? 'json', }; diff --git a/packages/server/destinations/redis/src/examples/step.ts b/packages/server/destinations/redis/src/examples/step.ts index 6de12f39e..32df5ed2a 100644 --- a/packages/server/destinations/redis/src/examples/step.ts +++ b/packages/server/destinations/redis/src/examples/step.ts @@ -114,3 +114,14 @@ export const ignoredEvent: RedisStepExample = { mapping: { ignore: true }, out: [], }; + +/** + * Silent rule -- mapping.silent: true appends nothing. + */ +export const silentRule: RedisStepExample = { + public: false, + description: 'A rule with silent: true appends nothing to the stream.', + in: getEvent('page view', { timestamp: 1700000106 }), + mapping: { silent: true }, + out: [], +}; diff --git a/packages/server/destinations/redis/src/index.ts b/packages/server/destinations/redis/src/index.ts index 589f373dd..54cf6d17c 100644 --- a/packages/server/destinations/redis/src/index.ts +++ b/packages/server/destinations/redis/src/index.ts @@ -6,6 +6,7 @@ import type { RedisClientConstructor, RedisClientOptions, } from './types'; +import { StepError } from '@walkeros/core'; import { getConfig, isRedisEnv } from './config'; import { push } from './push'; @@ -17,8 +18,8 @@ export const destinationRedis: Destination = { config: {}, - async init({ config: partialConfig, logger, env }) { - const config = getConfig(partialConfig, logger); + async init({ config: partialConfig, env, reportError }) { + const config = getConfig(partialConfig); const settings = config.settings as Settings; const redis = settings.redis; @@ -44,15 +45,17 @@ export const destinationRedis: Destination = { // ioredis exports the class as default (ESM) and as named Redis export. Constructor = ioredis.default ?? ioredis.Redis; } catch (err) { - logger.throw(`Failed to load ioredis: ${String(err)}`); - return config; + throw new StepError('Failed to load ioredis', { + code: 'not-ready', + cause: err, + }); } } - if (!Constructor) { - logger.throw('ioredis constructor not found'); - return config; - } + if (!Constructor) + throw new StepError('ioredis constructor not found', { + code: 'not-ready', + }); let client: RedisClientMock; if (redis.url) { @@ -61,6 +64,10 @@ export const destinationRedis: Destination = { client = new Constructor(redis.options ?? ({} as RedisClientOptions)); } + // ioredis reports a lost connection as an `error` event; without a + // listener it is unhandled. A failed XADD still fails its own push. + client.on?.('error', (err) => reportError?.(err)); + redis._client = client; redis._ownedClient = true; @@ -80,7 +87,11 @@ export const destinationRedis: Destination = { // Only close clients the destination created (not user-provided) if (client && redis._ownedClient) { try { - await client.quit(); + // A client that is not connected would queue QUIT behind its unsent + // commands and never answer: close it at once instead. + if (client.status && client.status !== 'ready' && client.disconnect) + client.disconnect(); + else await client.quit(); } finally { redis._client = undefined; redis._ownedClient = undefined; diff --git a/packages/server/destinations/redis/src/push.ts b/packages/server/destinations/redis/src/push.ts index 00ac1a918..3a056b92a 100644 --- a/packages/server/destinations/redis/src/push.ts +++ b/packages/server/destinations/redis/src/push.ts @@ -1,26 +1,22 @@ import type { PushFn, RedisSettings, - RedisClientMock, XaddArg, SerializationMode, } from './types'; -import { isString } from '@walkeros/core'; +import { isString, StepError } from '@walkeros/core'; -export const push: PushFn = async function (event, { config, rule, logger }) { - const settings = config.settings as { redis?: RedisSettings } | undefined; - const redis: RedisSettings | undefined = settings?.redis; - - if (!redis) { - logger.warn('Redis settings missing'); - return; - } +// A silent rule has no XADD of its own: nothing to append. +export const push: PushFn = async function (event, context) { + if (!context.rule?.silent) await append(event, context); +}; - const client = redis._client; - if (!client) { - logger.warn('Redis client not initialized'); - return; - } +const append: PushFn = async function (event, { config, rule, logger }) { + const settings = config.settings; + const client = settings?.redis?._client; + if (!settings?.redis || !client) + throw new StepError('Redis client not initialized', { code: 'not-ready' }); + const redis: RedisSettings = settings.redis; // Derive stream key: rule override -> destination default const ruleSettings = rule?.settings ?? {}; @@ -33,7 +29,7 @@ export const push: PushFn = async function (event, { config, rule, logger }) { // Serialize event const fields: string[] = serialization === 'flat' - ? flattenEvent(event as unknown as Record) + ? flattenEvent(event) : ['event', JSON.stringify(event)]; // Build XADD arguments @@ -51,18 +47,20 @@ export const push: PushFn = async function (event, { config, rule, logger }) { logger.debug('Redis XADD', { stream: streamKey }); + let entryId: string | null; try { - const entryId = await (client as RedisClientMock).xadd(...args); - logger.debug('Redis XADD complete', { stream: streamKey, entryId }); + entryId = await client.xadd(...args); } catch (error) { - logger.error('Redis XADD failed', { - stream: streamKey, - error: error instanceof Error ? error.message : String(error), + // The collector reports it; the client's own message stays the cause. + throw new StepError('Redis XADD failed', { + code: 'transport', + cause: error, }); } + logger.debug('Redis XADD complete', { stream: streamKey, entryId }); }; -function flattenEvent(event: Record): string[] { +function flattenEvent(event: object): string[] { const fields: string[] = []; for (const [key, value] of Object.entries(event)) { fields.push( diff --git a/packages/server/destinations/redis/src/schemas/settings.ts b/packages/server/destinations/redis/src/schemas/settings.ts index 8e1ed81e7..0d293e1fe 100644 --- a/packages/server/destinations/redis/src/schemas/settings.ts +++ b/packages/server/destinations/redis/src/schemas/settings.ts @@ -1,50 +1,54 @@ import { z } from '@walkeros/core/dev'; -export const RedisSettingsSchema = z.object({ - streamKey: z - .string() - .min(1) - .describe( - "Redis stream key name (like 'walkeros:events'). All events are appended to this stream via XADD.", - ), - url: z - .string() - .describe( - "Redis connection URL (like 'redis://localhost:6379' or 'rediss://:password@host:6380'). Supports redis:// and rediss:// (TLS) protocols.", - ) - .optional(), - options: z - .record(z.string(), z.unknown()) - .describe( - 'ioredis connection options. Used when url is not provided. Supports host, port, password, db, tls, and all other ioredis options.', - ) - .optional(), - maxLen: z - .number() - .int() - .positive() - .describe( - 'Maximum stream length. Enables approximate MAXLEN trimming on every XADD to bound memory usage (like 50000).', - ) - .optional(), - exactTrimming: z - .boolean() - .describe( - 'Use exact MAXLEN instead of approximate (~). Not recommended for production. Default: false.', - ) - .optional(), - serialization: z - .enum(['json', 'flat']) - .describe( - "Serialization mode. 'json' stores the full event as a single 'event' field (default). 'flat' stores top-level event fields as separate stream entry fields.", - ) - .optional(), -}); +export const RedisSettingsSchema = z + .object({ + streamKey: z + .string() + .min(1) + .describe( + "Redis stream key name (like 'walkeros:events'). All events are appended to this stream via XADD.", + ), + url: z + .string() + .describe( + "Redis connection URL (like 'redis://localhost:6379' or 'rediss://:password@host:6380'). Supports redis:// and rediss:// (TLS) protocols.", + ) + .optional(), + options: z + .record(z.string(), z.unknown()) + .describe( + 'ioredis connection options. Used when url is not provided. Supports host, port, password, db, tls, and all other ioredis options.', + ) + .optional(), + maxLen: z + .number() + .int() + .positive() + .describe( + 'Maximum stream length. Enables approximate MAXLEN trimming on every XADD to bound memory usage (like 50000).', + ) + .optional(), + exactTrimming: z + .boolean() + .describe( + 'Use exact MAXLEN instead of approximate (~). Not recommended for production. Default: false.', + ) + .optional(), + serialization: z + .enum(['json', 'flat']) + .describe( + "Serialization mode. 'json' stores the full event as a single 'event' field (default). 'flat' stores top-level event fields as separate stream entry fields.", + ) + .optional(), + }) + .strict(); -export const SettingsSchema = z.object({ - redis: RedisSettingsSchema.describe( - "Redis Streams configuration (like { streamKey: 'walkeros:events', url: 'redis://localhost:6379' })", - ), -}); +export const SettingsSchema = z + .object({ + redis: RedisSettingsSchema.describe( + "Redis Streams configuration (like { streamKey: 'walkeros:events', url: 'redis://localhost:6379' })", + ), + }) + .strict(); export type Settings = z.infer; diff --git a/packages/server/destinations/redis/src/types/index.ts b/packages/server/destinations/redis/src/types/index.ts index eda157d84..a8a8121c4 100644 --- a/packages/server/destinations/redis/src/types/index.ts +++ b/packages/server/destinations/redis/src/types/index.ts @@ -24,6 +24,10 @@ export interface RedisClientMock { xadd: (...args: XaddArg[]) => Promise; pipeline: () => RedisPipelineMock; quit: () => Promise; + /** Closes at once, without waiting for queued commands (ioredis). */ + disconnect?: () => void; + /** The connection state (ioredis): `ready` once it can send. */ + status?: string; on?: (event: string, listener: (...args: unknown[]) => void) => unknown; } diff --git a/packages/server/destinations/rudderstack/src/__tests__/conformance.test.ts b/packages/server/destinations/rudderstack/src/__tests__/conformance.test.ts new file mode 100644 index 000000000..1eb84f84a --- /dev/null +++ b/packages/server/destinations/rudderstack/src/__tests__/conformance.test.ts @@ -0,0 +1,54 @@ +import path from 'path'; +import { conformance, runExamples } from '@walkeros/collector/dev'; +import { destinationRudderStack } from '..'; +import { examples, schemas } from '../dev'; + +// The runs without env load the SDK: a stand-in that sends every call. +jest.mock('@rudderstack/rudder-sdk-node', () => { + const sent = (_params: unknown, callback?: () => void) => callback?.(); + return class { + track = sent; + identify = sent; + group = sent; + page = sent; + screen = sent; + alias = sent; + flush = () => Promise.resolve(); + }; +}); + +describe('rudderstack conformance', () => { + beforeEach(() => { + jest.useRealTimers(); + }); + + it('conforms', async () => { + await conformance('destination', destinationRudderStack, { + profile: 'sdk-env', + examples: examples.step, + schema: schemas.settings, + settings: { + writeKey: 'test-write-key', + dataPlaneUrl: 'https://hosted.rudderlabs.com', + }, + env: { push: examples.env.push, simulation: examples.env.simulation }, + root: path.join(__dirname, '..', '..'), + }); + }); + + it('sends nothing for a silent rule that finds nothing to send', async () => { + const { companyGroupWithoutData } = examples.step; + await runExamples( + 'destination', + destinationRudderStack, + { companyGroupWithoutData }, + { + settings: { + writeKey: 'test-write-key', + dataPlaneUrl: 'https://hosted.rudderlabs.com', + }, + env: { push: examples.env.push, simulation: examples.env.simulation }, + }, + ); + }); +}); diff --git a/packages/server/destinations/rudderstack/src/__tests__/failure.test.ts b/packages/server/destinations/rudderstack/src/__tests__/failure.test.ts new file mode 100644 index 000000000..cd58195e1 --- /dev/null +++ b/packages/server/destinations/rudderstack/src/__tests__/failure.test.ts @@ -0,0 +1,83 @@ +import { startFlow } from '@walkeros/collector'; +import { createEvent, getEvent } from '@walkeros/core'; +import { destinationRudderStack } from '..'; +import type { RudderStackAnalyticsMock, RudderStackCallback } from '../types'; + +const settings = { + writeKey: 'test', + dataPlaneUrl: 'https://hosted.rudderlabs.com', +}; + +/** + * A client that sends every call except those `fails` picks, which it answers + * the way the SDK does: through the callback, with an error. + */ +function analyticsFailing( + fails: (params: Record) => boolean, +): RudderStackAnalyticsMock { + const send = ( + params: Record, + callback?: RudderStackCallback, + ) => callback?.(fails(params) ? new Error('Service Unavailable') : undefined); + return { + track: send, + identify: send, + group: send, + page: send, + screen: send, + alias: send, + flush: () => Promise.resolve(), + }; +} + +describe('rudderstack send failures', () => { + it('fails a single push (batching off) when RudderStack reports the send failed', async () => { + const { collector } = await startFlow({ + logger: { level: 'ERROR', handler: () => undefined }, + destinations: { + rudderstack: { + code: destinationRudderStack, + config: { settings, batch: undefined }, + env: { analytics: analyticsFailing(() => true) }, + }, + }, + }); + + const result = await collector.push(getEvent('product view')); + + expect(result.ok).toBe(false); + expect(result.failed?.rudderstack?.reason).toMatchObject({ + code: 'transport', + message: 'RudderStack track failed', + }); + expect(collector.destinations.rudderstack.dlq).toHaveLength(1); + + await collector.command('shutdown'); + }); + + it('gives each entry of a batch its own outcome', async () => { + const { collector } = await startFlow({ + logger: { level: 'ERROR', handler: () => undefined }, + destinations: { + rudderstack: { + code: destinationRudderStack, + config: { settings }, + env: { + analytics: analyticsFailing(({ userId }) => userId === 'bad'), + }, + }, + }, + }); + + for (const id of ['a', 'bad', 'b']) + await collector.push(createEvent({ name: 'product view', user: { id } })); + await collector.command('flush'); + + const status = collector.status.destinations.rudderstack; + expect(status.count).toBe(2); + expect(status.failed).toBe(1); + expect(collector.destinations.rudderstack.dlq).toHaveLength(1); + + await collector.command('shutdown'); + }); +}); diff --git a/packages/server/destinations/rudderstack/src/__tests__/init.test.ts b/packages/server/destinations/rudderstack/src/__tests__/init.test.ts index 79d3278f5..7ff7cd03f 100644 --- a/packages/server/destinations/rudderstack/src/__tests__/init.test.ts +++ b/packages/server/destinations/rudderstack/src/__tests__/init.test.ts @@ -48,7 +48,16 @@ describe('init', () => { describe('destroy', () => { it('calls flush on the analytics instance', async () => { const flush = jest.fn().mockResolvedValue(undefined); - const mockAnalytics = { flush } as unknown as RudderStackAnalyticsMock; + const sent = () => undefined; + const mockAnalytics: RudderStackAnalyticsMock = { + track: sent, + identify: sent, + group: sent, + page: sent, + screen: sent, + alias: sent, + flush, + }; const config = { settings: { diff --git a/packages/server/destinations/rudderstack/src/__tests__/stepExamples.test.ts b/packages/server/destinations/rudderstack/src/__tests__/stepExamples.test.ts index a8ec85418..22e3dbc8b 100644 --- a/packages/server/destinations/rudderstack/src/__tests__/stepExamples.test.ts +++ b/packages/server/destinations/rudderstack/src/__tests__/stepExamples.test.ts @@ -29,23 +29,29 @@ function spyEnv(): { env: Env; collected: () => Captured[] } { const calls: Captured[] = []; const analytics: RudderStackAnalyticsMock = { - track: (params) => { + track: (params, callback) => { calls.push(['analytics.track', params]); + callback?.(); }, - identify: (params) => { + identify: (params, callback) => { calls.push(['analytics.identify', params]); + callback?.(); }, - group: (params) => { + group: (params, callback) => { calls.push(['analytics.group', params]); + callback?.(); }, - page: (params) => { + page: (params, callback) => { calls.push(['analytics.page', params]); + callback?.(); }, - screen: (params) => { + screen: (params, callback) => { calls.push(['analytics.screen', params]); + callback?.(); }, - alias: (params) => { + alias: (params, callback) => { calls.push(['analytics.alias', params]); + callback?.(); }, flush: () => Promise.resolve(), }; @@ -93,6 +99,8 @@ describe('rudderstack server destination -- step examples', () => { }); await elb(event); + // The destination batches by default: send what it holds. + await elb('walker flush'); expect(collected()).toEqual(example.out); }); diff --git a/packages/server/destinations/rudderstack/src/config.ts b/packages/server/destinations/rudderstack/src/config.ts index d55f8062a..546eb5571 100644 --- a/packages/server/destinations/rudderstack/src/config.ts +++ b/packages/server/destinations/rudderstack/src/config.ts @@ -1,15 +1,19 @@ import type { Config, Settings, PartialConfig } from './types'; -import type { Logger } from '@walkeros/core'; +import { StepError } from '@walkeros/core'; -export function getConfig( - partialConfig: PartialConfig = {}, - logger: Logger.Instance, -): Config { - const settings = (partialConfig.settings || {}) as Partial; +export function getConfig(partialConfig: PartialConfig = {}): Config { + const settings: Partial = partialConfig.settings || {}; const { writeKey, dataPlaneUrl } = settings; - if (!writeKey) logger.throw('Config settings writeKey missing'); - if (!dataPlaneUrl) logger.throw('Config settings dataPlaneUrl missing'); + if (!writeKey || !dataPlaneUrl) { + const missing = [ + ...(writeKey ? [] : ['writeKey']), + ...(dataPlaneUrl ? [] : ['dataPlaneUrl']), + ]; + throw new StepError(`Config settings ${missing.join(', ')} missing`, { + code: 'config', + }); + } const settingsConfig: Settings = { ...settings, diff --git a/packages/server/destinations/rudderstack/src/examples/env.ts b/packages/server/destinations/rudderstack/src/examples/env.ts index 5bb915009..9cf1662ab 100644 --- a/packages/server/destinations/rudderstack/src/examples/env.ts +++ b/packages/server/destinations/rudderstack/src/examples/env.ts @@ -1,21 +1,21 @@ -import type { Env, RudderStackAnalyticsMock } from '../types'; +import type { + Env, + RudderStackAnalyticsMock, + RudderStackCallback, +} from '../types'; -type SdkCall = (_params: Record) => void; -type SdkFlush = ( - _callback?: (err?: Error, data?: unknown) => void, -) => Promise; - -const noop: SdkCall = () => {}; -const asyncNoop: SdkFlush = () => Promise.resolve(); +/** Every call is sent: its callback answers without an error. */ +const sent = (_params: unknown, callback?: RudderStackCallback) => callback?.(); +const asyncNoop = () => Promise.resolve(); function createMockAnalytics(): RudderStackAnalyticsMock { return { - track: noop, - identify: noop, - group: noop, - page: noop, - screen: noop, - alias: noop, + track: sent, + identify: sent, + group: sent, + page: sent, + screen: sent, + alias: sent, flush: asyncNoop, }; } diff --git a/packages/server/destinations/rudderstack/src/examples/step.ts b/packages/server/destinations/rudderstack/src/examples/step.ts index 7517fb6ca..699a16fac 100644 --- a/packages/server/destinations/rudderstack/src/examples/step.ts +++ b/packages/server/destinations/rudderstack/src/examples/step.ts @@ -374,6 +374,47 @@ export const anonymousOnly: RudderStackStepExample = { ], }; +/** + * No identity -- neither userId nor anonymousId resolves from the event, so + * RudderStack cannot take it: the push fails with missing-input instead of + * counting as delivered. One per event name above, so every example's event + * without identity has its declared outcome. + */ +function withoutIdentity( + name: string, + timestamp: number, +): RudderStackStepExample { + const [entity, action] = name.split(' '); + return { + public: false, + description: `A ${name} event without userId or anonymousId cannot be sent to RudderStack: the push fails with missing-input instead of counting as delivered.`, + in: { name, entity, action, timestamp }, + out: [], + outcome: { status: 'failed', code: 'missing-input', retryable: false }, + }; +} + +export const noIdentityProduct = withoutIdentity('product view', 1700000110); +export const noIdentityOrder = withoutIdentity('order complete', 1700000111); +export const noIdentityPage = withoutIdentity('page view', 1700000112); +export const noIdentityLogin = withoutIdentity('user login', 1700000113); +export const noIdentityCompany = withoutIdentity('company update', 1700000114); +export const noIdentityScreen = withoutIdentity('screen view', 1700000115); +export const noIdentityMerge = withoutIdentity('identity merge', 1700000116); + +/** A silent rule without identify, group, page, screen or alias calls nothing. */ +export const silentRule: RudderStackStepExample = { + public: false, + description: + 'A rule with silent: true and no identify, group, page, screen or alias mapping sends nothing to RudderStack.', + in: getEvent('page view', { + timestamp: 1700000117, + user: { id: 'us3r', session: 's3ss10n' }, + }), + mapping: { silent: true }, + out: [], +}; + /** * Wildcard ignore -- the event matches a mapping rule with ignore: true. * The destination fires zero SDK calls. @@ -388,3 +429,22 @@ export const wildcardIgnored: RudderStackStepExample = { mapping: { ignore: true }, out: [], }; + +/** + * The silent group rule of companyGroup on an event without company data: no + * groupId resolves and the rule suppresses the track, so nothing is sent. + */ +export const companyGroupWithoutData: RudderStackStepExample = { + public: false, + description: + 'A silent group rule whose groupId finds no data sends nothing: an unmet mapping, not a failure, so the push returns normally.', + in: { + name: 'company update', + entity: 'company', + action: 'update', + timestamp: 1700000120, + user: { id: 'us3r', session: 's3ss10n' }, + }, + mapping: companyGroup.mapping, + out: [], +}; diff --git a/packages/server/destinations/rudderstack/src/index.ts b/packages/server/destinations/rudderstack/src/index.ts index b4f9e8fe5..6e8d043b3 100644 --- a/packages/server/destinations/rudderstack/src/index.ts +++ b/packages/server/destinations/rudderstack/src/index.ts @@ -1,6 +1,7 @@ -import type { Destination, Settings, RudderStackAnalyticsMock } from './types'; +import type { Destination } from './types'; +import { StepError } from '@walkeros/core'; import { getConfig } from './config'; -import { push } from './push'; +import { push, pushBatch } from './push'; // Types export * as DestinationRudderStack from './types'; @@ -8,18 +9,15 @@ export * as DestinationRudderStack from './types'; export const destinationRudderStack: Destination = { type: 'rudderstack', - config: {}, + // Batched by default, at the SDK's own former defaults (20 events, 10 s). + config: { batch: { size: 20, wait: 10_000 } }, - init({ config: partialConfig, logger, env }) { - const config = getConfig(partialConfig, logger); - const settings = config.settings as Settings; + init({ config: partialConfig, env }) { + const config = getConfig(partialConfig); + const { settings } = config; // Use env.analytics mock if provided (testing), otherwise create real SDK - const envAnalytics = ( - env as { analytics?: RudderStackAnalyticsMock } | undefined - )?.analytics; - - if (!envAnalytics) { + if (!env?.analytics) { // Production path: create real Analytics instance try { const Analytics = require('@rudderstack/rudder-sdk-node'); @@ -36,14 +34,21 @@ export const destinationRudderStack: Destination = { ...sdkOptions } = settings; - // RudderStack uses positional constructor: new Analytics(writeKey, opts) - const analytics = new Analytics(writeKey, { + // RudderStack uses positional constructor: new Analytics(writeKey, opts). + // Each push and each batch flushes the SDK and waits until its calls + // are sent. A failed send reaches each call's callback; the error + // handler only keeps the SDK from rethrowing it as an unhandled + // rejection. + settings._analytics = new Analytics(writeKey, { dataPlaneUrl, ...sdkOptions, + errorHandler: () => undefined, }); - settings._analytics = analytics; } catch (err) { - logger.throw(`Failed to initialize RudderStack SDK: ${err}`); + throw new StepError('Failed to initialize the RudderStack SDK', { + code: 'not-ready', + cause: err, + }); } } @@ -52,16 +57,13 @@ export const destinationRudderStack: Destination = { return config; }, - async push(event, context) { - return await push(event, context); - }, + push, + + pushBatch, async destroy({ config }) { - const settings = (config?.settings || {}) as Settings; - const analytics = settings._analytics; - if (analytics) { - await (analytics as unknown as RudderStackAnalyticsMock).flush(); - } + const analytics = config?.settings?._analytics; + if (analytics) await analytics.flush(); }, }; diff --git a/packages/server/destinations/rudderstack/src/push.ts b/packages/server/destinations/rudderstack/src/push.ts index 5c13c48cd..fcff22522 100644 --- a/packages/server/destinations/rudderstack/src/push.ts +++ b/packages/server/destinations/rudderstack/src/push.ts @@ -1,31 +1,73 @@ -import type { WalkerOS } from '@walkeros/core'; +import type { Collector, Destination, WalkerOS } from '@walkeros/core'; import type { PushFn, + PushBatchFn, Settings, RuntimeState, RudderStackAnalyticsMock, + RudderStackCallback, + Rule, } from './types'; -import { getMappingValue, isObject, isString, isArray } from '@walkeros/core'; +import { + getMappingValue, + isObject, + isString, + isArray, + isThenable, + StepError, +} from '@walkeros/core'; + +/** + * Hand one call to the SDK. The promise settles once RudderStack sent it: the + * SDK answers through the callback, with the error when the send failed; an + * injected client may answer with a promise instead. + */ +function call( + message: string, + send: (done: RudderStackCallback) => unknown, +): Promise { + return new Promise((resolve, reject) => { + const fail = (error: unknown) => + reject(new StepError(message, { code: 'transport', cause: error })); + const answer: unknown = send((error) => (error ? fail(error) : resolve())); + if (isThenable(answer)) answer.then(() => resolve(), fail); + }); +} -export const push: PushFn = async function ( - event, - { config, rule, data, collector, env, logger }, -) { - const settings = config.settings as Settings; - const analytics = settings._analytics as unknown as - | RudderStackAnalyticsMock - | undefined; - const envAnalytics = ( - env as { analytics?: RudderStackAnalyticsMock } | undefined - )?.analytics; - const sdk = envAnalytics || analytics; - - if (!sdk) { - logger.warn('RudderStack Analytics not initialized'); - return; - } +/** + * Ask the SDK to send what it holds, one flush at a time. A failed send + * reaches each call's callback; the flush itself does not reject (the SDK's + * error handler is set). + */ +function flush(settings: Partial, sdk: RudderStackAnalyticsMock) { + const flushing = (settings._flushing ?? Promise.resolve()).then(() => + sdk.flush(), + ); + settings._flushing = flushing.catch(() => undefined); + return flushing; +} +/** The settled calls of one event: the first failure, if any. */ +async function delivered(calls: Promise[]>) { + for (const result of await calls) + if (result.status === 'rejected') throw result.reason; +} + +/** + * Hand the calls of one event to the SDK (identify, group, page, screen, + * track) and answer their deliveries. An event without userId or anonymousId + * cannot be sent and fails with missing-input. + */ +async function enqueue( + event: WalkerOS.Event, + rule: Rule | undefined, + data: Destination.Data, + settings: Partial, + sdk: RudderStackAnalyticsMock, + collector: Collector.Instance, +): Promise[]> { const state: RuntimeState = settings._state || {}; + settings._state = state; const mappingSettings = rule?.settings || {}; const timestamp = new Date(event.timestamp || Date.now()); @@ -41,100 +83,198 @@ export const push: PushFn = async function ( ) : undefined; - if (!userId && !anonymousId) { - logger.warn('RudderStack requires userId or anonymousId; skipping event', { - event: event.name, - }); - return; - } + if (!userId && !anonymousId) + throw new StepError( + 'RudderStack needs a userId or anonymousId for the event', + { + code: 'missing-input', + }, + ); // Build identity object for every SDK call const identity: Record = {}; if (userId) identity.userId = userId; if (anonymousId) identity.anonymousId = anonymousId; + const calls: Array | undefined> = []; + // 2. Identify -- rule-level overrides destination-level const identifyMapping = mappingSettings.identify ?? settings.identify; if (identifyMapping !== undefined) { const resolved = await getMappingValue(event, identifyMapping, { collector, }); - if (isObject(resolved)) { - applyIdentify( - sdk, - resolved as Record, - identity, - state, - timestamp, + if (isObject(resolved)) + calls.push( + applyIdentify( + sdk, + resolved as Record, + identity, + state, + timestamp, + ), ); - } } // 3. Group -- rule-level overrides destination-level const groupMapping = mappingSettings.group ?? settings.group; if (groupMapping !== undefined) { - const resolved = await getMappingValue(event, groupMapping, { collector }); - if (isObject(resolved)) { - applyGroup( - sdk, - resolved as Record, - identity, - state, - timestamp, + const resolved = await getMappingValue(event, groupMapping, { + collector, + }); + if (isObject(resolved)) + calls.push( + applyGroup( + sdk, + resolved as Record, + identity, + state, + timestamp, + ), ); - } } - // 4. Page - if (mappingSettings.page !== undefined) { - await applyPage( - sdk, - mappingSettings.page, - event, - identity, - collector, - timestamp, + // 4. Page (the call is not awaited here: it is delivered at the flush) + if (mappingSettings.page !== undefined) + calls.push( + ( + await applyPage( + sdk, + mappingSettings.page, + event, + identity, + collector, + timestamp, + ) + ).sent, ); - } // 5. Screen - if (mappingSettings.screen !== undefined) { - await applyScreen( - sdk, - mappingSettings.screen, - event, - identity, - collector, - timestamp, + if (mappingSettings.screen !== undefined) + calls.push( + ( + await applyScreen( + sdk, + mappingSettings.screen, + event, + identity, + collector, + timestamp, + ) + ).sent, ); - } // 6. Alias - if (mappingSettings.alias !== undefined) { - await applyAlias( - sdk, - mappingSettings.alias, - event, - identity, - collector, - timestamp, + if (mappingSettings.alias !== undefined) + calls.push( + ( + await applyAlias( + sdk, + mappingSettings.alias, + event, + identity, + collector, + timestamp, + ) + ).sent, ); - } // 7. Track (unless silent: true) if (rule?.silent !== true) { const eventName = isString(rule?.name) ? rule.name : event.name; const properties = isObject(data) ? (data as Record) : {}; - sdk.track({ + const params: Record = { ...identity, event: eventName, properties, timestamp, + }; + + calls.push( + call('RudderStack track failed', (done) => sdk.track(params, done)), + ); + } + + return calls.filter((sent): sent is Promise => sent !== undefined); +} + +/** The SDK the destination sends through, or a StepError. */ +function sdkOf( + settings: Partial | undefined, + env: { analytics?: RudderStackAnalyticsMock } | undefined, +): RudderStackAnalyticsMock { + const sdk = env?.analytics || settings?._analytics; + if (!settings || !sdk) + throw new StepError('RudderStack Analytics not initialized', { + code: 'not-ready', }); + return sdk; +} + +/** One event: its calls handed to the SDK, sent, and their deliveries awaited. */ +export const push: PushFn = async function ( + event, + { config, rule, data, collector, env }, +) { + const settings = config.settings ?? {}; + const sdk = sdkOf(config.settings, env); + const calls = await enqueue(event, rule, data, settings, sdk, collector); + if (calls.length) { + const settled = Promise.allSettled(calls); + await flush(settings, sdk); + await delivered(settled); } +}; - settings._state = state; +/** + * A flushed `config.batch` batch: every entry's calls handed to the SDK, one + * flush, and each entry's own outcome from its deliveries. An entry that + * cannot be sent fails alone; when nothing landed, the batch throws. + */ +export const pushBatch: PushBatchFn = async function ( + batch, + { config, collector, env }, +) { + const settings = config.settings ?? {}; + const sdk = sdkOf(config.settings, env); + // Each entry's deliveries, or why its calls could not be handed over. + const outcomes: Array<{ + settled?: Promise[]>; + error?: unknown; + }> = []; + + for (const entry of batch.entries) { + try { + const calls = await enqueue( + entry.event, + entry.rule, + entry.data, + settings, + sdk, + collector, + ); + outcomes.push({ settled: Promise.allSettled(calls) }); + } catch (error) { + outcomes.push({ error }); + } + } + + await flush(settings, sdk); + + const failed: Destination.BatchFailure[] = []; + for (const [index, { settled, error }] of outcomes.entries()) { + if (!settled) failed.push({ index, error }); + else + await delivered(settled).catch((reason: unknown) => + failed.push({ index, error: reason }), + ); + } + + // Nothing landed: a whole-batch failure, which the breaker counts. + if (failed.length && failed.length === batch.entries.length) + throw failed[0].error; + if (failed.length) return { failed }; }; function resolveString(value: unknown): string | undefined { @@ -157,7 +297,7 @@ function applyIdentify( identity: Record, state: RuntimeState, timestamp: Date, -): void { +): Promise | undefined { const last = state.lastIdentity || {}; // Per-event identify can override userId @@ -173,7 +313,7 @@ function applyIdentify( const userIdChanged = identifyUserId !== last.userId; const traitsChanged = traitsHash !== (last.traitsHash ?? ''); - if (!userIdChanged && !traitsChanged) return; + if (!userIdChanged && !traitsChanged) return undefined; const params: Record = { ...identity, @@ -182,13 +322,20 @@ function applyIdentify( if (identifyUserId) params.userId = identifyUserId; if (traits) params.traits = traits; - sdk.identify(params); - - state.lastIdentity = { + // Remembered at once, so a batch identifies a user once; forgotten again + // when the delivery fails, so the next event identifies again. + const current = { userId: identifyUserId, anonymousId: identity.anonymousId as string | undefined, traitsHash, }; + state.lastIdentity = current; + return call('RudderStack identify failed', (done) => + sdk.identify(params, done), + ).catch((error: unknown) => { + if (state.lastIdentity === current) state.lastIdentity = undefined; + throw error; + }); } function applyGroup( @@ -197,9 +344,9 @@ function applyGroup( identity: Record, state: RuntimeState, timestamp: Date, -): void { +): Promise | undefined { const groupId = isString(resolved.groupId) ? resolved.groupId : undefined; - if (!groupId) return; + if (!groupId) return undefined; const last = state.lastGroup || {}; const traits = @@ -210,7 +357,7 @@ function applyGroup( const traitsHash = hashTraits(traits); const groupIdChanged = groupId !== last.groupId; const traitsChanged = traitsHash !== (last.traitsHash ?? ''); - if (!groupIdChanged && !traitsChanged) return; + if (!groupIdChanged && !traitsChanged) return undefined; const params: Record = { ...identity, @@ -219,11 +366,18 @@ function applyGroup( }; if (traits) params.traits = traits; - sdk.group(params); - - state.lastGroup = { groupId, traitsHash }; + // Remembered at once, forgotten again when the delivery fails. + const current = { groupId, traitsHash }; + state.lastGroup = current; + return call('RudderStack group failed', (done) => + sdk.group(params, done), + ).catch((error: unknown) => { + if (state.lastGroup === current) state.lastGroup = undefined; + throw error; + }); } +/** The page call, handed to the SDK; its delivery settles at the flush. */ async function applyPage( sdk: RudderStackAnalyticsMock, pageMapping: unknown, @@ -231,14 +385,13 @@ async function applyPage( identity: Record, collector: unknown, timestamp: Date, -): Promise { +): Promise<{ sent?: Promise }> { const params: Record = { ...identity, timestamp }; if (pageMapping === true) { - // Minimal page call -- RudderStack requires name, use empty string - params.name = ''; - sdk.page(params); - return; + return { + sent: call('RudderStack page failed', (done) => sdk.page(params, done)), + }; } const resolved = await getMappingValue( @@ -246,21 +399,25 @@ async function applyPage( pageMapping as Parameters[1], { collector } as Parameters[2], ); - if (!isObject(resolved)) return; + if (!isObject(resolved)) return {}; const r = resolved as { + category?: unknown; name?: unknown; properties?: unknown; }; - // RudderStack requires name -- fall back to empty string - params.name = isString(r.name) ? r.name : ''; + if (isString(r.category)) params.category = r.category; + if (isString(r.name)) params.name = r.name; if (isObject(r.properties) && !isArray(r.properties)) { params.properties = r.properties; } - sdk.page(params); + return { + sent: call('RudderStack page failed', (done) => sdk.page(params, done)), + }; } +/** The screen call, handed to the SDK; its delivery settles at the flush. */ async function applyScreen( sdk: RudderStackAnalyticsMock, screenMapping: unknown, @@ -268,14 +425,15 @@ async function applyScreen( identity: Record, collector: unknown, timestamp: Date, -): Promise { +): Promise<{ sent?: Promise }> { const params: Record = { ...identity, timestamp }; if (screenMapping === true) { - // Minimal screen call -- RudderStack requires name, use empty string - params.name = ''; - sdk.screen(params); - return; + return { + sent: call('RudderStack screen failed', (done) => + sdk.screen(params, done), + ), + }; } const resolved = await getMappingValue( @@ -283,21 +441,25 @@ async function applyScreen( screenMapping as Parameters[1], { collector } as Parameters[2], ); - if (!isObject(resolved)) return; + if (!isObject(resolved)) return {}; const r = resolved as { + category?: unknown; name?: unknown; properties?: unknown; }; - // RudderStack requires name -- fall back to empty string - params.name = isString(r.name) ? r.name : ''; + if (isString(r.category)) params.category = r.category; + if (isString(r.name)) params.name = r.name; if (isObject(r.properties) && !isArray(r.properties)) { params.properties = r.properties; } - sdk.screen(params); + return { + sent: call('RudderStack screen failed', (done) => sdk.screen(params, done)), + }; } +/** The alias call, handed to the SDK; its delivery settles at the flush. */ async function applyAlias( sdk: RudderStackAnalyticsMock, aliasMapping: unknown, @@ -305,17 +467,17 @@ async function applyAlias( identity: Record, collector: unknown, timestamp: Date, -): Promise { +): Promise<{ sent?: Promise }> { const resolved = await getMappingValue( event, aliasMapping as Parameters[1], { collector } as Parameters[2], ); - if (!isObject(resolved)) return; + if (!isObject(resolved)) return {}; const r = resolved as { previousId?: unknown }; const previousId = isString(r.previousId) ? r.previousId : undefined; - if (!previousId) return; + if (!previousId) return {}; const params: Record = { previousId, @@ -324,5 +486,7 @@ async function applyAlias( // Alias uses userId (the new canonical ID) -- not full identity spread if (identity.userId) params.userId = identity.userId; - sdk.alias(params); + return { + sent: call('RudderStack alias failed', (done) => sdk.alias(params, done)), + }; } diff --git a/packages/server/destinations/rudderstack/src/schemas/settings.ts b/packages/server/destinations/rudderstack/src/schemas/settings.ts index 71ed07727..6d7ff46d9 100644 --- a/packages/server/destinations/rudderstack/src/schemas/settings.ts +++ b/packages/server/destinations/rudderstack/src/schemas/settings.ts @@ -1,93 +1,99 @@ import { z } from '@walkeros/core/dev'; -export const SettingsSchema = z.object({ - writeKey: z - .string() - .min(1) - .describe( - 'RudderStack source write key. Find it in the RudderStack dashboard under Sources > Setup.', - ), - dataPlaneUrl: z - .string() - .url() - .describe( - 'RudderStack data plane URL. Required. Example: https://your-data-plane.rudderstack.com', - ), - userId: z - .string() - .describe( - 'walkerOS mapping value path to resolve userId from each event (like user.id).', - ) - .optional(), - anonymousId: z - .string() - .describe( - 'walkerOS mapping value path to resolve anonymousId from each event (like user.session).', - ) - .optional(), - path: z.string().describe('API path route (like /v1/batch).').optional(), - flushAt: z - .number() - .int() - .positive() - .describe('Events to enqueue before flushing a batch. Default: 20.') - .optional(), - flushInterval: z - .number() - .int() - .positive() - .describe('Max milliseconds before auto-flush. Default: 10000.') - .optional(), - maxQueueSize: z - .number() - .int() - .positive() - .describe('Maximum batch payload size in bytes. Default: 460800 (~500KB).') - .optional(), - maxInternalQueueSize: z - .number() - .int() - .positive() - .describe('Maximum in-memory queue length. Default: 20000.') - .optional(), - logLevel: z - .string() - .describe( - "SDK log level: 'info', 'debug', 'error', or 'silly'. Default: 'info'.", - ) - .optional(), - retryCount: z - .number() - .int() - .min(0) - .describe('Retry attempts for failed batches. Default: 3.') - .optional(), - enable: z - .boolean() - .describe('Set to false to no-op all SDK calls. Default: true.') - .optional(), - gzip: z - .boolean() - .describe('Enable gzip compression for requests. Default: true.') - .optional(), - identify: z - .unknown() - .describe( - 'Destination-level identity mapping. Resolves to { traits } object. Fires identify() on the first push and re-fires when values change.', - ) - .optional(), - group: z - .unknown() - .describe( - 'Destination-level group mapping. Resolves to { groupId, traits }. Fires group() on the first push and re-fires on change.', - ) - .optional(), - integrations: z - .record(z.string(), z.unknown()) - .describe( - 'Enable/disable downstream RudderStack destinations. Example: { "All": true, "Mixpanel": false }.', - ) - .optional(), -}); +export const SettingsSchema = z + .object({ + writeKey: z + .string() + .min(1) + .describe( + 'RudderStack source write key. Find it in the RudderStack dashboard under Sources > Setup.', + ), + dataPlaneUrl: z + .string() + .url() + .describe( + 'RudderStack data plane URL. Required. Example: https://your-data-plane.rudderstack.com', + ), + userId: z + .string() + .describe( + 'walkerOS mapping value path to resolve userId from each event (like user.id).', + ) + .optional(), + anonymousId: z + .string() + .describe( + 'walkerOS mapping value path to resolve anonymousId from each event (like user.session).', + ) + .optional(), + path: z.string().describe('API path route (like /v1/batch).').optional(), + flushAt: z + .number() + .int() + .positive() + .describe( + 'Events the SDK sends in one request. Each push and each config.batch batch flushes the SDK and waits until its calls are sent. Default: 20.', + ) + .optional(), + flushInterval: z + .number() + .int() + .positive() + .describe('Max milliseconds before auto-flush. Default: 10000.') + .optional(), + maxQueueSize: z + .number() + .int() + .positive() + .describe( + 'Maximum batch payload size in bytes. Default: 460800 (~500KB).', + ) + .optional(), + maxInternalQueueSize: z + .number() + .int() + .positive() + .describe('Maximum in-memory queue length. Default: 20000.') + .optional(), + logLevel: z + .string() + .describe( + "SDK log level: 'info', 'debug', 'error', or 'silly'. Default: 'info'.", + ) + .optional(), + retryCount: z + .number() + .int() + .min(0) + .describe('Retry attempts for failed batches. Default: 3.') + .optional(), + enable: z + .boolean() + .describe('Set to false to no-op all SDK calls. Default: true.') + .optional(), + gzip: z + .boolean() + .describe('Enable gzip compression for requests. Default: true.') + .optional(), + identify: z + .unknown() + .describe( + 'Destination-level identity mapping. Resolves to { traits } object. Fires identify() on the first push and re-fires when values change.', + ) + .optional(), + group: z + .unknown() + .describe( + 'Destination-level group mapping. Resolves to { groupId, traits }. Fires group() on the first push and re-fires on change.', + ) + .optional(), + integrations: z + .record(z.string(), z.unknown()) + .describe( + 'Enable/disable downstream RudderStack destinations. Example: { "All": true, "Mixpanel": false }.', + ) + .optional(), + }) + .strict(); export type Settings = z.infer; diff --git a/packages/server/destinations/rudderstack/src/types/index.ts b/packages/server/destinations/rudderstack/src/types/index.ts index 6a78a0153..186e22aa2 100644 --- a/packages/server/destinations/rudderstack/src/types/index.ts +++ b/packages/server/destinations/rudderstack/src/types/index.ts @@ -39,8 +39,10 @@ export interface Settings { /** Enable gzip compression. Default: true. */ gzip?: boolean; /** Runtime state -- not user-facing. Mutated by init/push. */ - _analytics?: unknown; + _analytics?: RudderStackAnalyticsMock; _state?: RuntimeState; + /** The running flush; the next one waits for it. Runtime only. */ + _flushing?: Promise; } export interface RuntimeState { @@ -84,14 +86,35 @@ export interface Env extends DestinationServer.Env { * destination actually calls. Tests can provide this via env.analytics * instead of the real SDK. */ +/** Called once the SDK sent the call, with the error if it failed. */ +export type RudderStackCallback = (error?: Error, data?: unknown) => void; + export interface RudderStackAnalyticsMock { - track: (params: Record) => void; - identify: (params: Record) => void; - group: (params: Record) => void; - page: (params: Record) => void; - screen: (params: Record) => void; - alias: (params: Record) => void; - flush: (callback?: (err?: Error, data?: unknown) => void) => Promise; + track: ( + params: Record, + callback?: RudderStackCallback, + ) => unknown; + identify: ( + params: Record, + callback?: RudderStackCallback, + ) => unknown; + group: ( + params: Record, + callback?: RudderStackCallback, + ) => unknown; + page: ( + params: Record, + callback?: RudderStackCallback, + ) => unknown; + screen: ( + params: Record, + callback?: RudderStackCallback, + ) => unknown; + alias: ( + params: Record, + callback?: RudderStackCallback, + ) => unknown; + flush: (callback?: RudderStackCallback) => Promise; } export type Types = CoreDestination.Types; @@ -106,6 +129,7 @@ export type Config = { export type InitFn = DestinationServer.InitFn; export type PushFn = DestinationServer.PushFn; +export type PushBatchFn = CoreDestination.PushBatchFn; export type PartialConfig = DestinationServer.PartialConfig; export type PushEvents = DestinationServer.PushEvents; export type Rule = WalkerOSMapping.Rule; diff --git a/packages/server/destinations/segment/src/__tests__/conformance.test.ts b/packages/server/destinations/segment/src/__tests__/conformance.test.ts new file mode 100644 index 000000000..5096384d0 --- /dev/null +++ b/packages/server/destinations/segment/src/__tests__/conformance.test.ts @@ -0,0 +1,51 @@ +import path from 'path'; +import { conformance, runExamples } from '@walkeros/collector/dev'; +import { destinationSegment } from '..'; +import { examples, schemas } from '../dev'; + +// The runs without env load the SDK: a stand-in that delivers every call. +jest.mock('@segment/analytics-node', () => { + const delivered = (_params: unknown, callback?: () => void) => callback?.(); + return { + __esModule: true, + Analytics: class { + track = delivered; + identify = delivered; + group = delivered; + page = delivered; + screen = delivered; + flush = () => Promise.resolve(); + closeAndFlush = () => Promise.resolve(); + }, + }; +}); + +describe('segment conformance', () => { + beforeEach(() => { + jest.useRealTimers(); + }); + + it('conforms', async () => { + await conformance('destination', destinationSegment, { + profile: 'sdk-env', + examples: examples.step, + schema: schemas.settings, + settings: { writeKey: 'test-write-key' }, + env: { push: examples.env.push, simulation: examples.env.simulation }, + root: path.join(__dirname, '..', '..'), + }); + }); + + it('sends nothing for a silent rule that finds nothing to send', async () => { + const { companyGroupWithoutData } = examples.step; + await runExamples( + 'destination', + destinationSegment, + { companyGroupWithoutData }, + { + settings: { writeKey: 'test-write-key' }, + env: { push: examples.env.push, simulation: examples.env.simulation }, + }, + ); + }); +}); diff --git a/packages/server/destinations/segment/src/__tests__/failure.test.ts b/packages/server/destinations/segment/src/__tests__/failure.test.ts new file mode 100644 index 000000000..2ffeb77f8 --- /dev/null +++ b/packages/server/destinations/segment/src/__tests__/failure.test.ts @@ -0,0 +1,76 @@ +import { startFlow } from '@walkeros/collector'; +import { createEvent, getEvent } from '@walkeros/core'; +import { destinationSegment } from '..'; +import type { SegmentAnalyticsMock, SegmentCallback } from '../types'; + +/** + * A client that delivers every call except those `fails` picks, which it + * answers the way the SDK does: through the callback, with a reason. + */ +function analyticsFailing( + fails: (params: Record) => boolean, +): SegmentAnalyticsMock { + const send = (params: Record, callback?: SegmentCallback) => + callback?.(fails(params) ? new Error('[503] Service Unavailable') : null); + return { + track: send, + identify: send, + group: send, + page: send, + screen: send, + flush: () => Promise.resolve(), + closeAndFlush: () => Promise.resolve(), + }; +} + +describe('segment delivery failures', () => { + it('fails a single push (batching off) when Segment reports the delivery failed', async () => { + const { collector } = await startFlow({ + logger: { level: 'ERROR', handler: () => undefined }, + destinations: { + segment: { + code: destinationSegment, + config: { settings: { writeKey: 'test' }, batch: undefined }, + env: { analytics: analyticsFailing(() => true) }, + }, + }, + }); + + const result = await collector.push(getEvent('product view')); + + expect(result.ok).toBe(false); + expect(result.failed?.segment?.reason).toMatchObject({ + code: 'transport', + message: 'Segment track failed', + }); + expect(collector.destinations.segment.dlq).toHaveLength(1); + + await collector.command('shutdown'); + }); + + it('gives each entry of a batch its own outcome', async () => { + const { collector } = await startFlow({ + logger: { level: 'ERROR', handler: () => undefined }, + destinations: { + segment: { + code: destinationSegment, + config: { settings: { writeKey: 'test' } }, + env: { + analytics: analyticsFailing(({ userId }) => userId === 'bad'), + }, + }, + }, + }); + + for (const id of ['a', 'bad', 'b']) + await collector.push(createEvent({ name: 'product view', user: { id } })); + await collector.command('flush'); + + const status = collector.status.destinations.segment; + expect(status.count).toBe(2); + expect(status.failed).toBe(1); + expect(collector.destinations.segment.dlq).toHaveLength(1); + + await collector.command('shutdown'); + }); +}); diff --git a/packages/server/destinations/segment/src/__tests__/init.test.ts b/packages/server/destinations/segment/src/__tests__/init.test.ts index bc2b21923..cbdee9d3e 100644 --- a/packages/server/destinations/segment/src/__tests__/init.test.ts +++ b/packages/server/destinations/segment/src/__tests__/init.test.ts @@ -27,7 +27,16 @@ describe('init', () => { describe('destroy', () => { it('calls closeAndFlush on the analytics instance', async () => { const closeAndFlush = jest.fn().mockResolvedValue(undefined); - const mockAnalytics = { closeAndFlush } as unknown as SegmentAnalyticsMock; + const delivered = () => undefined; + const mockAnalytics: SegmentAnalyticsMock = { + track: delivered, + identify: delivered, + group: delivered, + page: delivered, + screen: delivered, + flush: () => Promise.resolve(), + closeAndFlush, + }; const config = { settings: { diff --git a/packages/server/destinations/segment/src/__tests__/stepExamples.test.ts b/packages/server/destinations/segment/src/__tests__/stepExamples.test.ts index 0e3734df9..5231bdbd7 100644 --- a/packages/server/destinations/segment/src/__tests__/stepExamples.test.ts +++ b/packages/server/destinations/segment/src/__tests__/stepExamples.test.ts @@ -29,21 +29,27 @@ function spyEnv(): { env: Env; collected: () => Captured[] } { const calls: Captured[] = []; const analytics: SegmentAnalyticsMock = { - track: (params) => { + track: (params, callback) => { calls.push(['analytics.track', params]); + callback?.(); }, - identify: (params) => { + identify: (params, callback) => { calls.push(['analytics.identify', params]); + callback?.(); }, - group: (params) => { + group: (params, callback) => { calls.push(['analytics.group', params]); + callback?.(); }, - page: (params) => { + page: (params, callback) => { calls.push(['analytics.page', params]); + callback?.(); }, - screen: (params) => { + screen: (params, callback) => { calls.push(['analytics.screen', params]); + callback?.(); }, + flush: () => Promise.resolve(), closeAndFlush: () => Promise.resolve(), }; @@ -86,6 +92,8 @@ describe('segment server destination -- step examples', () => { }); await elb(event); + // The destination batches by default: send what it holds. + await elb('walker flush'); expect(collected()).toEqual(example.out); }); diff --git a/packages/server/destinations/segment/src/config.ts b/packages/server/destinations/segment/src/config.ts index 652194ff8..0e8cf5a49 100644 --- a/packages/server/destinations/segment/src/config.ts +++ b/packages/server/destinations/segment/src/config.ts @@ -1,14 +1,12 @@ import type { Config, Settings, PartialConfig } from './types'; -import type { Logger } from '@walkeros/core'; +import { StepError } from '@walkeros/core'; -export function getConfig( - partialConfig: PartialConfig = {}, - logger: Logger.Instance, -): Config { - const settings = (partialConfig.settings || {}) as Partial; +export function getConfig(partialConfig: PartialConfig = {}): Config { + const settings: Partial = partialConfig.settings || {}; const { writeKey } = settings; - if (!writeKey) logger.throw('Config settings writeKey missing'); + if (!writeKey) + throw new StepError('Config settings writeKey missing', { code: 'config' }); const settingsConfig: Settings = { ...settings, diff --git a/packages/server/destinations/segment/src/examples/env.ts b/packages/server/destinations/segment/src/examples/env.ts index a8f47322b..3a0c2218a 100644 --- a/packages/server/destinations/segment/src/examples/env.ts +++ b/packages/server/destinations/segment/src/examples/env.ts @@ -1,15 +1,18 @@ -import type { Env, SegmentAnalyticsMock } from '../types'; +import type { Env, SegmentAnalyticsMock, SegmentCallback } from '../types'; -const noop = () => {}; +/** Every call is delivered: its callback answers without a reason. */ +const delivered = (_params: unknown, callback?: SegmentCallback) => + callback?.(); const asyncNoop = () => Promise.resolve(); function createMockAnalytics(): SegmentAnalyticsMock { return { - track: noop, - identify: noop, - group: noop, - page: noop, - screen: noop, + track: delivered, + identify: delivered, + group: delivered, + page: delivered, + screen: delivered, + flush: asyncNoop, closeAndFlush: asyncNoop, }; } diff --git a/packages/server/destinations/segment/src/examples/step.ts b/packages/server/destinations/segment/src/examples/step.ts index 522ee668c..af04c60d6 100644 --- a/packages/server/destinations/segment/src/examples/step.ts +++ b/packages/server/destinations/segment/src/examples/step.ts @@ -381,6 +381,43 @@ export const consentForwarding: SegmentStepExample = { ], }; +/** + * No identity -- neither userId nor anonymousId resolves from the event, so + * Segment cannot take it: the push fails with missing-input instead of + * counting as delivered. One per event name above, so every example's event + * without identity has its declared outcome. + */ +function withoutIdentity(name: string, timestamp: number): SegmentStepExample { + const [entity, action] = name.split(' '); + return { + public: false, + description: `A ${name} event without userId or anonymousId cannot be sent to Segment: the push fails with missing-input instead of counting as delivered.`, + in: { name, entity, action, timestamp }, + out: [], + outcome: { status: 'failed', code: 'missing-input', retryable: false }, + }; +} + +export const noIdentityProduct = withoutIdentity('product view', 1700000110); +export const noIdentityOrder = withoutIdentity('order complete', 1700000111); +export const noIdentityPage = withoutIdentity('page view', 1700000112); +export const noIdentityLogin = withoutIdentity('user login', 1700000113); +export const noIdentityCompany = withoutIdentity('company update', 1700000114); +export const noIdentityScreen = withoutIdentity('screen view', 1700000115); + +/** A silent rule without identify, group, page or screen calls nothing. */ +export const silentRule: SegmentStepExample = { + public: false, + description: + 'A rule with silent: true and no identify, group, page or screen mapping sends nothing to Segment.', + in: getEvent('page view', { + timestamp: 1700000116, + user: { id: 'us3r', session: 's3ss10n' }, + }), + mapping: { silent: true }, + out: [], +}; + /** * Wildcard ignore -- the event matches a mapping rule with ignore: true. * The destination fires zero SDK calls. @@ -395,3 +432,22 @@ export const wildcardIgnored: SegmentStepExample = { mapping: { ignore: true }, out: [], }; + +/** + * The silent group rule of companyGroup on an event without company data: no + * groupId resolves and the rule suppresses the track, so nothing is sent. + */ +export const companyGroupWithoutData: SegmentStepExample = { + public: false, + description: + 'A silent group rule whose groupId finds no data sends nothing: an unmet mapping, not a failure, so the push returns normally.', + in: { + name: 'company update', + entity: 'company', + action: 'update', + timestamp: 1700000120, + user: { id: 'us3r', session: 's3ss10n' }, + }, + mapping: companyGroup.mapping, + out: [], +}; diff --git a/packages/server/destinations/segment/src/index.ts b/packages/server/destinations/segment/src/index.ts index c5fd599e5..13abf108c 100644 --- a/packages/server/destinations/segment/src/index.ts +++ b/packages/server/destinations/segment/src/index.ts @@ -1,6 +1,7 @@ -import type { Destination, Settings, SegmentAnalyticsMock } from './types'; +import type { Destination } from './types'; +import { StepError } from '@walkeros/core'; import { getConfig } from './config'; -import { push } from './push'; +import { push, pushBatch } from './push'; // Types export * as DestinationSegment from './types'; @@ -8,18 +9,15 @@ export * as DestinationSegment from './types'; export const destinationSegment: Destination = { type: 'segment', - config: {}, + // Batched by default, at the SDK's own former defaults (15 events, 10 s). + config: { batch: { size: 15, wait: 10_000 } }, - init({ config: partialConfig, logger, env }) { - const config = getConfig(partialConfig, logger); - const settings = config.settings as Settings; + init({ config: partialConfig, env }) { + const config = getConfig(partialConfig); + const { settings } = config; // Use env.analytics mock if provided (testing), otherwise create real SDK - const envAnalytics = ( - env as { analytics?: SegmentAnalyticsMock } | undefined - )?.analytics; - - if (!envAnalytics) { + if (!env?.analytics) { // Production path: create real Analytics instance try { const { Analytics } = require('@segment/analytics-node'); @@ -36,10 +34,14 @@ export const destinationSegment: Destination = { ...sdkOptions } = settings; - const analytics = new Analytics({ writeKey, ...sdkOptions }); - settings._analytics = analytics; + // Each push and each batch flushes the SDK and waits until Segment + // delivered its calls. + settings._analytics = new Analytics({ writeKey, ...sdkOptions }); } catch (err) { - logger.throw(`Failed to initialize Segment SDK: ${err}`); + throw new StepError('Failed to initialize the Segment SDK', { + code: 'not-ready', + cause: err, + }); } } @@ -48,18 +50,13 @@ export const destinationSegment: Destination = { return config; }, - async push(event, context) { - return await push(event, context); - }, + push, + + pushBatch, async destroy({ config }) { - const settings = (config?.settings || {}) as Settings; - const analytics = settings._analytics; - if (analytics) { - await (analytics as unknown as SegmentAnalyticsMock).closeAndFlush({ - timeout: 5000, - }); - } + const analytics = config?.settings?._analytics; + if (analytics) await analytics.closeAndFlush({ timeout: 5000 }); }, }; diff --git a/packages/server/destinations/segment/src/push.ts b/packages/server/destinations/segment/src/push.ts index 1242de2c6..809dcc800 100644 --- a/packages/server/destinations/segment/src/push.ts +++ b/packages/server/destinations/segment/src/push.ts @@ -1,30 +1,75 @@ -import type { WalkerOS } from '@walkeros/core'; +import type { Collector, Destination, WalkerOS } from '@walkeros/core'; import type { PushFn, + PushBatchFn, Settings, RuntimeState, SegmentAnalyticsMock, + SegmentCallback, + Rule, } from './types'; -import { getMappingValue, isObject, isString, isArray } from '@walkeros/core'; +import { + getMappingValue, + isObject, + isString, + isArray, + isThenable, + StepError, +} from '@walkeros/core'; + +/** + * Hand one call to the SDK. The promise settles once Segment delivered it: + * the SDK answers through the callback, with the reason when the delivery + * failed; an injected client may answer with a promise instead. + */ +function call( + message: string, + send: (done: SegmentCallback) => unknown, +): Promise { + return new Promise((resolve, reject) => { + const fail = (error: unknown) => + reject(new StepError(message, { code: 'transport', cause: error })); + const answer: unknown = send((error) => + error === undefined || error === null ? resolve() : fail(error), + ); + if (isThenable(answer)) answer.then(() => resolve(), fail); + }); +} -export const push: PushFn = async function ( - event, - { config, rule, data, collector, env, logger }, -) { - const settings = config.settings as Settings; - const analytics = settings._analytics as unknown as - | SegmentAnalyticsMock - | undefined; - const envAnalytics = (env as { analytics?: SegmentAnalyticsMock } | undefined) - ?.analytics; - const sdk = envAnalytics || analytics; - - if (!sdk) { - logger.warn('Segment Analytics not initialized'); - return; - } +/** + * Ask the SDK to send what it holds, one flush at a time: the SDK skips a + * flush that overlaps a running one, which would leave calls waiting for its + * flush interval. + */ +function flush(settings: Partial, sdk: SegmentAnalyticsMock) { + const flushing = (settings._flushing ?? Promise.resolve()).then(() => + sdk.flush(), + ); + settings._flushing = flushing.catch(() => undefined); + return flushing; +} + +/** The settled calls of one event: the first failure, if any. */ +async function delivered(calls: Promise[]>) { + for (const result of await calls) + if (result.status === 'rejected') throw result.reason; +} +/** + * Hand the calls of one event to the SDK (identify, group, page, screen, + * track) and answer their deliveries. An event without userId or anonymousId + * cannot be sent and fails with missing-input. + */ +async function enqueue( + event: WalkerOS.Event, + rule: Rule | undefined, + data: Destination.Data, + settings: Partial, + sdk: SegmentAnalyticsMock, + collector: Collector.Instance, +): Promise[]> { const state: RuntimeState = settings._state || {}; + settings._state = state; const mappingSettings = rule?.settings || {}; const timestamp = new Date(event.timestamp || Date.now()); @@ -40,12 +85,10 @@ export const push: PushFn = async function ( ) : undefined; - if (!userId && !anonymousId) { - logger.warn('Segment requires userId or anonymousId; skipping event', { - event: event.name, + if (!userId && !anonymousId) + throw new StepError('Segment needs a userId or anonymousId for the event', { + code: 'missing-input', }); - return; - } // Build identity object for every SDK call const identity: Record = {}; @@ -54,6 +97,7 @@ export const push: PushFn = async function ( // Build consent context const context = buildConsentContext(event, settings); + const calls: Array | undefined> = []; // 2. Identify -- rule-level overrides destination-level const identifyMapping = mappingSettings.identify ?? settings.identify; @@ -61,59 +105,69 @@ export const push: PushFn = async function ( const resolved = await getMappingValue(event, identifyMapping, { collector, }); - if (isObject(resolved)) { - applyIdentify( - sdk, - resolved as Record, - identity, - state, - timestamp, - context, + if (isObject(resolved)) + calls.push( + applyIdentify( + sdk, + resolved as Record, + identity, + state, + timestamp, + context, + ), ); - } } // 3. Group -- rule-level overrides destination-level const groupMapping = mappingSettings.group ?? settings.group; if (groupMapping !== undefined) { - const resolved = await getMappingValue(event, groupMapping, { collector }); - if (isObject(resolved)) { - applyGroup( - sdk, - resolved as Record, - identity, - state, - timestamp, - context, + const resolved = await getMappingValue(event, groupMapping, { + collector, + }); + if (isObject(resolved)) + calls.push( + applyGroup( + sdk, + resolved as Record, + identity, + state, + timestamp, + context, + ), ); - } } - // 4. Page - if (mappingSettings.page !== undefined) { - await applyPage( - sdk, - mappingSettings.page, - event, - identity, - collector, - timestamp, - context, + // 4. Page (the call is not awaited here: it is delivered at the flush) + if (mappingSettings.page !== undefined) + calls.push( + ( + await applyPage( + sdk, + mappingSettings.page, + event, + identity, + collector, + timestamp, + context, + ) + ).sent, ); - } // 5. Screen - if (mappingSettings.screen !== undefined) { - await applyScreen( - sdk, - mappingSettings.screen, - event, - identity, - collector, - timestamp, - context, + if (mappingSettings.screen !== undefined) + calls.push( + ( + await applyScreen( + sdk, + mappingSettings.screen, + event, + identity, + collector, + timestamp, + context, + ) + ).sent, ); - } // 6. Track (unless silent: true) if (rule?.silent !== true) { @@ -128,10 +182,88 @@ export const push: PushFn = async function ( }; if (context) params.context = context; - sdk.track(params); + calls.push(call('Segment track failed', (done) => sdk.track(params, done))); } - settings._state = state; + return calls.filter((sent): sent is Promise => sent !== undefined); +} + +/** The SDK the destination sends through, or a StepError. */ +function sdkOf( + settings: Partial | undefined, + env: { analytics?: SegmentAnalyticsMock } | undefined, +): SegmentAnalyticsMock { + const sdk = env?.analytics || settings?._analytics; + if (!settings || !sdk) + throw new StepError('Segment Analytics not initialized', { + code: 'not-ready', + }); + return sdk; +} + +/** One event: its calls handed to the SDK, sent, and their deliveries awaited. */ +export const push: PushFn = async function ( + event, + { config, rule, data, collector, env }, +) { + const settings = config.settings ?? {}; + const sdk = sdkOf(config.settings, env); + const calls = await enqueue(event, rule, data, settings, sdk, collector); + if (calls.length) { + const settled = Promise.allSettled(calls); + await flush(settings, sdk); + await delivered(settled); + } +}; + +/** + * A flushed `config.batch` batch: every entry's calls handed to the SDK, one + * flush, and each entry's own outcome from its deliveries. An entry that + * cannot be sent fails alone; when nothing landed, the batch throws. + */ +export const pushBatch: PushBatchFn = async function ( + batch, + { config, collector, env }, +) { + const settings = config.settings ?? {}; + const sdk = sdkOf(config.settings, env); + // Each entry's deliveries, or why its calls could not be handed over. + const outcomes: Array<{ + settled?: Promise[]>; + error?: unknown; + }> = []; + + for (const entry of batch.entries) { + try { + const calls = await enqueue( + entry.event, + entry.rule, + entry.data, + settings, + sdk, + collector, + ); + outcomes.push({ settled: Promise.allSettled(calls) }); + } catch (error) { + outcomes.push({ error }); + } + } + + await flush(settings, sdk); + + const failed: Destination.BatchFailure[] = []; + for (const [index, { settled, error }] of outcomes.entries()) { + if (!settled) failed.push({ index, error }); + else + await delivered(settled).catch((reason: unknown) => + failed.push({ index, error: reason }), + ); + } + + // Nothing landed: a whole-batch failure, which the breaker counts. + if (failed.length && failed.length === batch.entries.length) + throw failed[0].error; + if (failed.length) return { failed }; }; function resolveString(value: unknown): string | undefined { @@ -150,7 +282,7 @@ function hashTraits(traits: Record | undefined): string { function buildConsentContext( event: WalkerOS.Event, - settings: Settings, + settings: Partial, ): Record | undefined { const map = settings.consent; if (!map || Object.keys(map).length === 0) return undefined; @@ -177,7 +309,7 @@ function applyIdentify( state: RuntimeState, timestamp: Date, context?: Record, -): void { +): Promise | undefined { const last = state.lastIdentity || {}; // Per-event identify can override userId @@ -193,7 +325,7 @@ function applyIdentify( const userIdChanged = identifyUserId !== last.userId; const traitsChanged = traitsHash !== (last.traitsHash ?? ''); - if (!userIdChanged && !traitsChanged) return; + if (!userIdChanged && !traitsChanged) return undefined; const params: Record = { ...identity, @@ -203,13 +335,20 @@ function applyIdentify( if (traits) params.traits = traits; if (context) params.context = context; - sdk.identify(params); - - state.lastIdentity = { + // Remembered at once, so a batch identifies a user once; forgotten again + // when the delivery fails, so the next event identifies again. + const current = { userId: identifyUserId, anonymousId: identity.anonymousId as string | undefined, traitsHash, }; + state.lastIdentity = current; + return call('Segment identify failed', (done) => + sdk.identify(params, done), + ).catch((error: unknown) => { + if (state.lastIdentity === current) state.lastIdentity = undefined; + throw error; + }); } function applyGroup( @@ -219,9 +358,9 @@ function applyGroup( state: RuntimeState, timestamp: Date, context?: Record, -): void { +): Promise | undefined { const groupId = isString(resolved.groupId) ? resolved.groupId : undefined; - if (!groupId) return; + if (!groupId) return undefined; const last = state.lastGroup || {}; const traits = @@ -232,7 +371,7 @@ function applyGroup( const traitsHash = hashTraits(traits); const groupIdChanged = groupId !== last.groupId; const traitsChanged = traitsHash !== (last.traitsHash ?? ''); - if (!groupIdChanged && !traitsChanged) return; + if (!groupIdChanged && !traitsChanged) return undefined; const params: Record = { ...identity, @@ -242,11 +381,18 @@ function applyGroup( if (traits) params.traits = traits; if (context) params.context = context; - sdk.group(params); - - state.lastGroup = { groupId, traitsHash }; + // Remembered at once, forgotten again when the delivery fails. + const current = { groupId, traitsHash }; + state.lastGroup = current; + return call('Segment group failed', (done) => sdk.group(params, done)).catch( + (error: unknown) => { + if (state.lastGroup === current) state.lastGroup = undefined; + throw error; + }, + ); } +/** The page call, handed to the SDK; its delivery settles at the flush. */ async function applyPage( sdk: SegmentAnalyticsMock, pageMapping: unknown, @@ -255,13 +401,14 @@ async function applyPage( collector: unknown, timestamp: Date, context?: Record, -): Promise { +): Promise<{ sent?: Promise }> { const params: Record = { ...identity, timestamp }; if (pageMapping === true) { if (context) params.context = context; - sdk.page(params); - return; + return { + sent: call('Segment page failed', (done) => sdk.page(params, done)), + }; } const resolved = await getMappingValue( @@ -269,7 +416,7 @@ async function applyPage( pageMapping as Parameters[1], { collector } as Parameters[2], ); - if (!isObject(resolved)) return; + if (!isObject(resolved)) return {}; const r = resolved as { category?: unknown; @@ -283,9 +430,12 @@ async function applyPage( } if (context) params.context = context; - sdk.page(params); + return { + sent: call('Segment page failed', (done) => sdk.page(params, done)), + }; } +/** The screen call, handed to the SDK; its delivery settles at the flush. */ async function applyScreen( sdk: SegmentAnalyticsMock, screenMapping: unknown, @@ -294,13 +444,14 @@ async function applyScreen( collector: unknown, timestamp: Date, context?: Record, -): Promise { +): Promise<{ sent?: Promise }> { const params: Record = { ...identity, timestamp }; if (screenMapping === true) { if (context) params.context = context; - sdk.screen(params); - return; + return { + sent: call('Segment screen failed', (done) => sdk.screen(params, done)), + }; } const resolved = await getMappingValue( @@ -308,7 +459,7 @@ async function applyScreen( screenMapping as Parameters[1], { collector } as Parameters[2], ); - if (!isObject(resolved)) return; + if (!isObject(resolved)) return {}; const r = resolved as { category?: unknown; @@ -322,5 +473,7 @@ async function applyScreen( } if (context) params.context = context; - sdk.screen(params); + return { + sent: call('Segment screen failed', (done) => sdk.screen(params, done)), + }; } diff --git a/packages/server/destinations/segment/src/schemas/settings.ts b/packages/server/destinations/segment/src/schemas/settings.ts index c37853b03..31f830db9 100644 --- a/packages/server/destinations/segment/src/schemas/settings.ts +++ b/packages/server/destinations/segment/src/schemas/settings.ts @@ -1,84 +1,90 @@ import { z } from '@walkeros/core/dev'; -export const SettingsSchema = z.object({ - writeKey: z - .string() - .min(1) - .describe( - 'Segment source write key. Find it in your Segment workspace under Connections > Sources > Settings > API Keys.', - ), - userId: z - .string() - .describe( - 'walkerOS mapping value path to resolve userId from each event (like user.id).', - ) - .optional(), - anonymousId: z - .string() - .describe( - 'walkerOS mapping value path to resolve anonymousId from each event (like user.session).', - ) - .optional(), - host: z - .string() - .url() - .describe( - 'Base URL of Segment API. Set to https://events.eu1.segmentapis.com for EU endpoint.', - ) - .optional(), - path: z.string().describe('API path route (like /v1/batch).').optional(), - flushAt: z - .number() - .int() - .positive() - .describe('Events to enqueue before flushing a batch. Default: 15.') - .optional(), - flushInterval: z - .number() - .int() - .positive() - .describe('Max milliseconds before auto-flush. Default: 10000.') - .optional(), - maxRetries: z - .number() - .int() - .min(0) - .describe('Retry attempts for failed batches. Default: 3.') - .optional(), - httpRequestTimeout: z - .number() - .int() - .positive() - .describe('HTTP request timeout in milliseconds. Default: 10000.') - .optional(), - disable: z - .boolean() - .describe('Completely disable the SDK (no-ops all calls). Default: false.') - .optional(), - identify: z - .unknown() - .describe( - 'Destination-level identity mapping. Resolves to { traits } object. Fires identify() on the first push and re-fires when values change.', - ) - .optional(), - group: z - .unknown() - .describe( - 'Destination-level group mapping. Resolves to { groupId, traits }. Fires group() on the first push and re-fires on change.', - ) - .optional(), - consent: z - .record(z.string(), z.string()) - .describe( - 'Mapping from walkerOS consent keys to Segment categoryPreferences keys. Example: { "marketing": "Advertising", "analytics": "Analytics" }.', - ) - .optional(), - integrations: z - .record(z.string(), z.unknown()) - .describe( - 'Enable/disable downstream Segment destinations. Example: { "All": true, "Mixpanel": false }.', - ) - .optional(), -}); +export const SettingsSchema = z + .object({ + writeKey: z + .string() + .min(1) + .describe( + 'Segment source write key. Find it in your Segment workspace under Connections > Sources > Settings > API Keys.', + ), + userId: z + .string() + .describe( + 'walkerOS mapping value path to resolve userId from each event (like user.id).', + ) + .optional(), + anonymousId: z + .string() + .describe( + 'walkerOS mapping value path to resolve anonymousId from each event (like user.session).', + ) + .optional(), + host: z + .string() + .url() + .describe( + 'Base URL of Segment API. Set to https://events.eu1.segmentapis.com for EU endpoint.', + ) + .optional(), + path: z.string().describe('API path route (like /v1/batch).').optional(), + flushAt: z + .number() + .int() + .positive() + .describe( + 'Events the SDK sends in one request. Each push and each config.batch batch flushes the SDK and waits until Segment delivered its calls. Default: 15.', + ) + .optional(), + flushInterval: z + .number() + .int() + .positive() + .describe('Max milliseconds before auto-flush. Default: 10000.') + .optional(), + maxRetries: z + .number() + .int() + .min(0) + .describe('Retry attempts for failed batches. Default: 3.') + .optional(), + httpRequestTimeout: z + .number() + .int() + .positive() + .describe('HTTP request timeout in milliseconds. Default: 10000.') + .optional(), + disable: z + .boolean() + .describe( + 'Completely disable the SDK (no-ops all calls). Default: false.', + ) + .optional(), + identify: z + .unknown() + .describe( + 'Destination-level identity mapping. Resolves to { traits } object. Fires identify() on the first push and re-fires when values change.', + ) + .optional(), + group: z + .unknown() + .describe( + 'Destination-level group mapping. Resolves to { groupId, traits }. Fires group() on the first push and re-fires on change.', + ) + .optional(), + consent: z + .record(z.string(), z.string()) + .describe( + 'Mapping from walkerOS consent keys to Segment categoryPreferences keys. Example: { "marketing": "Advertising", "analytics": "Analytics" }.', + ) + .optional(), + integrations: z + .record(z.string(), z.unknown()) + .describe( + 'Enable/disable downstream Segment destinations. Example: { "All": true, "Mixpanel": false }.', + ) + .optional(), + }) + .strict(); export type Settings = z.infer; diff --git a/packages/server/destinations/segment/src/types/index.ts b/packages/server/destinations/segment/src/types/index.ts index cf411a324..a488a57eb 100644 --- a/packages/server/destinations/segment/src/types/index.ts +++ b/packages/server/destinations/segment/src/types/index.ts @@ -3,7 +3,6 @@ import type { Destination as CoreDestination, } from '@walkeros/core'; import type { DestinationServer } from '@walkeros/server-core'; -import type { Analytics, AnalyticsSettings } from '@segment/analytics-node'; export interface Settings { /** Segment source write key (required). */ @@ -36,8 +35,10 @@ export interface Settings { /** Completely disable the SDK. Default: false. */ disable?: boolean; /** Runtime state -- not user-facing. Mutated by init/push. */ - _analytics?: Analytics; + _analytics?: SegmentAnalyticsMock; _state?: RuntimeState; + /** The running flush; the next one waits for it. Runtime only. */ + _flushing?: Promise; } export interface RuntimeState { @@ -80,12 +81,32 @@ export interface Env extends DestinationServer.Env { * destination actually calls. Tests can provide this via env.analytics * instead of the real SDK. */ +/** Called once the SDK delivered the call, with the reason if it failed. */ +export type SegmentCallback = (error?: unknown) => void; + export interface SegmentAnalyticsMock { - track: (params: Record) => void; - identify: (params: Record) => void; - group: (params: Record) => void; - page: (params: Record) => void; - screen: (params: Record) => void; + track: ( + params: Record, + callback?: SegmentCallback, + ) => unknown; + identify: ( + params: Record, + callback?: SegmentCallback, + ) => unknown; + group: ( + params: Record, + callback?: SegmentCallback, + ) => unknown; + page: ( + params: Record, + callback?: SegmentCallback, + ) => unknown; + screen: ( + params: Record, + callback?: SegmentCallback, + ) => unknown; + /** Sends what the SDK holds; resolves once it is delivered. */ + flush: () => Promise; closeAndFlush: (opts?: { timeout?: number }) => Promise; } @@ -101,6 +122,7 @@ export type Config = { export type InitFn = DestinationServer.InitFn; export type PushFn = DestinationServer.PushFn; +export type PushBatchFn = CoreDestination.PushBatchFn; export type PartialConfig = DestinationServer.PartialConfig; export type PushEvents = DestinationServer.PushEvents; export type Rule = WalkerOSMapping.Rule; diff --git a/packages/server/destinations/sqlite/src/__tests__/conformance.test.ts b/packages/server/destinations/sqlite/src/__tests__/conformance.test.ts new file mode 100644 index 000000000..430721760 --- /dev/null +++ b/packages/server/destinations/sqlite/src/__tests__/conformance.test.ts @@ -0,0 +1,23 @@ +import path from 'path'; +import { conformance } from '@walkeros/collector/dev'; +import { destinationSQLite } from '..'; +import { examples, schemas } from '../dev'; + +describe('sqlite conformance', () => { + beforeEach(() => { + jest.useRealTimers(); + }); + + it('conforms', async () => { + await conformance('destination', destinationSQLite, { + profile: 'socket', + examples: examples.step, + schema: schemas.settings, + // An in-memory database that creates its table at the first push, so + // the real driver runs without a setup step. + settings: { sqlite: { url: ':memory:', schema: 'auto' } }, + env: { push: examples.env.push, simulation: examples.env.simulation }, + root: path.join(__dirname, '..', '..'), + }); + }); +}); diff --git a/packages/server/destinations/sqlite/src/__tests__/init.test.ts b/packages/server/destinations/sqlite/src/__tests__/init.test.ts index a55c3988d..17b8d2f76 100644 --- a/packages/server/destinations/sqlite/src/__tests__/init.test.ts +++ b/packages/server/destinations/sqlite/src/__tests__/init.test.ts @@ -1,3 +1,4 @@ +import { createEvent, createMockContext } from '@walkeros/core'; import { destinationSQLite } from '..'; import type { Settings, SqliteClient, SqliteClientFactory } from '../types'; @@ -43,7 +44,36 @@ function makeClientSpy(): { return { client, calls }; } +/** The first push, which checks the table and prepares the insert. */ +function firstPush(config: Awaited>) { + if (!config) throw new Error('init returned no config'); + return destinationSQLite.push( + createEvent({ name: 'page view' }), + createMockContext({ config, id: 'sqlite' }), + ); +} + +/** The init the tests call, for the type of its result. */ +const initOf = destinationSQLite.init; + describe('init', () => { + it('stays offline: no query, no CREATE TABLE and no prepare', async () => { + const logger = makeLogger(); + const { client, calls } = makeClientSpy(); + const query = jest.fn(client.query); + client.query = query; + + await destinationSQLite.init({ + config: { settings: { sqlite: { url: ':memory:', schema: 'auto' } } }, + env: { client }, + logger, + id: 'test', + } as never); + + expect(calls).toEqual([]); + expect(query).not.toHaveBeenCalled(); + }); + it('throws when url is missing', async () => { const logger = makeLogger(); await expect( @@ -85,6 +115,8 @@ describe('init', () => { const settings = (result as { settings: Settings }).settings; expect(settings.sqlite._client).toBe(client); expect(settings.sqlite._ownedClient).toBe(false); + + await firstPush(result); expect(calls[0].kind).toBe('execute'); expect(calls[0].sql).toMatch(/^CREATE TABLE IF NOT EXISTS events/); expect(calls.some((c) => c.kind === 'prepare')).toBe(true); @@ -123,7 +155,7 @@ describe('init', () => { const logger = makeLogger(); const { client, calls } = makeClientSpy(); - await destinationSQLite.init({ + const result = await destinationSQLite.init({ config: { settings: { sqlite: { url: ':memory:', schema: 'manual' } }, mapping: { page: { view: { name: 'ok' } } }, @@ -132,9 +164,9 @@ describe('init', () => { logger, id: 'test', } as never); + await firstPush(result); - // Init never issues CREATE TABLE under schema=manual. The probe runs once - // (table presence check) but no execute is called for table creation. + // No CREATE TABLE under schema=manual, and no table probe either. const tableCreate = calls.find( (c) => c.kind === 'execute' && (c.sql ?? '').startsWith('CREATE TABLE'), ); @@ -150,16 +182,20 @@ describe('init', () => { const { client } = makeClientSpy(); // Default makeClientSpy.query returns [], so tableExists -> false. - await expect( - destinationSQLite.init({ - config: { settings: { sqlite: { url: ':memory:' } } }, - env: { client }, - logger, - id: 'sqlite', - } as never), - ).rejects.toThrow( - /SQLite table "events" not found.*walkeros setup destination\.sqlite/, - ); + const result = await destinationSQLite.init({ + config: { settings: { sqlite: { url: ':memory:' } } }, + env: { client }, + logger, + id: 'sqlite', + } as never); + + await expect(firstPush(result)).rejects.toMatchObject({ + name: 'StepError', + code: 'config', + message: expect.stringMatching( + /SQLite table "events" not found.*walkeros setup destination\.sqlite/, + ), + }); }); it('does NOT auto-create when setup: true is configured (table created by setup, not init)', async () => { @@ -174,7 +210,7 @@ describe('init', () => { return []; }; - await destinationSQLite.init({ + const result = await destinationSQLite.init({ config: { settings: { sqlite: { url: ':memory:' } }, setup: true, @@ -183,6 +219,7 @@ describe('init', () => { logger, id: 'sqlite', } as never); + await firstPush(result); const tableCreate = calls.find( (c) => c.kind === 'execute' && (c.sql ?? '').startsWith('CREATE TABLE'), @@ -252,7 +289,7 @@ describe('destroy', () => { }); describe('push error handling', () => { - it('logs and drops on insert failure (does not throw)', async () => { + it('fails the push on an insert failure, without a log line of its own', async () => { const logger = makeLogger(); const failingClient: SqliteClient = { execute: () => Promise.resolve(), @@ -285,11 +322,12 @@ describe('push error handling', () => { } as never, { config, logger, id: 'test' } as never, ), - ).resolves.toBeUndefined(); + ).rejects.toMatchObject({ + name: 'StepError', + code: 'transport', + message: 'SQLite INSERT failed', + }); - expect(logger.error).toHaveBeenCalledWith( - 'SQLite INSERT failed', - expect.any(Object), - ); + expect(logger.error).not.toHaveBeenCalled(); }); }); diff --git a/packages/server/destinations/sqlite/src/config.ts b/packages/server/destinations/sqlite/src/config.ts index a4ba21cd3..91a9ca676 100644 --- a/packages/server/destinations/sqlite/src/config.ts +++ b/packages/server/destinations/sqlite/src/config.ts @@ -6,7 +6,7 @@ import type { SqliteSettings, } from './types'; import type { Logger } from '@walkeros/core'; -import { isObject } from '@walkeros/core'; +import { isObject, StepError } from '@walkeros/core'; export function getConfig( partialConfig: PartialConfig = {}, @@ -16,13 +16,14 @@ export function getConfig( const sqlite: Partial = raw.sqlite ?? ({} as Partial); - if (!sqlite.url) { - logger.throw('Config settings sqlite.url missing'); - } + if (!sqlite.url) + throw new StepError('Config settings sqlite.url missing', { + code: 'config', + }); const sqliteSettings: SqliteSettings = { ...sqlite, - url: sqlite.url as string, + url: sqlite.url, table: sqlite.table ?? 'events', }; @@ -37,8 +38,9 @@ export function getConfig( ); if (sqlite.schema === 'manual' && !partialConfig.mapping) { // Preserve the existing throw for the legacy "manual without mapping" case. - logger.throw( + throw new StepError( 'Config settings sqlite.schema="manual" requires a mapping (no default column mapping will be used).', + { code: 'config' }, ); } sqliteSettings._legacyAutoCreate = sqlite.schema === 'auto'; diff --git a/packages/server/destinations/sqlite/src/examples/step.ts b/packages/server/destinations/sqlite/src/examples/step.ts index 9595b1982..aadcde42b 100644 --- a/packages/server/destinations/sqlite/src/examples/step.ts +++ b/packages/server/destinations/sqlite/src/examples/step.ts @@ -197,3 +197,14 @@ export const ignoredEvent: SqliteStepExample = { mapping: { ignore: true }, out: [], }; + +/** + * Silent rule -- mapping.silent: true writes nothing. + */ +export const silentRule: SqliteStepExample = { + public: false, + description: 'A rule with silent: true inserts no row.', + in: getEvent('page view', { timestamp: 1700000105, id: 'evt-6' }), + mapping: { silent: true }, + out: [], +}; diff --git a/packages/server/destinations/sqlite/src/index.ts b/packages/server/destinations/sqlite/src/index.ts index bfb853e8d..05067b3a1 100644 --- a/packages/server/destinations/sqlite/src/index.ts +++ b/packages/server/destinations/sqlite/src/index.ts @@ -1,9 +1,9 @@ import type { Destination, Env, Settings, SqliteClient } from './types'; import { getConfig, isSqliteEnv } from './config'; import { createClientFromSettings } from './drivers'; -import { buildCreateTableSql, buildInsertSql } from './serialize'; +import { StepError } from '@walkeros/core'; import { push } from './push'; -import { setup, tableExists } from './setup'; +import { setup } from './setup'; // Types re-export export * as DestinationSQLite from './types'; @@ -29,10 +29,9 @@ export const destinationSQLite: Destination = { setup, - async init({ config: partialConfig, logger, env, id }) { + async init({ config: partialConfig, logger, env }) { const config = getConfig(partialConfig, logger); - const settings = config.settings as Settings; - const sqlite = settings.sqlite; + const sqlite = config.settings.sqlite; // Skip creation if a client has already been wired in (testing). if (sqlite._client) return config; @@ -54,52 +53,16 @@ export const destinationSQLite: Destination = { try { client = await createClientFromSettings(sqlite.url, sqlite.authToken); } catch (err) { - logger.throw( - `@walkeros/server-destination-sqlite: driver load failed: ${String(err)}`, - ); - return config; + throw new StepError('SQLite driver load failed', { + code: 'not-ready', + cause: err, + }); } } - // Legacy auto-create path. Only runs when the user opted in via the - // deprecated `settings.sqlite.schema: 'auto'` field. New users run - // `walkeros setup destination.` (see setup.ts) or set `config.setup`. - if (sqlite._legacyAutoCreate) { - try { - await client.execute(buildCreateTableSql(sqlite.table ?? 'events')); - } catch (err) { - logger.throw( - `@walkeros/server-destination-sqlite: CREATE TABLE failed: ${String(err)}`, - ); - return config; - } - } else if (!sqlite._legacySkipProbe) { - // Modern path: assume the table exists. Probe once and hard-fail with - // an actionable message if it does not. Legacy `schema: 'manual'` skips - // this probe (user supplies their own table shape). - let exists = false; - try { - exists = await tableExists(client, sqlite.table ?? 'events'); - } catch (err) { - logger.throw( - `@walkeros/server-destination-sqlite: table probe failed: ${String(err)}`, - ); - return config; - } - if (!exists) { - logger.throw( - `SQLite table "${sqlite.table ?? 'events'}" not found in ${sqlite.url}. ` + - `Run "walkeros setup destination.${id}" to create it.`, - ); - return config; - } - } - - // Cache the prepared insert for per-event calls. + // The table check and the prepared insert run at the first push, so + // init does no query that can fail. sqlite._client = client; - sqlite._runInsert = client.prepare( - buildInsertSql(sqlite.table ?? 'events'), - ); sqlite._ownedClient = !(isSqliteEnv(env) && (env as Env).client); return config; @@ -122,6 +85,7 @@ export const destinationSQLite: Destination = { } finally { sqlite._client = undefined; sqlite._runInsert = undefined; + sqlite._ready = undefined; sqlite._ownedClient = undefined; } } diff --git a/packages/server/destinations/sqlite/src/push.ts b/packages/server/destinations/sqlite/src/push.ts index eecf8f5bf..8ad70b986 100644 --- a/packages/server/destinations/sqlite/src/push.ts +++ b/packages/server/destinations/sqlite/src/push.ts @@ -1,21 +1,94 @@ -import type { PushFn, SqliteSettings } from './types'; -import { isString } from '@walkeros/core'; -import { buildInsertSql, eventToRow } from './serialize'; +import type { PushFn, SqliteClient, SqliteSettings } from './types'; +import { isString, StepError } from '@walkeros/core'; +import { buildCreateTableSql, buildInsertSql, eventToRow } from './serialize'; +import { tableExists } from './setup'; -export const push: PushFn = async function (event, { config, rule, logger }) { - const settings = config.settings as { sqlite?: SqliteSettings } | undefined; - const sqlite: SqliteSettings | undefined = settings?.sqlite; +type RunInsert = (args: ReadonlyArray) => Promise; - if (!sqlite) { - logger.warn('SQLite settings missing'); - return; +/** + * Check the table and prepare the insert, once per client: the legacy + * `schema: 'auto'` creates it, the default path probes it and fails with + * the setup hint when it is missing. A failed attempt is retried at the next + * push. + */ +async function prepareTable( + sqlite: SqliteSettings, + client: SqliteClient, + id: string, +): Promise { + const table = sqlite.table ?? 'events'; + if (sqlite._legacyAutoCreate) { + try { + await client.execute(buildCreateTableSql(table)); + } catch (err) { + throw new StepError('SQLite CREATE TABLE failed', { + code: 'transport', + cause: err, + }); + } + } else if (!sqlite._legacySkipProbe) { + let exists: boolean; + try { + exists = await tableExists(client, table); + } catch (err) { + throw new StepError('SQLite table probe failed', { + code: 'transport', + cause: err, + }); + } + if (!exists) + throw new StepError( + `SQLite table "${table}" not found in ${sqlite.url}. ` + + `Run "walkeros setup destination.${id}" to create it.`, + { code: 'config' }, + ); } + return prepare(client, table); +} - const client = sqlite._client; - if (!client) { - logger.warn('SQLite client not initialized'); - return; +/** A prepared insert; the driver call may answer synchronously or not. */ +async function prepare( + client: SqliteClient, + table: string, +): Promise { + try { + return await client.prepare(buildInsertSql(table)); + } catch (err) { + throw new StepError('SQLite prepare failed', { + code: 'transport', + cause: err, + }); + } +} + +/** The insert of the destination's table, prepared at the first push. */ +function insertOf( + sqlite: SqliteSettings, + client: SqliteClient, + id: string, +): Promise { + if (!sqlite._ready) { + const ready = prepareTable(sqlite, client, id); + ready.catch(() => { + if (sqlite._ready === ready) sqlite._ready = undefined; + }); + sqlite._ready = ready; } + return sqlite._ready; +} + +// A silent rule has no INSERT of its own: nothing to write. +export const push: PushFn = async function (event, context) { + if (!context.rule?.silent) await insert(event, context); +}; + +const insert: PushFn = async function (event, { config, rule, logger, id }) { + const sqlite = config.settings?.sqlite; + const client = sqlite?._client; + if (!sqlite || !client) + throw new StepError('SQLite client not initialized', { + code: 'not-ready', + }); const ruleSettings = rule?.settings ?? {}; const ruleTable = isString(ruleSettings.table) ? ruleSettings.table : ''; @@ -25,13 +98,8 @@ export const push: PushFn = async function (event, { config, rule, logger }) { const effectiveTable = ruleTable || sqlite.table || 'events'; const runInsert = ruleTable && ruleTable !== sqlite.table - ? client.prepare(buildInsertSql(effectiveTable)) - : sqlite._runInsert; - - if (!runInsert) { - logger.warn('SQLite insert statement not prepared'); - return; - } + ? await prepare(client, effectiveTable) + : (sqlite._runInsert ?? (await insertOf(sqlite, client, id))); const row = eventToRow(event); @@ -42,11 +110,12 @@ export const push: PushFn = async function (event, { config, rule, logger }) { try { await runInsert(row); - logger.debug('SQLite INSERT complete', { table: effectiveTable }); } catch (error) { - logger.error('SQLite INSERT failed', { - table: effectiveTable, - error: error instanceof Error ? error.message : String(error), + // The collector reports it; the driver's own message stays the cause. + throw new StepError('SQLite INSERT failed', { + code: 'transport', + cause: error, }); } + logger.debug('SQLite INSERT complete', { table: effectiveTable }); }; diff --git a/packages/server/destinations/sqlite/src/schemas/settings.ts b/packages/server/destinations/sqlite/src/schemas/settings.ts index 16ddabf3d..7c7daea64 100644 --- a/packages/server/destinations/sqlite/src/schemas/settings.ts +++ b/packages/server/destinations/sqlite/src/schemas/settings.ts @@ -1,34 +1,38 @@ import { z } from '@walkeros/core/dev'; -export const SqliteSettingsSchema = z.object({ - url: z - .string() - .min(1) - .describe( - "SQLite connection URL. libsql://, http(s)://, ws(s):// route to libSQL/Turso. Anything else is treated as a local file path via better-sqlite3. Use ':memory:' for an ephemeral in-memory database.", - ), - authToken: z - .string() - .describe( - 'libSQL / Turso auth token. Ignored for better-sqlite3 (local) connections.', - ) - .optional(), - table: z - .string() - .describe('Target table name. Defaults to "events".') - .optional(), - schema: z - .enum(['auto', 'manual']) - .describe( - '[DEPRECATED] Use config.setup instead. "auto" maps to "setup: true" (run `walkeros setup destination.`); "manual" maps to "setup: false". Removed in the next major.', - ) - .optional(), -}); +export const SqliteSettingsSchema = z + .object({ + url: z + .string() + .min(1) + .describe( + "SQLite connection URL. libsql://, http(s)://, ws(s):// route to libSQL/Turso. Anything else is treated as a local file path via better-sqlite3. Use ':memory:' for an ephemeral in-memory database.", + ), + authToken: z + .string() + .describe( + 'libSQL / Turso auth token. Ignored for better-sqlite3 (local) connections.', + ) + .optional(), + table: z + .string() + .describe('Target table name. Defaults to "events".') + .optional(), + schema: z + .enum(['auto', 'manual']) + .describe( + '[DEPRECATED] Use config.setup instead. "auto" maps to "setup: true" (run `walkeros setup destination.`); "manual" maps to "setup: false". Removed in the next major.', + ) + .optional(), + }) + .strict(); -export const SettingsSchema = z.object({ - sqlite: SqliteSettingsSchema.describe( - "SQLite / libSQL configuration (like { url: './events.db' } or { url: 'libsql://my-db.turso.io', authToken: '...' })", - ), -}); +export const SettingsSchema = z + .object({ + sqlite: SqliteSettingsSchema.describe( + "SQLite / libSQL configuration (like { url: './events.db' } or { url: 'libsql://my-db.turso.io', authToken: '...' })", + ), + }) + .strict(); export type Settings = z.infer; diff --git a/packages/server/destinations/sqlite/src/types/index.ts b/packages/server/destinations/sqlite/src/types/index.ts index 74680c90e..4f4cdbba9 100644 --- a/packages/server/destinations/sqlite/src/types/index.ts +++ b/packages/server/destinations/sqlite/src/types/index.ts @@ -63,16 +63,18 @@ export interface SqliteSettings { // Runtime, set during init/setup, not user-facing. _client?: SqliteClient; _runInsert?: (args: ReadonlyArray) => Promise; + /** The table check and insert preparation, run once at the first push. */ + _ready?: Promise<(args: ReadonlyArray) => Promise>; _ownedClient?: boolean; /** - * Internal flag set by the migration shim. When true, init() runs the legacy - * 13-column CREATE TABLE path for backward compatibility. Otherwise init() - * assumes the table already exists (created via `walkeros setup destination.`). + * Internal flag set by the migration shim. When true, the first push runs + * the legacy 13-column CREATE TABLE path for backward compatibility. + * Otherwise the table must exist (created via `walkeros setup destination.`). */ _legacyAutoCreate?: boolean; /** * Internal flag set by the migration shim. When true (legacy `schema: 'manual'`), - * init() skips the modern table-existence probe so the user can bring their own + * the first push skips the table-existence probe so the user can bring their own * table and column shape without hitting the new "table not found" hard-fail. */ _legacySkipProbe?: boolean; diff --git a/packages/server/destinations/tiktok/src/__tests__/conformance.test.ts b/packages/server/destinations/tiktok/src/__tests__/conformance.test.ts new file mode 100644 index 000000000..e8a08db99 --- /dev/null +++ b/packages/server/destinations/tiktok/src/__tests__/conformance.test.ts @@ -0,0 +1,32 @@ +import path from 'path'; +import { conformance, runExamples } from '@walkeros/collector/dev'; +import { destinationTikTok } from '..'; +import { examples, schemas } from '../dev'; + +const settings = { accessToken: 's3cr3t', pixelCode: 'PIXEL_CODE' }; + +describe('tiktok conformance', () => { + beforeEach(() => { + jest.useRealTimers(); + }); + + it('conforms', async () => { + await conformance('destination', destinationTikTok, { + profile: 'http', + examples: examples.step, + schema: schemas.settings, + settings, + root: path.join(__dirname, '..', '..'), + }); + }); + + it('fails an event TikTok rejects in its HTTP 200 body', async () => { + const { rejectedEvent, busy } = examples.step; + await runExamples( + 'destination', + destinationTikTok, + { rejectedEvent, busy }, + { settings }, + ); + }); +}); diff --git a/packages/server/destinations/tiktok/src/__tests__/index.test.ts b/packages/server/destinations/tiktok/src/__tests__/index.test.ts index 72d126652..b9d150c89 100644 --- a/packages/server/destinations/tiktok/src/__tests__/index.test.ts +++ b/packages/server/destinations/tiktok/src/__tests__/index.test.ts @@ -59,7 +59,9 @@ describe('Server Destination TikTok', () => { logger: createMockLogger(), id: 'test-tiktok', }), - ).rejects.toThrow('Config settings pixelCode missing'); + ).rejects.toThrow( + new StepError('Config settings pixelCode missing', { code: 'config' }), + ); }); test('init - missing accessToken throws', async () => { @@ -72,7 +74,7 @@ describe('Server Destination TikTok', () => { logger: createMockLogger(), id: 'test-tiktok', }), - ).rejects.toThrow('Config settings accessToken missing'); + ).rejects.toThrow('Config settings accessToken, pixelCode missing'); }); test('init - valid settings returns config with defaults', async () => { diff --git a/packages/server/destinations/tiktok/src/__tests__/stepExamples.test.ts b/packages/server/destinations/tiktok/src/__tests__/stepExamples.test.ts index 08b882a38..fff8dc30e 100644 --- a/packages/server/destinations/tiktok/src/__tests__/stepExamples.test.ts +++ b/packages/server/destinations/tiktok/src/__tests__/stepExamples.test.ts @@ -21,7 +21,12 @@ describe('Step Examples', () => { }); }); - it.each(Object.entries(examples.step))('%s', async (_name, example) => { + // Non-public examples (silent, failures) run in the conformance test. + const published = Object.entries(examples.step).filter( + ([, example]) => example.public !== false, + ); + + it.each(published)('%s', async (_name, example) => { const event = example.in as WalkerOS.Event; const mapping = example.mapping; diff --git a/packages/server/destinations/tiktok/src/config.ts b/packages/server/destinations/tiktok/src/config.ts index b3f756ab1..bd2189af4 100644 --- a/packages/server/destinations/tiktok/src/config.ts +++ b/packages/server/destinations/tiktok/src/config.ts @@ -1,15 +1,19 @@ import type { Config, Settings, PartialConfig } from './types'; -import type { Logger } from '@walkeros/core'; +import { StepError } from '@walkeros/core'; -export function getConfig( - partialConfig: PartialConfig = {}, - logger: Logger.Instance, -): Config { - const settings = (partialConfig.settings || {}) as Partial; +export function getConfig(partialConfig: PartialConfig = {}): Config { + const settings: Partial = partialConfig.settings || {}; const { accessToken, pixelCode } = settings; - if (!accessToken) logger.throw('Config settings accessToken missing'); - if (!pixelCode) logger.throw('Config settings pixelCode missing'); + if (!accessToken || !pixelCode) { + const missing = [ + ...(accessToken ? [] : ['accessToken']), + ...(pixelCode ? [] : ['pixelCode']), + ]; + throw new StepError(`Config settings ${missing.join(', ')} missing`, { + code: 'config', + }); + } const settingsConfig: Settings = { partner_name: 'walkerOS', diff --git a/packages/server/destinations/tiktok/src/examples/step.ts b/packages/server/destinations/tiktok/src/examples/step.ts index 0ada1762d..14fb148a3 100644 --- a/packages/server/destinations/tiktok/src/examples/step.ts +++ b/packages/server/destinations/tiktok/src/examples/step.ts @@ -313,3 +313,44 @@ export const lead: Flow.StepExample = { ], ], }; + +/** A silent rule sends nothing to TikTok. */ +export const silentRule: Flow.StepExample = { + public: false, + description: 'A rule with silent: true sends no request to TikTok.', + in: getEvent('page view', { + id: 'd1e2f3a4b5c60005', + timestamp: 1700000904000, + }), + mapping: { silent: true }, + out: [], +}; + +/** TikTok rejects the event in an HTTP 200 body. */ +export const rejectedEvent: Flow.StepExample = { + public: false, + description: + 'TikTok answers HTTP 200 with a non-zero code: the event was rejected, so the push fails with transport, not retryable, and the event goes to the DLQ.', + in: getEvent('page view', { + id: 'd1e2f3a4b5c60006', + timestamp: 1700000905000, + }), + respond: { + status: 200, + body: { code: 40002, message: 'Invalid value for field event' }, + }, + outcome: { status: 'failed', code: 'transport', retryable: false }, +}; + +/** TikTok is busy: a server error code in the HTTP 200 body. */ +export const busy: Flow.StepExample = { + public: false, + description: + 'TikTok answers HTTP 200 with a server error code: the push fails with transport, retryable.', + in: getEvent('page view', { + id: 'd1e2f3a4b5c60007', + timestamp: 1700000906000, + }), + respond: { status: 200, body: { code: 50000, message: 'System error' } }, + outcome: { status: 'failed', code: 'transport', retryable: true }, +}; diff --git a/packages/server/destinations/tiktok/src/index.ts b/packages/server/destinations/tiktok/src/index.ts index eaefc31d4..42b05cf2c 100644 --- a/packages/server/destinations/tiktok/src/index.ts +++ b/packages/server/destinations/tiktok/src/index.ts @@ -10,9 +10,8 @@ export const destinationTikTok: Destination = { config: {}, - async init({ config: partialConfig, logger }) { - const config = getConfig(partialConfig, logger); - return config; + async init({ config: partialConfig }) { + return getConfig(partialConfig); }, async push(event, context) { diff --git a/packages/server/destinations/tiktok/src/push.ts b/packages/server/destinations/tiktok/src/push.ts index bc32c2772..d5f000822 100644 --- a/packages/server/destinations/tiktok/src/push.ts +++ b/packages/server/destinations/tiktok/src/push.ts @@ -3,17 +3,28 @@ import type { EventContext, PushFn, RequestBody, + Settings, UserData, Env, } from './types'; import type { Collector, Mapping as WalkerOSMapping } from '@walkeros/core'; -import { createMappingRoot, getMappingValue, isObject } from '@walkeros/core'; +import { + createMappingRoot, + getMappingValue, + isObject, + StepError, +} from '@walkeros/core'; import { sendServer } from '@walkeros/server-core'; import { hashUserData } from './hash'; -export const push: PushFn = async function ( +// A silent rule has no TikTok call of its own: nothing to send. +export const push: PushFn = async function (event, context) { + if (!context.rule?.silent) await send(event, context); +}; + +const send: PushFn = async function ( event, - { config, rule, data, ingest, collector, env, logger }, + { config, data, ingest, collector, env, logger }, ) { const { accessToken, @@ -25,7 +36,7 @@ export const push: PushFn = async function ( partner_name, ip, userAgent, - } = config.settings as import('./types').Settings; + } = config.settings as Settings; const eventData = isObject(data) ? data : {}; const configData = config.data @@ -112,14 +123,34 @@ export const push: PushFn = async function ( }); const sendServerFn = (env as Env)?.sendServer || sendServer; - await sendServerFn(url, JSON.stringify(body), { + const response = await sendServerFn(url, JSON.stringify(body), { headers: { 'Access-Token': accessToken, 'Content-Type': 'application/json', }, }); + checkResponse(response.data); }; +/** + * TikTok answers HTTP 200 with a non-zero `code` when it rejects the event + * (`{ code: 0, message: 'OK' }` on success). A rejection fails the push; the + * body stays on the error. Rate limiting (40100) and server errors (5xxxx) + * can pass on a retry, a rejected event cannot. + */ +function checkResponse(data: unknown): void { + if (!isObject(data) || typeof data.code !== 'number' || data.code === 0) + return; + throw new StepError( + `TikTok Events API rejected the event (code ${data.code})`, + { + code: 'transport', + retryable: data.code === 40100 || data.code >= 50000, + body: data, + }, + ); +} + const DEFAULT_IP: WalkerOSMapping.Value = ['ingest.ip', 'event.user.ip']; const DEFAULT_USER_AGENT: WalkerOSMapping.Value = [ 'ingest.userAgent', diff --git a/packages/server/destinations/tiktok/src/schemas/settings.ts b/packages/server/destinations/tiktok/src/schemas/settings.ts index 434a2664e..a1dbf8588 100644 --- a/packages/server/destinations/tiktok/src/schemas/settings.ts +++ b/packages/server/destinations/tiktok/src/schemas/settings.ts @@ -5,42 +5,44 @@ const InputSchema = z.union([ z.literal(false), ]); -export const SettingsSchema = z.object({ - pixelCode: z - .string() - .min(1) - .describe('TikTok Pixel Code from Events Manager (like C0ABCDEF12345)'), - accessToken: z - .string() - .min(1) - .describe('Events API access token from TikTok Events Manager'), - url: z.string().url().describe('Custom Events API endpoint URL').optional(), - test_event_code: z - .string() - .describe( - 'Test event code for debugging in TikTok Events Manager (like TEST12345)', - ) - .optional(), - doNotHash: z - .array(z.string()) - .describe( - "User data fields to skip hashing (like ['email', 'phone_number'])", - ) - .optional(), - user_data: z - .record(z.string(), z.string()) - .describe("Mapping for user data fields (like { email: 'user.email' })") - .optional(), - partner_name: z - .string() - .describe('Partner name for TikTok attribution') - .optional(), - ip: InputSchema.describe( - 'Client IP, sent as context.ip. Resolves against { ingest, event }. Default: ["ingest.ip", "event.user.ip"]. false switches it off.', - ).optional(), - userAgent: InputSchema.describe( - 'Client user agent, sent as context.user_agent. Resolves against { ingest, event }. Default: ["ingest.userAgent", "event.user.userAgent"]. false switches it off.', - ).optional(), -}); +export const SettingsSchema = z + .object({ + pixelCode: z + .string() + .min(1) + .describe('TikTok Pixel Code from Events Manager (like C0ABCDEF12345)'), + accessToken: z + .string() + .min(1) + .describe('Events API access token from TikTok Events Manager'), + url: z.string().url().describe('Custom Events API endpoint URL').optional(), + test_event_code: z + .string() + .describe( + 'Test event code for debugging in TikTok Events Manager (like TEST12345)', + ) + .optional(), + doNotHash: z + .array(z.string()) + .describe( + "User data fields to skip hashing (like ['email', 'phone_number'])", + ) + .optional(), + user_data: z + .record(z.string(), z.string()) + .describe("Mapping for user data fields (like { email: 'user.email' })") + .optional(), + partner_name: z + .string() + .describe('Partner name for TikTok attribution') + .optional(), + ip: InputSchema.describe( + 'Client IP, sent as context.ip. Resolves against { ingest, event }. Default: ["ingest.ip", "event.user.ip"]. false switches it off.', + ).optional(), + userAgent: InputSchema.describe( + 'Client user agent, sent as context.user_agent. Resolves against { ingest, event }. Default: ["ingest.userAgent", "event.user.userAgent"]. false switches it off.', + ).optional(), + }) + .strict(); export type Settings = z.infer; diff --git a/packages/server/stores/fs/src/__tests__/conformance.test.ts b/packages/server/stores/fs/src/__tests__/conformance.test.ts new file mode 100644 index 000000000..182d3a2d7 --- /dev/null +++ b/packages/server/stores/fs/src/__tests__/conformance.test.ts @@ -0,0 +1,39 @@ +import path from 'path'; +import { conformance } from '@walkeros/collector/dev'; +import { serializeStoreValue } from '@walkeros/core'; +import { storeFsInit } from '..'; +import { examples, schemas } from '../dev'; +import type { FsStoreEnv } from '../types'; + +/** + * File operations without a disk, as the kit's boundary: every read answers + * one stored value, writes and deletes resolve. It lives here, not in the + * package's dev examples: a simulation runs the fs store on the real file + * system, as configured. + */ +const push: FsStoreEnv = { + fs: { + readFile: async () => serializeStoreValue({ stored: true }), + writeFile: async () => undefined, + mkdir: async () => undefined, + unlink: async () => undefined, + }, +}; +const simulation = ['fs.readFile', 'fs.writeFile', 'fs.mkdir', 'fs.unlink']; + +describe('fs store conformance', () => { + beforeEach(() => { + jest.useRealTimers(); + }); + + it('conforms', async () => { + await conformance('store', storeFsInit, { + profile: 'socket', + examples: examples.step, + schema: schemas.settings, + settings: { basePath: './state' }, + env: { push, simulation }, + root: path.join(__dirname, '..', '..'), + }); + }); +}); diff --git a/packages/server/stores/fs/src/__tests__/store.test.ts b/packages/server/stores/fs/src/__tests__/store.test.ts index 02eaec2a5..542b567e9 100644 --- a/packages/server/stores/fs/src/__tests__/store.test.ts +++ b/packages/server/stores/fs/src/__tests__/store.test.ts @@ -75,16 +75,28 @@ describe('FsStore', () => { expect(await store.get('missing')).toBeUndefined(); }); - it('degrades to undefined on an empty file (does not throw)', async () => { + it('throws a StepError for a read that fails, not a miss', async () => { const store = await createStore(tmpDir); - await fs.writeFile(path.join(tmpDir, 'empty'), ''); - await expect(store.get('empty')).resolves.toBeUndefined(); - }); - - it('degrades to undefined on corrupt non-JSON bytes (does not throw)', async () => { + await fs.mkdir(path.join(tmpDir, 'folder')); + await expect(store.get('folder')).rejects.toMatchObject({ + name: 'StepError', + code: 'transport', + message: 'fs store read failed (EISDIR)', + }); + }); + + it.each([ + ['an empty file', ''], + ['corrupt non-JSON bytes', 'not valid json {'], + ])('throws a decode StepError on %s, not a miss', async (_, content) => { const store = await createStore(tmpDir); - await fs.writeFile(path.join(tmpDir, 'corrupt'), 'not valid json {'); - await expect(store.get('corrupt')).resolves.toBeUndefined(); + await fs.writeFile(path.join(tmpDir, 'stored'), content); + await expect(store.get('stored')).rejects.toMatchObject({ + name: 'StepError', + code: 'decode', + retryable: false, + message: 'fs store value could not be decoded', + }); }); it('creates intermediate directories', async () => { diff --git a/packages/server/stores/fs/src/examples/step.ts b/packages/server/stores/fs/src/examples/step.ts index 525defd96..9addd06d5 100644 --- a/packages/server/stores/fs/src/examples/step.ts +++ b/packages/server/stores/fs/src/examples/step.ts @@ -27,3 +27,13 @@ export const pathTraversalRejection: Flow.StepExample = { in: { operation: 'get', key: '../../etc/passwd' }, out: [['get', '../../etc/passwd', undefined]], }; + +/** A read that fails for another reason than a missing file. */ +export const failedRead: Flow.StepExample = { + public: false, + description: + 'The file system fails the read with another error than a missing file (permission denied, a broken disk): get throws a StepError instead of answering a miss.', + in: { operation: 'get', key: 'private.json' }, + respond: { error: 'network' }, + outcome: { status: 'failed', code: 'transport' }, +}; diff --git a/packages/server/stores/fs/src/schemas/settings.ts b/packages/server/stores/fs/src/schemas/settings.ts index bc801a58f..2e706be29 100644 --- a/packages/server/stores/fs/src/schemas/settings.ts +++ b/packages/server/stores/fs/src/schemas/settings.ts @@ -1,12 +1,14 @@ import { z } from '@walkeros/core/dev'; -export const SettingsSchema = z.object({ - basePath: z - .string() - .min(1) - .describe( - 'Root directory for file operations. All keys are resolved relative to this path.', - ), -}); +export const SettingsSchema = z + .object({ + basePath: z + .string() + .min(1) + .describe( + 'Root directory for file operations. All keys are resolved relative to this path.', + ), + }) + .strict(); export type Settings = z.infer; diff --git a/packages/server/stores/fs/src/store.ts b/packages/server/stores/fs/src/store.ts index 2a02df02b..97b5cc8ec 100644 --- a/packages/server/stores/fs/src/store.ts +++ b/packages/server/stores/fs/src/store.ts @@ -1,23 +1,59 @@ import * as fs from 'fs/promises'; import * as path from 'path'; import type { Store } from '@walkeros/core'; -import { serializeStoreValue, deserializeStoreValue } from '@walkeros/core'; -import type { FsStoreSettings } from './types'; +import { + serializeStoreValue, + deserializeStoreValue, + StepError, +} from '@walkeros/core'; +import type { FsStoreEnv, FsStoreFs, FsStoreSettings } from './types'; function isValidKey(key: string): boolean { if (!key || key.startsWith('/') || key.startsWith('\\')) return false; return !key.split(/[/\\]/).includes('..'); } -export const storeFsInit: Store.Init> = ( - context, -) => { - const settings = context.config.settings as FsStoreSettings; +/** The error code of a failed file operation (`ENOENT`, `EACCES`). */ +function errorCode(err: unknown): string | undefined { + // Duck-typed: errors from node:fs may come from another realm. + if (typeof err !== 'object' || err === null || !('code' in err)) return; + return typeof err.code === 'string' ? err.code : undefined; +} + +/** The file (or a directory on its path) does not exist: a miss. */ +function isMissing(err: unknown): boolean { + const code = errorCode(err); + return code === 'ENOENT' || code === 'ENOTDIR'; +} + +/** A failed file operation as a StepError, its message without the key. */ +function storeError(operation: string, err: unknown): StepError { + const code = errorCode(err); + return new StepError( + `fs store ${operation} failed${code ? ` (${code})` : ''}`, + { code: 'transport', cause: err }, + ); +} + +type Types = Store.Types; + +export const storeFsInit: Store.Init = (context) => { + const settings: Partial = context.config.settings || {}; + if (!settings.basePath) + throw new StepError('Config settings basePath missing', { + code: 'config', + }); + const config: Store.Config = { + ...context.config, + settings: { basePath: settings.basePath }, + }; const basePath = path.resolve(settings.basePath); // One mode per instance, decided at init. file:true persists raw bytes // byte-exact; the default structured mode round-trips StoreValue through the // shared core codec (utf8 JSON, binary leaves base64-tagged). const fileMode = context.config.file === true; + // An injected fs (tests, simulate); the file system otherwise. + const files: FsStoreFs = context.env.fs ?? fs; function resolvePath(key: string): string | undefined { if (!isValidKey(key)) { @@ -32,60 +68,67 @@ export const storeFsInit: Store.Init> = ( return { type: 'fs', - config: context.config as Store.Config>, + config, async get(key: string): Promise { const filePath = resolvePath(key); if (!filePath) return undefined; - let bytes: Buffer; + let bytes: Uint8Array; try { - bytes = await fs.readFile(filePath); - } catch { - return undefined; + bytes = await files.readFile(filePath); + } catch (err) { + // A missing file is a miss; any other failed read is reported. + if (isMissing(err)) return undefined; + throw storeError('read', err); } // File mode hands the raw bytes back untouched (Buffer is a Uint8Array, - // a valid StoreValue leaf). Structured mode decodes the utf8-JSON payload - // to a StoreValue (binary leaves become Uint8Array). A corrupt or empty - // payload degrades to a miss rather than throwing a raw SyntaxError. + // a valid StoreValue leaf). Structured mode decodes the utf8-JSON + // payload; a stored value that does not decode (corrupt or empty) is a + // failed read, never a miss. The message names no value. if (fileMode) return bytes; try { return deserializeStoreValue(bytes); } catch (err) { - context.logger.debug('structured decode failed, degrading to miss', { - key, - error: err instanceof Error ? err.message : String(err), + throw new StepError('fs store value could not be decoded', { + code: 'decode', + retryable: false, + cause: err, }); - return undefined; } }, async set(key: string, value: Store.StoreValue): Promise { const filePath = resolvePath(key); if (!filePath) return; - await fs.mkdir(path.dirname(filePath), { recursive: true }); - if (fileMode) { - // Byte-exact passthrough: accept Uint8Array (Buffer included) or string. - if (!(value instanceof Uint8Array) && typeof value !== 'string') { - throw new Error( - `fs store in file mode persists Uint8Array or string values only, received ${typeof value}`, - ); - } - await fs.writeFile(filePath, value); - return; - } + // Byte-exact passthrough in file mode: Uint8Array (Buffer included) or + // string. Structured mode: any StoreValue as a utf8-JSON byte payload. + let bytes: Uint8Array | string; + if (!fileMode) bytes = serializeStoreValue(value); + else if (value instanceof Uint8Array || typeof value === 'string') + bytes = value; + else + throw new StepError( + `fs store in file mode persists Uint8Array or string values only, received ${typeof value}`, + { code: 'config' }, + ); - // Structured mode: any StoreValue serializes to a utf8-JSON byte payload. - await fs.writeFile(filePath, serializeStoreValue(value)); + try { + await files.mkdir(path.dirname(filePath), { recursive: true }); + await files.writeFile(filePath, bytes); + } catch (err) { + throw storeError('write', err); + } }, async delete(key: string): Promise { const filePath = resolvePath(key); if (!filePath) return; try { - await fs.unlink(filePath); - } catch { - /* ignore */ + await files.unlink(filePath); + } catch (err) { + // Deleting a missing file is a no-op; any other failure is reported. + if (!isMissing(err)) throw storeError('delete', err); } }, }; diff --git a/packages/server/stores/fs/src/types.ts b/packages/server/stores/fs/src/types.ts index 6ae58581e..5a4e2dad2 100644 --- a/packages/server/stores/fs/src/types.ts +++ b/packages/server/stores/fs/src/types.ts @@ -2,3 +2,16 @@ export interface FsStoreSettings { /** Root directory for file lookups. All keys resolved relative to this. */ basePath: string; } + +/** The file operations the store uses; `fs/promises` by default. */ +export interface FsStoreFs { + readFile(path: string): Promise; + writeFile(path: string, data: Uint8Array | string): Promise; + mkdir(path: string, options: { recursive: true }): Promise; + unlink(path: string): Promise; +} + +export interface FsStoreEnv { + /** Replaces `fs/promises`, for tests and simulation. */ + fs?: FsStoreFs; +} diff --git a/packages/server/stores/gcs/src/__tests__/conformance.test.ts b/packages/server/stores/gcs/src/__tests__/conformance.test.ts new file mode 100644 index 000000000..90d54fed2 --- /dev/null +++ b/packages/server/stores/gcs/src/__tests__/conformance.test.ts @@ -0,0 +1,29 @@ +import path from 'path'; +import { conformance, runExamples } from '@walkeros/collector/dev'; +import { storeGcsInit } from '..'; +import { examples, schemas } from '../dev'; + +const settings = { bucket: 'my-assets' }; +const env = { push: examples.env.push, simulation: examples.env.simulation }; + +describe('gcs store conformance', () => { + beforeEach(() => { + jest.useRealTimers(); + }); + + it('conforms', async () => { + await conformance('store', storeGcsInit, { + profile: 'socket', + examples: examples.step, + schema: schemas.settings, + settings, + env, + root: path.join(__dirname, '..', '..'), + }); + }); + + it('reports a failed read instead of a miss', async () => { + const { failedRead } = examples.step; + await runExamples('store', storeGcsInit, { failedRead }, { settings, env }); + }); +}); diff --git a/packages/server/stores/gcs/src/__tests__/store.test.ts b/packages/server/stores/gcs/src/__tests__/store.test.ts index 97c54da4d..30df388f7 100644 --- a/packages/server/stores/gcs/src/__tests__/store.test.ts +++ b/packages/server/stores/gcs/src/__tests__/store.test.ts @@ -125,13 +125,27 @@ describe('storeGcsInit', () => { expect(result).toBeUndefined(); }); - it('should return undefined on error', async () => { + it('throws a StepError on a network error, not a miss', async () => { mockFetch.mockRejectedValueOnce(new Error('Network error')); const store = await createStore(); - const result = await store.get('fail.txt'); + await expect(store.get('fail.txt')).rejects.toMatchObject({ + name: 'StepError', + code: 'transport', + message: 'GCS request failed', + }); + }); - expect(result).toBeUndefined(); + it('throws a StepError with the status on a forbidden read', async () => { + mockFetch.mockResolvedValueOnce({ ok: false, status: 403 }); + + const store = await createStore(); + await expect(store.get('secret.txt')).rejects.toMatchObject({ + name: 'StepError', + code: 'transport', + status: 403, + retryable: false, + }); }); it('should prepend prefix to key', async () => { @@ -174,6 +188,20 @@ describe('storeGcsInit', () => { }); }); + describe('set', () => { + it('throws a StepError when the upload is refused', async () => { + mockFetch.mockResolvedValueOnce({ ok: false, status: 403 }); + + const store = await createStore(); + await expect(store.set('a.json', { a: 1 })).rejects.toMatchObject({ + name: 'StepError', + code: 'transport', + status: 403, + message: 'GCS write failed (HTTP 403)', + }); + }); + }); + describe('structured mode (default)', () => { it('uploads serialized JSON with Content-Type application/json', async () => { mockFetch.mockResolvedValueOnce({ ok: true }); @@ -220,16 +248,19 @@ describe('storeGcsInit', () => { expect(result).toEqual({ name: 'logo', body: bytes }); }); - it('degrades to undefined on an empty body (does not throw)', async () => { + it('throws a decode StepError on an empty body, not a miss', async () => { mockFetch.mockResolvedValueOnce({ ok: true, arrayBuffer: () => Promise.resolve(new ArrayBuffer(0)), }); const store = await createStore(); - await expect(store.get('empty')).resolves.toBeUndefined(); + await expect(store.get('empty')).rejects.toMatchObject({ + code: 'decode', + retryable: false, + }); }); - it('degrades to undefined on corrupt non-JSON bytes (does not throw)', async () => { + it('throws a decode StepError on corrupt non-JSON bytes, not a miss', async () => { const corrupt = new TextEncoder().encode('not valid json {'); mockFetch.mockResolvedValueOnce({ ok: true, @@ -242,7 +273,12 @@ describe('storeGcsInit', () => { ), }); const store = await createStore(); - await expect(store.get('corrupt')).resolves.toBeUndefined(); + await expect(store.get('corrupt')).rejects.toMatchObject({ + name: 'StepError', + code: 'decode', + retryable: false, + message: 'GCS value could not be decoded', + }); }); it('should reject path traversal keys', async () => { @@ -327,11 +363,22 @@ describe('storeGcsInit', () => { }); it('should not throw on missing key', async () => { - mockFetch.mockRejectedValueOnce(new Error('Not Found')); + mockFetch.mockResolvedValueOnce({ ok: false, status: 404 }); const store = await createStore(); await expect(store.delete('missing.txt')).resolves.toBeUndefined(); }); + + it('throws a StepError when the delete fails', async () => { + mockFetch.mockResolvedValueOnce({ ok: false, status: 503 }); + + const store = await createStore(); + await expect(store.delete('old.txt')).rejects.toMatchObject({ + code: 'transport', + status: 503, + retryable: true, + }); + }); }); describe('bucket existence hard-fail', () => { @@ -366,10 +413,7 @@ describe('storeGcsInit', () => { // HEAD ok, then both subsequent fetches use the cached result mockFetch .mockResolvedValueOnce({ ok: true, status: 200 }) - .mockResolvedValueOnce({ - ok: true, - arrayBuffer: () => Promise.resolve(new ArrayBuffer(0)), - }) + .mockResolvedValueOnce({ ok: false, status: 404 }) .mockResolvedValueOnce({ ok: true }); const store = await createStore({ bucket: 'memoize-bucket' }); @@ -384,14 +428,11 @@ describe('storeGcsInit', () => { it('does not poison the operation on non-404 HEAD failures', async () => { // HEAD returns 500: treat as non-fatal (cache true, proceed). The GET - // then returns an empty body, which structured mode degrades to a miss - // without throwing. The operation completing is what is under test. + // then answers 404, a miss. The operation completing is what is under + // test. mockFetch .mockResolvedValueOnce({ ok: false, status: 500 }) - .mockResolvedValueOnce({ - ok: true, - arrayBuffer: () => Promise.resolve(new ArrayBuffer(0)), - }); + .mockResolvedValueOnce({ ok: false, status: 404 }); const store = await createStore({ bucket: 'transient-bucket' }); const result = await store.get('a.txt'); diff --git a/packages/server/stores/gcs/src/examples/step.ts b/packages/server/stores/gcs/src/examples/step.ts index 52c75eb5a..b2fa69962 100644 --- a/packages/server/stores/gcs/src/examples/step.ts +++ b/packages/server/stores/gcs/src/examples/step.ts @@ -21,3 +21,13 @@ export const prefixScoping: Flow.StepExample = { }, out: [['get', 'public/walker.js', 'Bytes<...>']], }; + +/** A read that fails is reported, never answered as a miss. */ +export const failedRead: Flow.StepExample = { + public: false, + description: + 'The request fails (no connection, a refused or forbidden read): get throws a StepError instead of answering a miss, and the collector reports it.', + in: { operation: 'get', key: 'walker.js' }, + respond: { error: 'network' }, + outcome: { status: 'failed', code: 'transport' }, +}; diff --git a/packages/server/stores/gcs/src/schemas/settings.ts b/packages/server/stores/gcs/src/schemas/settings.ts index ee8f38872..6255a30a7 100644 --- a/packages/server/stores/gcs/src/schemas/settings.ts +++ b/packages/server/stores/gcs/src/schemas/settings.ts @@ -1,23 +1,25 @@ import { z } from '@walkeros/core/dev'; -export const SettingsSchema = z.object({ - bucket: z.string().min(1).describe('GCS bucket name'), - prefix: z - .string() - .optional() - .describe('Key prefix prepended to all store keys for scoping'), - credentials: z - .union([ - z.string(), - z.object({ - client_email: z.string().describe('Service account email'), - private_key: z.string().describe('RSA private key in PEM format'), - }), - ]) - .optional() - .describe( - 'Service account JSON (string or object). Omit for ADC on Cloud Run/GKE (deprecated: use config.credentials)', - ), -}); +export const SettingsSchema = z + .object({ + bucket: z.string().min(1).describe('GCS bucket name'), + prefix: z + .string() + .optional() + .describe('Key prefix prepended to all store keys for scoping'), + credentials: z + .union([ + z.string(), + z.object({ + client_email: z.string().describe('Service account email'), + private_key: z.string().describe('RSA private key in PEM format'), + }), + ]) + .optional() + .describe( + 'Service account JSON (string or object). Omit for ADC on Cloud Run/GKE (deprecated: use config.credentials)', + ), + }) + .strict(); export type Settings = z.infer; diff --git a/packages/server/stores/gcs/src/store.ts b/packages/server/stores/gcs/src/store.ts index 3287ba071..116d0e530 100644 --- a/packages/server/stores/gcs/src/store.ts +++ b/packages/server/stores/gcs/src/store.ts @@ -1,5 +1,9 @@ import type { Logger, Store } from '@walkeros/core'; -import { serializeStoreValue, deserializeStoreValue } from '@walkeros/core'; +import { + serializeStoreValue, + deserializeStoreValue, + StepError, +} from '@walkeros/core'; import type { GcsStoreSettings, ServiceAccountCredentials, @@ -122,12 +126,18 @@ function assertGcsSettings( typeof settings.bucket !== 'string' || settings.bucket.length === 0 ) { - throw new Error( - 'storeGcsInit: settings.bucket is required (non-empty string)', - ); + throw new StepError('Config settings bucket missing', { code: 'config' }); } } +/** A GCS answer that is not a success, as a StepError with its status. */ +function failedResponse(operation: string, res: Response): StepError { + return new StepError(`GCS ${operation} failed (HTTP ${res.status})`, { + code: 'transport', + status: res.status, + }); +} + /** * Resolve a project ID for the runtime hard-fail message. Uses the same * resolution order as setup but never throws: returns "unknown" when no @@ -183,8 +193,9 @@ async function ensureBucketExists( if (res.status === 404) { const projectId = resolveProjectIdForMessage(credentials); - throw new Error( + throw new StepError( `GCS bucket not found: ${bucket} in project ${projectId}. Run "walkeros setup store.${id}" to create it.`, + { code: 'config' }, ); } if (!res.ok) { @@ -233,6 +244,29 @@ export const storeGcsInit: Store.Init = (context) => { return prefix + key; } + /** One authorized GCS request; a failed token or network call throws. */ + async function request( + url: string, + init: { + method?: string; + headers?: Record; + body?: Uint8Array; + } = {}, + ) { + try { + const token = await getToken(); + return await doFetch(url, { + ...init, + headers: { ...init.headers, Authorization: `Bearer ${token}` }, + }); + } catch (err) { + throw new StepError('GCS request failed', { + code: 'transport', + cause: err, + }); + } + } + const config: Store.Config = { settings, env: context.config.env, @@ -265,31 +299,26 @@ export const storeGcsInit: Store.Init = (context) => { doFetch, ); - let bytes: Buffer; - try { - const token = await getToken(); - const url = `${GCS_BASE}/download/storage/v1/b/${bucket}/o/${encodeURIComponent(gcsKey)}?alt=media`; - const res = await doFetch(url, { - headers: { Authorization: `Bearer ${token}` }, - }); - if (!res.ok) return undefined; - bytes = Buffer.from(await res.arrayBuffer()); - } catch { - return undefined; - } + const res = await request( + `${GCS_BASE}/download/storage/v1/b/${bucket}/o/${encodeURIComponent(gcsKey)}?alt=media`, + ); + // A missing object is a miss; any other failed read is reported. + if (res.status === 404) return undefined; + if (!res.ok) throw failedResponse('read', res); + const bytes = Buffer.from(await res.arrayBuffer()); // File mode hands the raw bytes back untouched (Buffer is a Uint8Array, - // a valid StoreValue leaf). Structured mode decodes the utf8-JSON payload. - // A corrupt or empty payload (GCS can return ok:true with an empty body) - // degrades to a miss rather than throwing a raw SyntaxError. + // a valid StoreValue leaf). Structured mode decodes the utf8-JSON + // payload; a stored value that does not decode (corrupt or empty) is a + // failed read, never a miss. The message names no value. if (fileMode) return bytes; try { return deserializeStoreValue(bytes); } catch (err) { - logger.debug('structured decode failed, degrading to miss', { - key, - error: err instanceof Error ? err.message : String(err), + throw new StepError('GCS value could not be decoded', { + code: 'decode', + retryable: false, + cause: err, }); - return undefined; } }, @@ -306,9 +335,10 @@ export const storeGcsInit: Store.Init = (context) => { } else if (value instanceof Uint8Array) { body = bufferToBody(value); } else { - throw new Error( + throw new StepError( 'storeGcsInit.set: in file mode value must be Uint8Array or string; got ' + typeof value, + { code: 'config' }, ); } contentType = mimeFromKey(key); @@ -327,16 +357,11 @@ export const storeGcsInit: Store.Init = (context) => { doFetch, ); - const token = await getToken(); - const url = `${GCS_BASE}/upload/storage/v1/b/${bucket}/o?uploadType=media&name=${encodeURIComponent(gcsKey)}`; - await doFetch(url, { - method: 'POST', - headers: { - Authorization: `Bearer ${token}`, - 'Content-Type': contentType, - }, - body, - }); + const res = await request( + `${GCS_BASE}/upload/storage/v1/b/${bucket}/o?uploadType=media&name=${encodeURIComponent(gcsKey)}`, + { method: 'POST', headers: { 'Content-Type': contentType }, body }, + ); + if (!res.ok) throw failedResponse('write', res); }, async delete(key: string): Promise { @@ -352,16 +377,12 @@ export const storeGcsInit: Store.Init = (context) => { doFetch, ); - try { - const token = await getToken(); - const url = `${GCS_BASE}/storage/v1/b/${bucket}/o/${encodeURIComponent(gcsKey)}`; - await doFetch(url, { - method: 'DELETE', - headers: { Authorization: `Bearer ${token}` }, - }); - } catch { - /* GCS delete is idempotent */ - } + const res = await request( + `${GCS_BASE}/storage/v1/b/${bucket}/o/${encodeURIComponent(gcsKey)}`, + { method: 'DELETE' }, + ); + // Deleting a missing object is a no-op; any other failure is reported. + if (!res.ok && res.status !== 404) throw failedResponse('delete', res); }, }; }; diff --git a/packages/server/stores/s3/src/__tests__/conformance.test.ts b/packages/server/stores/s3/src/__tests__/conformance.test.ts new file mode 100644 index 000000000..4796de516 --- /dev/null +++ b/packages/server/stores/s3/src/__tests__/conformance.test.ts @@ -0,0 +1,69 @@ +// s3mini ships ESM only, so jest cannot load the real client. This stand-in +// sends requests the way s3mini does (path-style URL, through the `fetch` its +// constructor receives), which is the seam the mock env plugs into. +jest.mock('s3mini', () => ({ + S3mini: jest + .fn() + .mockImplementation( + (config: { endpoint: string; fetch?: typeof fetch }) => { + const send = config.fetch ?? fetch; + const url = (key: string) => `${config.endpoint}/${key}`; + return { + async bucketExists() { + return ( + (await send(config.endpoint, { method: 'HEAD' })).status === 200 + ); + }, + async getObjectArrayBuffer(key: string) { + const res = await send(url(key), { method: 'GET' }); + return res.status === 200 ? res.arrayBuffer() : null; + }, + async putObject(key: string, data: string | Uint8Array) { + return send(url(key), { + method: 'PUT', + body: typeof data === 'string' ? data : new Uint8Array(data), + }); + }, + async deleteObject(key: string) { + const res = await send(url(key), { method: 'DELETE' }); + return res.status === 204; + }, + }; + }, + ), +})); + +import path from 'path'; +import { conformance, runExamples } from '@walkeros/collector/dev'; +import { storeS3Init } from '..'; +import { examples, schemas } from '../dev'; + +const settings = { + bucket: 'my-assets', + endpoint: 'https://s3.eu-west-1.amazonaws.com', + accessKeyId: 'AKID', + secretAccessKey: 'secret', +}; +const env = { push: examples.env.push, simulation: examples.env.simulation }; + +describe('s3 store conformance', () => { + beforeEach(() => { + jest.useRealTimers(); + }); + + it('conforms', async () => { + await conformance('store', storeS3Init, { + profile: 'socket', + examples: examples.step, + schema: schemas.settings, + settings, + env, + root: path.join(__dirname, '..', '..'), + }); + }); + + it('reports a failed read instead of a miss', async () => { + const { failedRead } = examples.step; + await runExamples('store', storeS3Init, { failedRead }, { settings, env }); + }); +}); diff --git a/packages/server/stores/s3/src/__tests__/store.test.ts b/packages/server/stores/s3/src/__tests__/store.test.ts index 7393adaa5..4add3b7e2 100644 --- a/packages/server/stores/s3/src/__tests__/store.test.ts +++ b/packages/server/stores/s3/src/__tests__/store.test.ts @@ -65,7 +65,7 @@ describe('storeS3Init', () => { mockBucketExists.mockReset(); mockCreateBucket.mockReset(); // Default to bucket existing so existing tests are unaffected by the - // new init-time bucket-exists guard. + // first-operation bucket-exists guard. mockBucketExists.mockResolvedValue(true); }); @@ -77,11 +77,26 @@ describe('storeS3Init', () => { expect(store.delete).toBeDefined(); }); + it('checks the bucket at the first operation, not at init', async () => { + const store = await createStore(); + expect(mockBucketExists).not.toHaveBeenCalled(); + + mockGetObjectArrayBuffer.mockResolvedValue(null); + await store.get('a'); + await store.get('b'); + expect(mockBucketExists).toHaveBeenCalledTimes(1); + }); + it('should throw an actionable error when the bucket does not exist', async () => { mockBucketExists.mockResolvedValue(false); - await expect(createStore()).rejects.toThrow( - /S3 bucket not found: my-bucket.*walkeros setup store\.test-s3/, - ); + const store = await createStore(); + await expect(store.get('walker.js')).rejects.toMatchObject({ + name: 'StepError', + code: 'config', + message: expect.stringMatching( + /S3 bucket not found: my-bucket.*walkeros setup store\.test-s3/, + ), + }); }); // Connect the put/get mocks into a single in-memory backing store so a @@ -114,13 +129,17 @@ describe('storeS3Init', () => { expect(result).toBeUndefined(); }); - it('should return undefined on error', async () => { - mockGetObjectArrayBuffer.mockRejectedValue(new Error('NoSuchKey')); + it('throws a StepError for a failed read, not a miss', async () => { + mockGetObjectArrayBuffer.mockRejectedValue( + new Error('Request failed with status 403: AccessDenied'), + ); const store = await createStore(); - const result = await store.get('missing.txt'); - - expect(result).toBeUndefined(); + await expect(store.get('private.txt')).rejects.toMatchObject({ + name: 'StepError', + code: 'transport', + message: 'S3 read failed', + }); }); it('should prepend prefix to key', async () => { @@ -156,6 +175,19 @@ describe('storeS3Init', () => { }); }); + describe('set', () => { + it('throws a StepError when the upload fails', async () => { + mockPutObject.mockRejectedValue(new Error('Request failed: 403')); + + const store = await createStore(); + await expect(store.set('a.json', { a: 1 })).rejects.toMatchObject({ + name: 'StepError', + code: 'transport', + message: 'S3 write failed', + }); + }); + }); + describe('structured mode (default)', () => { it('round-trips an object and stores it as application/json', async () => { wireBackingStore(); @@ -187,13 +219,16 @@ describe('storeS3Init', () => { expect(result).toEqual({ name: 'logo', body: bytes }); }); - it('degrades to undefined on an empty object (does not throw)', async () => { + it('throws a decode StepError on an empty object, not a miss', async () => { mockGetObjectArrayBuffer.mockResolvedValue(new ArrayBuffer(0)); const store = await createStore(); - await expect(store.get('empty')).resolves.toBeUndefined(); + await expect(store.get('empty')).rejects.toMatchObject({ + code: 'decode', + retryable: false, + }); }); - it('degrades to undefined on corrupt non-JSON bytes (does not throw)', async () => { + it('throws a decode StepError on corrupt non-JSON bytes, not a miss', async () => { const corrupt = Buffer.from('not valid json {'); mockGetObjectArrayBuffer.mockResolvedValue( corrupt.buffer.slice( @@ -202,7 +237,12 @@ describe('storeS3Init', () => { ), ); const store = await createStore(); - await expect(store.get('corrupt')).resolves.toBeUndefined(); + await expect(store.get('corrupt')).rejects.toMatchObject({ + name: 'StepError', + code: 'decode', + retryable: false, + message: 'S3 value could not be decoded', + }); }); it('reject path traversal keys', async () => { @@ -277,11 +317,14 @@ describe('storeS3Init', () => { expect(mockDeleteObject).toHaveBeenCalledWith('old.txt'); }); - it('should not throw on missing key', async () => { - mockDeleteObject.mockRejectedValue(new Error('NoSuchKey')); + it('throws a StepError for a failed delete', async () => { + mockDeleteObject.mockRejectedValue(new Error('Request failed: 503')); const store = await createStore(); - await expect(store.delete('missing.txt')).resolves.toBeUndefined(); + await expect(store.delete('old.txt')).rejects.toMatchObject({ + code: 'transport', + message: 'S3 delete failed', + }); }); }); diff --git a/packages/server/stores/s3/src/examples/step.ts b/packages/server/stores/s3/src/examples/step.ts index 3eb029f0f..d6c65acfd 100644 --- a/packages/server/stores/s3/src/examples/step.ts +++ b/packages/server/stores/s3/src/examples/step.ts @@ -20,3 +20,13 @@ export const prefixScoping: Flow.StepExample = { }, out: [['get', 'public/walker.js', 'Bytes<...>']], }; + +/** A read that fails is reported, never answered as a miss. */ +export const failedRead: Flow.StepExample = { + public: false, + description: + 'The request fails (no connection, a refused or forbidden read): get throws a StepError instead of answering a miss, and the collector reports it.', + in: { operation: 'get', key: 'walker.js' }, + respond: { error: 'network' }, + outcome: { status: 'failed', code: 'transport' }, +}; diff --git a/packages/server/stores/s3/src/schemas/settings.ts b/packages/server/stores/s3/src/schemas/settings.ts index a06eefcde..fe312704b 100644 --- a/packages/server/stores/s3/src/schemas/settings.ts +++ b/packages/server/stores/s3/src/schemas/settings.ts @@ -1,20 +1,22 @@ import { z } from '@walkeros/core/dev'; -export const SettingsSchema = z.object({ - bucket: z.string().min(1).describe('S3 bucket name'), - endpoint: z - .string() - .url() - .describe( - 'S3-compatible endpoint URL (like https://s3.eu-west-1.amazonaws.com)', - ), - accessKeyId: z.string().min(1).describe('S3 access key ID'), - secretAccessKey: z.string().min(1).describe('S3 secret access key'), - region: z.string().default('auto').describe('AWS region for SigV4 signing'), - prefix: z - .string() - .optional() - .describe('Key prefix prepended to all store keys for scoping'), -}); +export const SettingsSchema = z + .object({ + bucket: z.string().min(1).describe('S3 bucket name'), + endpoint: z + .string() + .url() + .describe( + 'S3-compatible endpoint URL (like https://s3.eu-west-1.amazonaws.com)', + ), + accessKeyId: z.string().min(1).describe('S3 access key ID'), + secretAccessKey: z.string().min(1).describe('S3 secret access key'), + region: z.string().default('auto').describe('AWS region for SigV4 signing'), + prefix: z + .string() + .optional() + .describe('Key prefix prepended to all store keys for scoping'), + }) + .strict(); export type Settings = z.infer; diff --git a/packages/server/stores/s3/src/store.ts b/packages/server/stores/s3/src/store.ts index 0a56291d8..5c9a19f11 100644 --- a/packages/server/stores/s3/src/store.ts +++ b/packages/server/stores/s3/src/store.ts @@ -1,5 +1,9 @@ import { S3mini } from 's3mini'; -import { serializeStoreValue, deserializeStoreValue } from '@walkeros/core'; +import { + serializeStoreValue, + deserializeStoreValue, + StepError, +} from '@walkeros/core'; import type { Store } from '@walkeros/core'; import type { S3StoreSettings, Types } from './types'; @@ -53,36 +57,30 @@ function buildEndpoint(endpoint: string, bucket: string): string { function assertSettings( settings: Partial | undefined, ): asserts settings is S3StoreSettings { - if ( - !settings || - typeof settings.bucket !== 'string' || - settings.bucket.length === 0 - ) { - throw new Error( - 'storeS3Init: settings.bucket is required (non-empty string)', - ); - } - if (typeof settings.endpoint !== 'string' || settings.endpoint.length === 0) { - throw new Error( - 'storeS3Init: settings.endpoint is required (non-empty string)', - ); - } - if ( - typeof settings.accessKeyId !== 'string' || - settings.accessKeyId.length === 0 - ) { - throw new Error( - 'storeS3Init: settings.accessKeyId is required (non-empty string)', - ); - } - if ( - typeof settings.secretAccessKey !== 'string' || - settings.secretAccessKey.length === 0 - ) { - throw new Error( - 'storeS3Init: settings.secretAccessKey is required (non-empty string)', - ); - } + // The settings S3 cannot work without, each a non-empty string. + const required: Record = { + bucket: settings?.bucket, + endpoint: settings?.endpoint, + accessKeyId: settings?.accessKeyId, + secretAccessKey: settings?.secretAccessKey, + }; + const missing = Object.keys(required).filter((key) => { + const value = required[key]; + return typeof value !== 'string' || value.length === 0; + }); + if (missing.length) + throw new StepError(`Config settings ${missing.join(', ')} missing`, { + code: 'config', + }); +} + +/** A failed S3 call as a StepError; the client's own message stays the cause. */ +function storeError(operation: string, err: unknown): StepError { + if (err instanceof StepError) return err; + return new StepError(`S3 ${operation} failed`, { + code: 'transport', + cause: err, + }); } export const storeS3Init: Store.Init = async (context) => { @@ -113,19 +111,32 @@ export const storeS3Init: Store.Init = async (context) => { return prefix + key; } - // Hard-fail with an actionable message if the bucket is missing. - // Mirrors the BigQuery init-time hard-fail pattern. - const exists = await client.bucketExists(); - if (!exists) { - const setupId = context.id ?? 's3'; - const errorMsg = - `S3 bucket not found: ${settings.bucket} at ${settings.endpoint}. ` + - `Run "walkeros setup store.${setupId}" to create it.`; - context.logger.error(errorMsg, { - bucket: settings.bucket, - endpoint: settings.endpoint, - }); - throw new Error(errorMsg); + // The bucket is checked once, at the first operation, so init stays + // offline. A missing bucket fails with an actionable message (S3 answers a + // missing object and a missing bucket alike); a failed check is retried. + let checked: Promise | undefined; + function ensureBucket(): Promise { + if (!checked) { + const checking = client.bucketExists().then( + (exists) => { + if (exists) return; + const setupId = context.id ?? 's3'; + throw new StepError( + `S3 bucket not found: ${settings.bucket} at ${settings.endpoint}. ` + + `Run "walkeros setup store.${setupId}" to create it.`, + { code: 'config' }, + ); + }, + (err: unknown) => { + throw storeError('bucket check', err); + }, + ); + checking.catch(() => { + if (checked === checking) checked = undefined; + }); + checked = checking; + } + return checked; } return { @@ -143,27 +154,29 @@ export const storeS3Init: Store.Init = async (context) => { const s3Key = resolveKey(key); if (!s3Key) return undefined; - let bytes: Buffer; + await ensureBucket(); + let arrayBuffer: ArrayBuffer | null; try { - const arrayBuffer = await client.getObjectArrayBuffer(s3Key); - if (!arrayBuffer) return undefined; - bytes = Buffer.from(arrayBuffer); - } catch { - return undefined; + arrayBuffer = await client.getObjectArrayBuffer(s3Key); + } catch (err) { + throw storeError('read', err); } + // A missing object is a miss; any other failed read threw above. + if (!arrayBuffer) return undefined; + const bytes = Buffer.from(arrayBuffer); // File mode hands the raw bytes back untouched (Buffer is a Uint8Array, - // a valid StoreValue leaf). Structured mode decodes the utf8-JSON payload. - // A corrupt or empty payload (e.g. a 0-byte object) degrades to a miss - // rather than throwing a raw SyntaxError. + // a valid StoreValue leaf). Structured mode decodes the utf8-JSON + // payload; a stored value that does not decode (corrupt or empty) is a + // failed read, never a miss. The message names no value. if (fileMode) return bytes; try { return deserializeStoreValue(bytes); } catch (err) { - context.logger.debug('structured decode failed, degrading to miss', { - key, - error: err instanceof Error ? err.message : String(err), + throw new StepError('S3 value could not be decoded', { + code: 'decode', + retryable: false, + cause: err, }); - return undefined; } }, @@ -171,34 +184,43 @@ export const storeS3Init: Store.Init = async (context) => { const s3Key = resolveKey(key); if (!s3Key) return; - if (fileMode) { - // Byte-exact passthrough: accept Uint8Array (Buffer included) or string. - if (!(value instanceof Uint8Array) && typeof value !== 'string') { - throw new Error( - 'storeS3Init.set: in file mode value must be Uint8Array or string; got ' + - typeof value, - ); - } - await client.putObject(s3Key, value, mimeFromKey(key)); - return; + // File mode: byte-exact passthrough of Uint8Array (Buffer included) + // or string, with a real mime. Structured mode: utf8-JSON bytes. + let body: Uint8Array | string; + let contentType: string; + if (!fileMode) { + body = serializeStoreValue(value); + contentType = JSON_CONTENT_TYPE; + } else if (value instanceof Uint8Array || typeof value === 'string') { + body = value; + contentType = mimeFromKey(key); + } else { + throw new StepError( + 'storeS3Init.set: in file mode value must be Uint8Array or string; got ' + + typeof value, + { code: 'config' }, + ); } - // Structured mode: serialize to utf8-JSON bytes, stored as JSON. - await client.putObject( - s3Key, - serializeStoreValue(value), - JSON_CONTENT_TYPE, - ); + await ensureBucket(); + try { + await client.putObject(s3Key, body, contentType); + } catch (err) { + throw storeError('write', err); + } }, async delete(key: string): Promise { const s3Key = resolveKey(key); if (!s3Key) return; + // S3 answers a delete of a missing object with success; any other + // failure is reported. + await ensureBucket(); try { await client.deleteObject(s3Key); - } catch { - /* S3 delete is idempotent */ + } catch (err) { + throw storeError('delete', err); } }, }; diff --git a/packages/server/stores/sheets/src/__tests__/conformance.test.ts b/packages/server/stores/sheets/src/__tests__/conformance.test.ts new file mode 100644 index 000000000..42e3d0ea4 --- /dev/null +++ b/packages/server/stores/sheets/src/__tests__/conformance.test.ts @@ -0,0 +1,34 @@ +import path from 'path'; +import { conformance, runExamples } from '@walkeros/collector/dev'; +import { storeSheetsInit } from '..'; +import { examples, schemas } from '../dev'; + +const settings = { id: 'spreadsheet-id-123' }; +const env = { push: examples.env.push, simulation: examples.env.simulation }; + +describe('sheets store conformance', () => { + beforeEach(() => { + jest.useRealTimers(); + }); + + it('conforms', async () => { + await conformance('store', storeSheetsInit, { + profile: 'socket', + examples: examples.step, + schema: schemas.settings, + settings, + env, + root: path.join(__dirname, '..', '..'), + }); + }); + + it('reports a failed read instead of a miss', async () => { + const { failedRead } = examples.step; + await runExamples( + 'store', + storeSheetsInit, + { failedRead }, + { settings, env }, + ); + }); +}); diff --git a/packages/server/stores/sheets/src/__tests__/store.test.ts b/packages/server/stores/sheets/src/__tests__/store.test.ts index c2eecffd9..51b0cb623 100644 --- a/packages/server/stores/sheets/src/__tests__/store.test.ts +++ b/packages/server/stores/sheets/src/__tests__/store.test.ts @@ -111,7 +111,7 @@ describe('storeSheetsInit', () => { }); describe('init', () => { - it('returns a store instance with type "sheets" and reads the key column', async () => { + it('returns a store instance and reads the key column at the first operation', async () => { const { calls, restore } = installFetch([ probeOk(), jsonResponse({ @@ -125,6 +125,10 @@ describe('storeSheetsInit', () => { expect(store.get).toBeDefined(); expect(store.set).toBeDefined(); expect(store.delete).toBeDefined(); + // Init stays offline. + expect(calls).toHaveLength(0); + + await store.get('unknown'); expect(calls).toHaveLength(2); expect(calls[0].url).toBe( 'https://sheets.googleapis.com/v4/spreadsheets/spreadsheet-id-123?fields=spreadsheetId', @@ -140,7 +144,8 @@ describe('storeSheetsInit', () => { it('hard-fails with an actionable error when the spreadsheet is missing', async () => { const { restore } = installFetch([{ status: 404 }]); try { - await expect(storeSheetsInit(createCtx())).rejects.toThrow( + const store = await storeSheetsInit(createCtx()); + await expect(store.get('alice')).rejects.toThrow( /Spreadsheet not found.*walkeros setup store\.crm/, ); } finally { @@ -197,12 +202,12 @@ describe('storeSheetsInit', () => { ]); try { const store = await storeSheetsInit(createCtx()); - const initialCallCount = calls.length; const result = await store.get('unknown'); expect(result).toBeUndefined(); - expect(calls).toHaveLength(initialCallCount); + // The spreadsheet check and the key column read, no cell read. + expect(calls).toHaveLength(2); } finally { restore(); } @@ -223,22 +228,60 @@ describe('storeSheetsInit', () => { } }); - it('returns undefined when JSON parse fails', async () => { + it('throws a StepError when the cell read fails, not a miss', async () => { + const { restore } = installFetch([ + probeOk(), + jsonResponse({ values: [['alice']] }), + { status: 403 }, + ]); + try { + const store = await storeSheetsInit(createCtx()); + await expect(store.get('alice')).rejects.toMatchObject({ + name: 'StepError', + code: 'transport', + status: 403, + retryable: false, + message: 'Sheets read failed (HTTP 403)', + }); + } finally { + restore(); + } + }); + + it('throws a StepError when the key column cannot be read, and retries it', async () => { + const { calls, restore } = installFetch([ + probeOk(), + { status: 503 }, + jsonResponse({ values: [['alice']] }), + ]); + try { + const store = await storeSheetsInit(createCtx()); + await expect(store.get('alice')).rejects.toMatchObject({ + code: 'transport', + status: 503, + }); + await expect(store.get('unknown')).resolves.toBeUndefined(); + expect(calls).toHaveLength(3); + } finally { + restore(); + } + }); + + it('throws a decode StepError when the cell is not JSON, not a miss', async () => { const { restore } = installFetch([ probeOk(), jsonResponse({ values: [['alice']] }), jsonResponse({ values: [['not json']] }), ]); try { - const ctx = createCtx(); - const store = await storeSheetsInit(ctx); - const result = await store.get('alice'); + const store = await storeSheetsInit(createCtx()); - expect(result).toBeUndefined(); - expect(ctx.logger.debug).toHaveBeenCalledWith( - expect.stringMatching(/JSON parse failed/), - expect.objectContaining({ key: 'alice' }), - ); + await expect(store.get('alice')).rejects.toMatchObject({ + name: 'StepError', + code: 'decode', + retryable: false, + message: 'Sheets value could not be decoded', + }); } finally { restore(); } @@ -387,11 +430,11 @@ describe('storeSheetsInit', () => { ]); try { const store = await storeSheetsInit(createCtx()); - const initial = calls.length; await store.delete('unknown'); - expect(calls).toHaveLength(initial); + // The spreadsheet check and the key column read, no cell write. + expect(calls).toHaveLength(2); } finally { restore(); } @@ -511,7 +554,8 @@ describe('storeSheetsInit', () => { jsonResponse({ values: [['alice']] }), ]); try { - await storeSheetsInit(createCtx({ sheet: 'My Sheet' })); + const store = await storeSheetsInit(createCtx({ sheet: 'My Sheet' })); + await store.get('unknown'); expect(calls[1].url).toContain(encodeURIComponent("'My Sheet'!A2:A")); } finally { diff --git a/packages/server/stores/sheets/src/examples/step.ts b/packages/server/stores/sheets/src/examples/step.ts index 0566ea442..bef377039 100644 --- a/packages/server/stores/sheets/src/examples/step.ts +++ b/packages/server/stores/sheets/src/examples/step.ts @@ -17,3 +17,13 @@ export const writeWithServiceAccount: Flow.StepExample = { in: { operation: 'set', key: 'bob', value: { tier: 'silver' } }, out: [['set', 'bob', '{ tier: "silver" }']], }; + +/** A read that fails is reported, never answered as a miss. */ +export const failedRead: Flow.StepExample = { + public: false, + description: + 'The request fails (no connection, a refused or forbidden read): get throws a StepError instead of answering a miss, and the collector reports it.', + in: { operation: 'get', key: 'alice' }, + respond: { error: 'network' }, + outcome: { status: 'failed', code: 'transport' }, +}; diff --git a/packages/server/stores/sheets/src/schemas/settings.ts b/packages/server/stores/sheets/src/schemas/settings.ts index 23d7554bc..e4f508d84 100644 --- a/packages/server/stores/sheets/src/schemas/settings.ts +++ b/packages/server/stores/sheets/src/schemas/settings.ts @@ -1,40 +1,42 @@ import { z } from '@walkeros/core/dev'; -export const SettingsSchema = z.object({ - id: z - .string() - .min(1) - .describe('Spreadsheet ID, the segment between /d/ and /edit in the URL'), - sheet: z - .string() - .default('Sheet1') - .describe('Sheet (tab) name within the spreadsheet'), - key: z - .string() - .default('A') - .describe('Column letter for keys (the lookup column)'), - value: z - .string() - .default('B') - .describe('Column letter for values (JSON-serialized blob)'), - headerRows: z - .number() - .int() - .min(0) - .default(1) - .describe('Number of header rows to skip when reading the key column'), - credentials: z - .union([ - z.string(), - z.object({ - client_email: z.string().describe('Service account email'), - private_key: z.string().describe('RSA private key in PEM format'), - }), - ]) - .optional() - .describe( - 'Service account JSON (string or object). Omit for ADC on Cloud Run/GKE (deprecated: use config.credentials)', - ), -}); +export const SettingsSchema = z + .object({ + id: z + .string() + .min(1) + .describe('Spreadsheet ID, the segment between /d/ and /edit in the URL'), + sheet: z + .string() + .default('Sheet1') + .describe('Sheet (tab) name within the spreadsheet'), + key: z + .string() + .default('A') + .describe('Column letter for keys (the lookup column)'), + value: z + .string() + .default('B') + .describe('Column letter for values (JSON-serialized blob)'), + headerRows: z + .number() + .int() + .min(0) + .default(1) + .describe('Number of header rows to skip when reading the key column'), + credentials: z + .union([ + z.string(), + z.object({ + client_email: z.string().describe('Service account email'), + private_key: z.string().describe('RSA private key in PEM format'), + }), + ]) + .optional() + .describe( + 'Service account JSON (string or object). Omit for ADC on Cloud Run/GKE (deprecated: use config.credentials)', + ), + }) + .strict(); export type Settings = z.infer; diff --git a/packages/server/stores/sheets/src/store.ts b/packages/server/stores/sheets/src/store.ts index 4d9915fcc..d6c8133a7 100644 --- a/packages/server/stores/sheets/src/store.ts +++ b/packages/server/stores/sheets/src/store.ts @@ -1,4 +1,5 @@ import type { Logger, Store } from '@walkeros/core'; +import { StepError } from '@walkeros/core'; import type { ServiceAccountCredentials, SheetsStoreSettings, @@ -76,12 +77,45 @@ function assertSheetsSettings( typeof settings.id !== 'string' || settings.id.length === 0 ) { - throw new Error( - 'storeSheetsInit: settings.id (spreadsheet ID) is required (non-empty string)', - ); + throw new StepError('Config settings id (spreadsheet ID) missing', { + code: 'config', + }); } } +/** One authorized Sheets request; a failed token or network call throws. */ +async function request( + getToken: TokenProvider, + doFetch: typeof fetch, + url: string, + init: { + method?: string; + headers?: Record; + body?: string; + } = {}, +): Promise { + try { + const token = await getToken(); + return await doFetch(url, { + ...init, + headers: { ...init.headers, Authorization: `Bearer ${token}` }, + }); + } catch (err) { + throw new StepError('Sheets request failed', { + code: 'transport', + cause: err, + }); + } +} + +/** A Sheets answer that is not a success, as a StepError with its status. */ +function failedResponse(operation: string, res: Response): StepError { + return new StepError(`Sheets ${operation} failed (HTTP ${res.status})`, { + code: 'transport', + status: res.status, + }); +} + async function ensureSpreadsheetExists( spreadsheetId: string, storeId: string, @@ -112,8 +146,9 @@ async function ensureSpreadsheetExists( } if (res.status === 404) { - throw new Error( + throw new StepError( `Spreadsheet not found: ${spreadsheetId}. Run "walkeros setup store.${storeId}" to ensure the sheet exists and is shared with the service account.`, + { code: 'config' }, ); } if (!res.ok) { @@ -161,17 +196,10 @@ async function buildKeyIndex( getToken: TokenProvider, doFetch: typeof fetch, ): Promise> { - const token = await getToken(); const range = buildColumnRange(sheet, keyCol, headerRows + 1); const url = `${SHEETS_BASE}/${encodeURIComponent(spreadsheetId)}/values/${encodeURIComponent(range)}`; - const res = await doFetch(url, { - headers: { Authorization: `Bearer ${token}` }, - }); - if (!res.ok) { - throw new Error( - `storeSheetsInit: failed to read key column (${res.status})`, - ); - } + const res = await request(getToken, doFetch, url); + if (!res.ok) throw failedResponse('key column read', res); const payload: unknown = await res.json(); const rows = readKeyColumnRows(payload); const index = new Map(); @@ -189,13 +217,10 @@ async function readCell( getToken: TokenProvider, doFetch: typeof fetch, ): Promise { - const token = await getToken(); const range = buildCellRange(sheet, column, row); const url = `${SHEETS_BASE}/${encodeURIComponent(spreadsheetId)}/values/${encodeURIComponent(range)}`; - const res = await doFetch(url, { - headers: { Authorization: `Bearer ${token}` }, - }); - if (!res.ok) return undefined; + const res = await request(getToken, doFetch, url); + if (!res.ok) throw failedResponse('read', res); const payload: unknown = await res.json(); if (!isRecord(payload)) return undefined; const values = (payload as KeyColumnResponse).values; @@ -215,20 +240,14 @@ async function writeCell( getToken: TokenProvider, doFetch: typeof fetch, ): Promise { - const token = await getToken(); const range = buildCellRange(sheet, column, row); const url = `${SHEETS_BASE}/${encodeURIComponent(spreadsheetId)}/values/${encodeURIComponent(range)}?valueInputOption=RAW`; - const res = await doFetch(url, { + const res = await request(getToken, doFetch, url, { method: 'PUT', - headers: { - Authorization: `Bearer ${token}`, - 'Content-Type': 'application/json', - }, + headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ values: [[value]], range }), }); - if (!res.ok) { - throw new Error(`storeSheets.writeCell: write failed (${res.status})`); - } + if (!res.ok) throw failedResponse('write', res); } interface AppendResponse { @@ -244,20 +263,14 @@ async function appendRow( getToken: TokenProvider, doFetch: typeof fetch, ): Promise { - const token = await getToken(); const range = buildAppendRange(sheet, keyCol, valueCol); const url = `${SHEETS_BASE}/${encodeURIComponent(spreadsheetId)}/values/${encodeURIComponent(range)}:append?valueInputOption=RAW`; - const res = await doFetch(url, { + const res = await request(getToken, doFetch, url, { method: 'POST', - headers: { - Authorization: `Bearer ${token}`, - 'Content-Type': 'application/json', - }, + headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ values: [values] }), }); - if (!res.ok) { - throw new Error(`storeSheets.appendRow: append failed (${res.status})`); - } + if (!res.ok) throw failedResponse('append', res); const payload: unknown = await res.json(); if (!isRecord(payload)) return undefined; const updates = (payload as AppendResponse).updates; @@ -304,8 +317,9 @@ export const storeSheetsInit: Store.Init = async (context) => { // Sheets stores structured JSON per cell; it cannot serve byte-exact raw // bytes. file:true is a hard error at init: pick fs, s3, or gcs instead. if (context.config.file) { - throw new Error( + throw new StepError( 'Sheets store does not support file mode (file: true). Sheets stores structured JSON values per cell. Use fs, s3, or gcs for byte-exact serving.', + { code: 'config' }, ); } const settings: SheetsStoreSettings = context.config.settings; @@ -321,16 +335,34 @@ export const storeSheetsInit: Store.Init = async (context) => { const valueCol = resolveValueColumn(settings); const headerRows = resolveHeaderRows(settings); - await ensureSpreadsheetExists(settings.id, id, getToken, logger, doFetch); - - const keyToRow = await buildKeyIndex( - settings.id, - sheet, - keyCol, - headerRows, - getToken, - doFetch, - ); + // The spreadsheet check and the key column read happen at the first + // operation, so init stays offline; a failed read is retried next time. + let index: Promise> | undefined; + function keyIndex(): Promise> { + if (!index) { + const loading = ensureSpreadsheetExists( + settings.id, + id, + getToken, + logger, + doFetch, + ).then(() => + buildKeyIndex( + settings.id, + sheet, + keyCol, + headerRows, + getToken, + doFetch, + ), + ); + loading.catch(() => { + if (index === loading) index = undefined; + }); + index = loading; + } + return index; + } const config: Store.Config = { settings, @@ -345,7 +377,7 @@ export const storeSheetsInit: Store.Init = async (context) => { setup: sheetsSetup, async get(key: string): Promise { - const row = keyToRow.get(key); + const row = (await keyIndex()).get(key); if (row === undefined) return undefined; const cell = await readCell( settings.id, @@ -356,17 +388,24 @@ export const storeSheetsInit: Store.Init = async (context) => { doFetch, ); if (cell === undefined || cell === '') return undefined; + // A cell that does not hold a stored value is a failed read, never a + // miss (a blank cell is the miss). The message names no value. let parsed: unknown; try { parsed = JSON.parse(cell); } catch (err) { - logger.debug('storeSheets.get: JSON parse failed', { - key, - error: err instanceof Error ? err.message : String(err), + throw new StepError('Sheets value could not be decoded', { + code: 'decode', + retryable: false, + cause: err, }); - return undefined; } - return isStoreValue(parsed) ? parsed : undefined; + if (!isStoreValue(parsed)) + throw new StepError('Sheets value could not be decoded', { + code: 'decode', + retryable: false, + }); + return parsed; }, async set(key: string, value: Store.StoreValue): Promise { @@ -375,11 +414,13 @@ export const storeSheetsInit: Store.Init = async (context) => { // base64 blob, and the 50k-character cell limit makes that unsafe. Reject // any binary leaf, at any depth, loudly. Plain JSON values are unaffected. if (hasBinaryLeaf(value)) { - throw new Error( + throw new StepError( 'Sheets store cannot persist binary values (Uint8Array). Cells hold structured JSON only; the 50k-character cell limit makes base64 binary unsafe. Use fs, s3, or gcs.', + { code: 'config' }, ); } const serialized = JSON.stringify(value); + const keyToRow = await keyIndex(); const existingRow = keyToRow.get(key); if (existingRow === undefined) { const newRow = await appendRow( @@ -406,7 +447,7 @@ export const storeSheetsInit: Store.Init = async (context) => { }, async delete(key: string): Promise { - const row = keyToRow.get(key); + const row = (await keyIndex()).get(key); if (row === undefined) return; await writeCell(settings.id, sheet, valueCol, row, '', getToken, doFetch); }, diff --git a/packages/server/transformers/fingerprint/README.md b/packages/server/transformers/fingerprint/README.md index c2ca86f3b..411a1b70b 100644 --- a/packages/server/transformers/fingerprint/README.md +++ b/packages/server/transformers/fingerprint/README.md @@ -40,8 +40,10 @@ await startFlow({ ``` The source must extract `ip` and `userAgent` into `ingest` (its -`config.ingest`). The event then carries the hash at `user.hash`. Without a salt -the hash can be reversed to the IP, so the transformer logs a warning. +`config.ingest`). The event then carries the hash at `user.hash`. The salt is +required: without it the hash could be reversed to the IP, so the transformer +does not start. An event whose IP and user agent both resolve empty fails with +`missing-input`, since every visitor would get the same hash. ## Documentation diff --git a/packages/server/transformers/fingerprint/src/__tests__/conformance.test.ts b/packages/server/transformers/fingerprint/src/__tests__/conformance.test.ts new file mode 100644 index 000000000..ca73491ac --- /dev/null +++ b/packages/server/transformers/fingerprint/src/__tests__/conformance.test.ts @@ -0,0 +1,32 @@ +import path from 'path'; +import { conformance, runExamples } from '@walkeros/collector/dev'; +import { transformerFingerprint } from '../transformer'; +import { examples, schemas } from '../dev'; + +const settings = { salt: 'test-salt', length: 16 }; + +describe('fingerprint conformance', () => { + beforeEach(() => { + jest.useRealTimers(); + }); + + it('conforms', async () => { + await conformance('transformer', transformerFingerprint, { + profile: 'transformer', + examples: examples.step, + schema: schemas.settings, + settings, + root: path.join(__dirname, '..', '..'), + }); + }); + + it('fails an event without visitor inputs on a real collector', async () => { + const { missingIngest } = examples.step; + await runExamples( + 'transformer', + transformerFingerprint, + { missingIngest }, + { settings }, + ); + }); +}); diff --git a/packages/server/transformers/fingerprint/src/__tests__/privacy.test.ts b/packages/server/transformers/fingerprint/src/__tests__/privacy.test.ts index b162f2b38..60ff45ea3 100644 --- a/packages/server/transformers/fingerprint/src/__tests__/privacy.test.ts +++ b/packages/server/transformers/fingerprint/src/__tests__/privacy.test.ts @@ -181,17 +181,28 @@ describe('Privacy defaults', () => { }); it.each([ - ['missing', undefined], - ['empty', ''], - ])('warns once when the salt is %s and still hashes', async (_, value) => { - const transformer = await init({ salt: value }); + ['missing', {}], + ['empty', { salt: '' }], + ])('does not start when the salt is %s', (_, settings) => { + expect(() => + transformerFingerprint( + createMockContext({ config: { settings }, logger, id: 'fp' }), + ), + ).toThrow( + expect.objectContaining({ + code: 'config', + message: 'Config settings salt missing', + }), + ); + }); - await hash(transformer); - const result = await hash(transformer); + it('fails with missing-input when neither ip nor user agent resolves', async () => { + const transformer = await init({ salt }); - expect(result).toMatch(/^[a-f0-9]{64}$/); - expect(logger.warn).toHaveBeenCalledTimes(1); - expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('salt')); + await expect(push(transformer, {})).rejects.toMatchObject({ + code: 'missing-input', + }); + expect(logger.warn).not.toHaveBeenCalled(); }); it.each([ diff --git a/packages/server/transformers/fingerprint/src/__tests__/schemas.test.ts b/packages/server/transformers/fingerprint/src/__tests__/schemas.test.ts index 4c327e769..4b94b362a 100644 --- a/packages/server/transformers/fingerprint/src/__tests__/schemas.test.ts +++ b/packages/server/transformers/fingerprint/src/__tests__/schemas.test.ts @@ -1,10 +1,18 @@ import { SettingsSchema } from '../schemas'; +// The salt is required, so every parse carries one. +const salt = 'test-salt'; + describe('SettingsSchema', () => { + it('requires a salt', () => { + expect(SettingsSchema.safeParse({}).success).toBe(false); + }); + it.each(['ip', 'userAgent', 'site'])( 'accepts a fallback list for %s, as Mapping.Value allows', (input) => { const result = SettingsSchema.safeParse({ + salt, [input]: ['event.source.url', 'ingest.origin'], }); expect(result.success).toBe(true); @@ -12,11 +20,12 @@ describe('SettingsSchema', () => { ); it('accepts false to switch an input off', () => { - expect(SettingsSchema.safeParse({ site: false }).success).toBe(true); + expect(SettingsSchema.safeParse({ salt, site: false }).success).toBe(true); }); it('accepts mapping values in fields', () => { const result = SettingsSchema.safeParse({ + salt, fields: ['ingest.ip', ['event.user.id', 'ingest.userId'], { key: 'x' }], }); expect(result.success).toBe(true); diff --git a/packages/server/transformers/fingerprint/src/__tests__/stepExamples.test.ts b/packages/server/transformers/fingerprint/src/__tests__/stepExamples.test.ts index 4aeadd6e4..3806afc45 100644 --- a/packages/server/transformers/fingerprint/src/__tests__/stepExamples.test.ts +++ b/packages/server/transformers/fingerprint/src/__tests__/stepExamples.test.ts @@ -61,9 +61,9 @@ describe('Step Examples', () => { }); }); - describe('missingFields', () => { - it('should still produce a hash when ingest is empty', async () => { - const example = examples.step.missingFields; + describe('missingIngest', () => { + it('fails with missing-input when ingest is empty', async () => { + const example = examples.step.missingIngest; const event = example.in as WalkerOS.DeepPartialEvent; const transformer = await transformerFingerprint( @@ -72,14 +72,10 @@ describe('Step Examples', () => { }), ); - // No ingest fields provided: each empty input warns once - const result = await transformer.push(event, createPushContext({})); - - expect(result).toBeDefined(); - expect(getByPath(result, 'event.user.hash')).toBeDefined(); - expect(typeof getByPath(result, 'event.user.hash')).toBe('string'); - expect(getByPath(result, 'event.user.hash')).toHaveLength(16); - expect(getByPath(result, 'event.name')).toBe(event.name); + // No ip and no user agent: every visitor would share one hash. + await expect( + transformer.push(event, createPushContext({})), + ).rejects.toMatchObject({ code: 'missing-input' }); }); }); diff --git a/packages/server/transformers/fingerprint/src/__tests__/transformer.test.ts b/packages/server/transformers/fingerprint/src/__tests__/transformer.test.ts index 5f40fb4e5..552424c25 100644 --- a/packages/server/transformers/fingerprint/src/__tests__/transformer.test.ts +++ b/packages/server/transformers/fingerprint/src/__tests__/transformer.test.ts @@ -11,11 +11,15 @@ import type { FingerprintSettings } from '../types'; describe('Transformer Fingerprint', () => { const mockLogger = createMockLogger(); - const createInitContext = ( - config: Transformer.Config>, - ) => + // Every config gets the required salt. + const createInitContext = (config: { + settings: Omit; + }) => createMockContext>({ - config, + config: { + ...config, + settings: { salt: 'test-salt', ...config.settings }, + }, logger: mockLogger, id: 'test-fingerprint', }); @@ -173,7 +177,7 @@ describe('Transformer Fingerprint', () => { expect(getByPath(result, 'event.user.hash')).toBeDefined(); }); - it('should handle undefined ingest', async () => { + it('fails with missing-input when every input resolves empty', async () => { const initContext = createInitContext({ settings: { fields: ['ingest.ip'], @@ -185,13 +189,12 @@ describe('Transformer Fingerprint', () => { const pushContext = createPushContext(undefined); - const result = await transformer.push(baseEvent, pushContext); - - expect(result).toBeDefined(); - expect(getByPath(result, 'event.user.hash')).toBeDefined(); + await expect( + transformer.push(baseEvent, pushContext), + ).rejects.toMatchObject({ name: 'StepError', code: 'missing-input' }); }); - it('should handle empty fields array', async () => { + it('does not start without an input that tells visitors apart', async () => { const initContext = createInitContext({ settings: { fields: [], @@ -199,15 +202,9 @@ describe('Transformer Fingerprint', () => { }, }); - const transformer = await transformerFingerprint(initContext); - - const pushContext = createPushContext({ ip: '192.168.1.1' }); - - const result = await transformer.push(baseEvent, pushContext); - - // Empty fields = hash of empty string - expect(result).toBeDefined(); - expect(getByPath(result, 'event.user.hash')).toBeDefined(); + expect(() => transformerFingerprint(initContext)).toThrow( + expect.objectContaining({ name: 'StepError', code: 'config' }), + ); }); }); diff --git a/packages/server/transformers/fingerprint/src/examples/step.ts b/packages/server/transformers/fingerprint/src/examples/step.ts index d22dc7ee1..53633a4be 100644 --- a/packages/server/transformers/fingerprint/src/examples/step.ts +++ b/packages/server/transformers/fingerprint/src/examples/step.ts @@ -43,38 +43,21 @@ export const serverFingerprint: Flow.StepExample = { ], }; -export const missingFields: Flow.StepExample = { +export const missingIngest: Flow.StepExample = { public: false, description: - 'Graceful handling when ingest is missing: each empty input logs one warning, and the hash is still generated.', + 'The ingest carries no ip and no user agent (the source sets no config.ingest): every visitor would get the same hash, so the transformer fails the event with missing-input.', in: { - name: 'session start', - data: { id: 's3ss10n' }, + name: 'page view', id: 'ev-1700000601', trigger: 'load', - entity: 'session', - action: 'start', + entity: 'page', + action: 'view', timestamp: 1700000601, source: { type: 'express', platform: 'server' }, }, - out: [ - [ - 'return', - { - event: { - name: 'session start', - data: { id: 's3ss10n' }, - user: { hash: 'e183220b699c10a8' }, - id: 'ev-1700000601', - trigger: 'load', - entity: 'session', - action: 'start', - timestamp: 1700000601, - source: { type: 'express', platform: 'server' }, - }, - }, - ], - ], + out: [], + outcome: { status: 'failed', code: 'missing-input', retryable: false }, }; export const ipAnonymization: Flow.StepExample = { diff --git a/packages/server/transformers/fingerprint/src/hints.ts b/packages/server/transformers/fingerprint/src/hints.ts index dec84680f..29312688a 100644 --- a/packages/server/transformers/fingerprint/src/hints.ts +++ b/packages/server/transformers/fingerprint/src/hints.ts @@ -39,7 +39,7 @@ export const hints: Hint.Hints = { ], }, 'salt-and-rotation': { - text: 'The salt keys the hash (HMAC), so it cannot be recomputed or reversed to the IP without it. Pass it via $env so it stays out of the flow file, and use one salt per deployment. Without a salt the transformer runs but logs a warning at startup. Rotation is built in (daily by default, UTC windows): do not add a date field for it. For another window, set rotate: "none" and add the window as a field. Setting fields switches the default inputs off, so name them too: { rotate: "none", ip: "ingest.ip", userAgent: "ingest.userAgent", site: "event.source.url", fields: [{ fn: () => new Date().toISOString().slice(0, 7) }] } rotates monthly. rotate: "none" without such a field keeps one hash per visitor indefinitely, which is a privacy decision, not a default.', + text: 'The salt keys the hash (HMAC), so it cannot be recomputed or reversed to the IP without it. Pass it via $env so it stays out of the flow file, and use one salt per deployment. The salt is required: without it the transformer does not start. Rotation is built in (daily by default, UTC windows): do not add a date field for it. For another window, set rotate: "none" and add the window as a field. Setting fields switches the default inputs off, so name them too: { rotate: "none", ip: "ingest.ip", userAgent: "ingest.userAgent", site: "event.source.url", fields: [{ fn: () => new Date().toISOString().slice(0, 7) }] } rotates monthly. rotate: "none" without such a field keeps one hash per visitor indefinitely, which is a privacy decision, not a default.', }, 'site-sources': { text: 'site defaults to the hostname of event.source.url, which the walkerOS browser and dataLayer sources set on every event and transformer-ga4 sets from the page URL of the hit (dl). Older transformer-ga4 versions did not set it, so upgrading transformer-ga4 changes the hash of GA4-decoded events once: site now joins the hash. Events without source.url (server-side events, GA4 hits without dl) need site pointed elsewhere: an ingest path such as ingest.property (a path parameter) or ingest.origin (map it in config.ingest first, e.g. origin: { key: "headers.origin" }), or a static { value: "example.com" }. site keeps one visitor apart across sites when one collector with one salt serves several of them; a single-site deployment with its own salt is already separated by the salt.', diff --git a/packages/server/transformers/fingerprint/src/schemas/settings.ts b/packages/server/transformers/fingerprint/src/schemas/settings.ts index cbc2c7f90..76f16bb71 100644 --- a/packages/server/transformers/fingerprint/src/schemas/settings.ts +++ b/packages/server/transformers/fingerprint/src/schemas/settings.ts @@ -18,9 +18,9 @@ export const SettingsSchema = z .object({ salt: z .string() - .optional() + .min(1) .describe( - 'Secret key for the hash (HMAC), e.g. "$env.FINGERPRINT_SALT". Without it the hash can be reversed to the IP; a warning is logged.', + 'Secret key for the hash (HMAC), e.g. "$env.FINGERPRINT_SALT". Required: without it the hash could be reversed to the IP.', ), rotate: z .enum(['daily', 'hourly', 'none']) @@ -58,6 +58,7 @@ export const SettingsSchema = z 'Truncate hash to this length. Default: full 64-char SHA-256 hash', ), }) + .strict() .describe( 'Fingerprint transformer: a cookieless, privacy-friendly visitor hash', ); diff --git a/packages/server/transformers/fingerprint/src/transformer.ts b/packages/server/transformers/fingerprint/src/transformer.ts index 7af78b80d..94a1e1fc6 100644 --- a/packages/server/transformers/fingerprint/src/transformer.ts +++ b/packages/server/transformers/fingerprint/src/transformer.ts @@ -7,6 +7,7 @@ import { getMappingValue, getOS, setByPath, + StepError, } from '@walkeros/core'; import { getHashServer } from '@walkeros/server-core'; import type { FingerprintSettings } from './types'; @@ -20,6 +21,10 @@ interface Input { const inputNames: InputName[] = ['ip', 'userAgent', 'site']; +// The site is the same for every visitor of a site, so it tells no visitors +// apart: a hash needs at least one of these inputs or a field. +const visitorInputs: InputName[] = ['ip', 'userAgent']; + const defaultInputs: Record = { ip: 'ingest.ip', userAgent: 'ingest.userAgent', @@ -70,10 +75,13 @@ export const transformerFingerprint: Transformer.Init< > = (context) => { const { config, logger } = context; const settings: Partial = config.settings ?? {}; + // Without a salt the hash can be reversed to the IP by brute force. + const salt = settings.salt; + if (!salt) + throw new StepError('Config settings salt missing', { code: 'config' }); const fields: Mapping.Value[] = settings.fields || []; const output: string = settings.output || 'user.hash'; const length: number | undefined = settings.length; - const salt: string | undefined = settings.salt || undefined; const rotate = settings.rotate ?? 'daily'; // Named inputs default on only without fields, so a fields config keeps its input set @@ -84,18 +92,20 @@ export const transformerFingerprint: Transformer.Init< if (value !== false) inputs.push({ name, value }); } - if (!salt) - logger.warn( - 'Fingerprint has no salt: the hash can be reversed to the IP by brute force. Set settings.salt, e.g. "$env.FINGERPRINT_SALT".', + if ( + !fields.length && + !inputs.some(({ name }) => visitorInputs.includes(name)) + ) + throw new StepError( + 'Fingerprint has no input that tells visitors apart: keep settings.ip or settings.userAgent, or set settings.fields', + { code: 'config' }, ); const warnedEmpty = new Set(); return { type: 'fingerprint', - config: config as Transformer.Config< - Transformer.Types - >, + config: { ...config, settings: { ...settings, salt } }, async push(event, context) { const { ingest, collector } = context; @@ -107,21 +117,32 @@ export const transformerFingerprint: Transformer.Init< const inputValues = await Promise.all( inputs.map(async ({ name, value }) => { const resolved = await resolve(value); - const reduced = resolved ? reduceInput[name](resolved) : ''; - - if (!reduced && !warnedEmpty.has(name)) { - warnedEmpty.add(name); - logger.warn( - `Fingerprint input "${name}" resolved empty, so hashes are less distinct. Check the source's config.ingest or set settings.${name}.`, - ); - } - - return reduced; + return resolved ? reduceInput[name](resolved) : ''; }), ); const fieldValues = await Promise.all(fields.map(resolve)); + // Every visitor would get the same hash: fail instead of collapsing + // the visitor counts. + const distinct = [ + ...inputValues.filter((_, i) => visitorInputs.includes(inputs[i].name)), + ...fieldValues, + ]; + if (!distinct.some(Boolean)) + throw new StepError( + "Fingerprint inputs resolved empty: no ip, user agent or field tells visitors apart. Check the source's config.ingest.", + { code: 'missing-input' }, + ); + + inputs.forEach(({ name }, i) => { + if (inputValues[i] || warnedEmpty.has(name)) return; + warnedEmpty.add(name); + logger.warn( + `Fingerprint input "${name}" resolved empty, so hashes are less distinct. Check the source's config.ingest or set settings.${name}.`, + ); + }); + // A separator keeps 'ab' + 'c' and 'a' + 'bc' apart const input = [ rotationWindow(rotate), @@ -129,11 +150,7 @@ export const transformerFingerprint: Transformer.Init< ...fieldValues, ].join('\u001f'); - const hash = await getHashServer( - input, - length, - salt ? { key: salt } : {}, - ); + const hash = await getHashServer(input, length, { key: salt }); return { event: setByPath(event, output, hash) }; }, }; diff --git a/packages/server/transformers/fingerprint/src/types.ts b/packages/server/transformers/fingerprint/src/types.ts index c8de311be..1bc866be4 100644 --- a/packages/server/transformers/fingerprint/src/types.ts +++ b/packages/server/transformers/fingerprint/src/types.ts @@ -10,10 +10,9 @@ import type { Mapping } from '@walkeros/core'; export interface FingerprintSettings { /** * Secret key for the hash (HMAC), e.g. "$env.FINGERPRINT_SALT". - * Without it the hash can be reversed to the IP by brute force; the - * transformer then warns once and hashes unkeyed. + * Required: without it the hash could be reversed to the IP by brute force. */ - salt?: string; + salt: string; /** * Rotation window in UTC. The same visitor gets a new hash each window. diff --git a/packages/transformers/ga4/src/__tests__/conformance.test.ts b/packages/transformers/ga4/src/__tests__/conformance.test.ts new file mode 100644 index 000000000..6009443f7 --- /dev/null +++ b/packages/transformers/ga4/src/__tests__/conformance.test.ts @@ -0,0 +1,25 @@ +import path from 'path'; +import { conformance, runExamples } from '@walkeros/collector/dev'; +import { transformerGa4 } from '../transformer'; +import { examples, schemas } from '../dev'; + +describe('transformer-ga4 conformance', () => { + beforeEach(() => { + jest.useRealTimers(); + }); + + it('conforms', async () => { + await conformance('transformer', transformerGa4, { + profile: 'transformer', + examples: examples.step, + schema: schemas.settings, + settings: {}, + root: path.join(__dirname, '..', '..'), + }); + }); + + it('fails an event without a GA4 request on a real collector', async () => { + const { noRequest } = examples.step; + await runExamples('transformer', transformerGa4, { noRequest }); + }); +}); diff --git a/packages/transformers/ga4/src/__tests__/examples.test.ts b/packages/transformers/ga4/src/__tests__/examples.test.ts index 471aface6..5fa5a2a2b 100644 --- a/packages/transformers/ga4/src/__tests__/examples.test.ts +++ b/packages/transformers/ga4/src/__tests__/examples.test.ts @@ -69,7 +69,12 @@ function withoutTimestamp(effects: readonly unknown[]): unknown[] { const emptyEvent: WalkerOS.DeepPartialEvent = {}; describe('Step Examples', () => { - it.each(Object.entries(examples.step))('%s', async (_name, example) => { + // Non-public examples (failures) run in the conformance test. + const published = Object.entries(examples.step).filter( + ([, example]) => example.public !== false, + ); + + it.each(published)('%s', async (_name, example) => { expect(isGA4Request(example.in)).toBe(true); if (!isGA4Request(example.in)) return; // type narrow diff --git a/packages/transformers/ga4/src/__tests__/transformer.test.ts b/packages/transformers/ga4/src/__tests__/transformer.test.ts index 0da73b68d..becf273b7 100644 --- a/packages/transformers/ga4/src/__tests__/transformer.test.ts +++ b/packages/transformers/ga4/src/__tests__/transformer.test.ts @@ -45,21 +45,25 @@ describe('transformerGa4', () => { }); }); - it('returns false when ingest has no url', async () => { + it('fails with missing-input when ingest has no url', async () => { const ctx = makeContext({}); const instance = await transformerGa4(ctx); - const result = await instance.push(emptyEvent, ctx); - expect(result).toBe(false); + await expect(instance.push(emptyEvent, ctx)).rejects.toMatchObject({ + name: 'StepError', + code: 'missing-input', + retryable: false, + }); }); - it('returns false when ingest.url is not a string', async () => { + it('fails with missing-input when ingest.url is not a string', async () => { const ctx = createMockContext({ config: {}, ingest: { ...createIngest('test'), url: 123 }, }); const instance = await transformerGa4(ctx); - const result = await instance.push(emptyEvent, ctx); - expect(result).toBe(false); + await expect(instance.push(emptyEvent, ctx)).rejects.toMatchObject({ + code: 'missing-input', + }); }); it('fan-outs multiple events as array of Results', async () => { @@ -241,16 +245,6 @@ describe('transformerGa4', () => { true, ); }); - - it('logs debug when no request is present in ingest', async () => { - const logger = createMockLogger(); - const ctx = makeContext({ logger }); - await (await transformerGa4(ctx)).push(emptyEvent, ctx); - const debugCalls = logger.debug.mock.calls.map((args) => String(args[0])); - expect(debugCalls.some((m) => m.includes('no request in ingest'))).toBe( - true, - ); - }); }); describe('Task 26 — try/catch around parseRequest', () => { @@ -265,7 +259,7 @@ describe('transformerGa4', () => { ); }); - it('catches and logs unexpected parser throws', async () => { + it('fails an unexpected parser throw, without a log line of its own', async () => { // Force a throw from inside `push` by making `ctx.ingest.url` lookup // throw at access time. The try/catch wraps `parseRequest` and the // hit-derived steps, but `readGA4Request` runs first and accesses @@ -286,11 +280,14 @@ describe('transformerGa4', () => { logger, url: 'https://x/g/collect?v=2&tid=G-X&en=page_view', }); - const result = await (await transformerGa4(ctx)).push(emptyEvent, ctx); - expect(result).toBe(false); - const errors = logger.error.mock.calls.map((args) => String(args[0])); - expect(errors.some((e) => e.includes('parse error'))).toBe(true); - expect(errors.some((e) => e.includes('boom'))).toBe(true); + await expect( + (await transformerGa4(ctx)).push(emptyEvent, ctx), + ).rejects.toMatchObject({ + name: 'StepError', + code: 'transformer', + message: 'transformer-ga4: the GA4 request could not be decoded', + }); + expect(logger.error).not.toHaveBeenCalled(); } finally { URLSearchParams.prototype.forEach = original; } diff --git a/packages/transformers/ga4/src/examples/step.ts b/packages/transformers/ga4/src/examples/step.ts index d7a81e6b6..37332c2db 100644 --- a/packages/transformers/ga4/src/examples/step.ts +++ b/packages/transformers/ga4/src/examples/step.ts @@ -382,3 +382,12 @@ export const batchPost: Flow.StepExample = { ], ], }; + +export const noRequest: Flow.StepExample = { + public: false, + description: + 'An event without a GA4 request in its ingest (the source sets no ingest.url): nothing to decode, so the transformer fails it with missing-input instead of dropping all traffic.', + in: { name: 'page view', source: { type: 'express', platform: 'server' } }, + out: [], + outcome: { status: 'failed', code: 'missing-input', retryable: false }, +}; diff --git a/packages/transformers/ga4/src/transformer.ts b/packages/transformers/ga4/src/transformer.ts index 22484e94c..10f4a4eaa 100644 --- a/packages/transformers/ga4/src/transformer.ts +++ b/packages/transformers/ga4/src/transformer.ts @@ -1,4 +1,5 @@ import type { Ingest, Transformer } from '@walkeros/core'; +import { StepError } from '@walkeros/core'; import { parseRequest, splitBodyLines } from './parse'; import { mapHitToEvents } from './map'; import { defaultMapping } from './defaults'; @@ -14,8 +15,8 @@ import type { GA4Request, GA4Settings } from './types'; * `Record` per `@walkeros/core`, so each field is * narrowed with a `typeof` guard instead of a cast. * - * Returns `null` if `url` is missing or not a string — the caller drops - * the event in that case. + * Returns `null` if `url` is missing or not a string: the caller fails the + * event with `missing-input`. */ function readGA4Request(ingest: Ingest): GA4Request | null { const url = ingest.url; @@ -42,11 +43,14 @@ export const transformerGa4: Transformer.Init< async push(_event, ctx) { const { logger } = ctx; + // Without a request there is nothing to decode: a wiring mistake (the + // source's config.ingest) that would drop all traffic, so it fails. const raw = readGA4Request(ctx.ingest); - if (!raw) { - logger.debug('transformer-ga4: no request in ingest; skipping'); - return false; - } + if (!raw) + throw new StepError( + 'transformer-ga4: no GA4 request in ingest (ingest.url missing)', + { code: 'missing-input' }, + ); // Bound the fan-out before decoding: every event becomes a concurrent // push, and the source's batch cap does not apply to a raw text body. @@ -85,9 +89,14 @@ export const transformerGa4: Transformer.Init< if (events.length === 1) return { event: events[0] }; return events.map((event) => ({ event })); } catch (err) { - const message = err instanceof Error ? err.message : String(err); - logger.error(`transformer-ga4: parse error: ${message}`); - return false; + // The collector reports it; the message names no request content. + throw new StepError( + 'transformer-ga4: the GA4 request could not be decoded', + { + code: 'transformer', + cause: err, + }, + ); } }, }; From 216f9eac2aab98dbadb7efe907758c0c9d2a757d Mon Sep 17 00:00:00 2001 From: alexanderkirtzel Date: Wed, 21 Oct 2026 06:47:33 +0000 Subject: [PATCH 19/41] server recipe --- .../api/src/__tests__/conformance.test.ts | 20 ++ .../api/src/__tests__/stepExamples.test.ts | 7 +- .../destinations/api/src/examples/step.ts | 9 + .../server/destinations/api/src/index.test.ts | 44 ++-- packages/server/destinations/api/src/index.ts | 68 ++++-- .../destinations/api/src/schemas/settings.ts | 43 ++-- .../bing/src/__tests__/conformance.test.ts | 20 ++ .../bing/src/__tests__/index.test.ts | 2 +- .../bing/src/__tests__/stepExamples.test.ts | 7 +- .../server/destinations/bing/src/config.ts | 18 +- .../destinations/bing/src/examples/step.ts | 9 + .../server/destinations/bing/src/index.ts | 4 +- packages/server/destinations/bing/src/push.ts | 7 +- .../destinations/bing/src/schemas/settings.ts | 82 ++++--- .../criteo/src/__tests__/conformance.test.ts | 20 ++ .../criteo/src/__tests__/index.test.ts | 2 +- .../criteo/src/__tests__/stepExamples.test.ts | 7 +- .../server/destinations/criteo/src/config.ts | 18 +- .../destinations/criteo/src/examples/step.ts | 9 + .../server/destinations/criteo/src/index.ts | 4 +- .../server/destinations/criteo/src/push.ts | 7 +- .../criteo/src/schemas/settings.ts | 84 ++++--- .../src/__tests__/conformance.test.ts | 21 ++ .../customerio/src/__tests__/status.test.ts | 46 ++++ .../src/__tests__/stepExamples.test.ts | 7 +- .../destinations/customerio/src/config.ts | 22 +- .../customerio/src/examples/step.ts | 49 ++++ .../destinations/customerio/src/index.ts | 15 +- .../destinations/customerio/src/push.ts | 131 +++++++--- .../customerio/src/schemas/settings.ts | 102 ++++---- .../src/__tests__/conformance.test.ts | 44 ++++ .../datamanager/src/__tests__/index.test.ts | 15 +- .../src/__tests__/stepExamples.test.ts | 7 +- .../destinations/datamanager/src/config.ts | 11 +- .../datamanager/src/examples/step.ts | 29 +++ .../destinations/datamanager/src/index.ts | 10 +- .../destinations/datamanager/src/push.ts | 78 +++--- .../datamanager/src/schemas/settings.ts | 228 +++++++++--------- .../klaviyo/src/__tests__/conformance.test.ts | 21 ++ .../klaviyo/src/__tests__/status.test.ts | 48 ++++ .../src/__tests__/stepExamples.test.ts | 7 +- .../server/destinations/klaviyo/src/config.ts | 14 +- .../destinations/klaviyo/src/examples/step.ts | 41 ++++ .../server/destinations/klaviyo/src/index.ts | 10 +- .../server/destinations/klaviyo/src/push.ts | 60 +++-- .../klaviyo/src/schemas/settings.ts | 92 +++---- .../src/__tests__/conformance.test.ts | 20 ++ .../linkedin/src/__tests__/index.test.ts | 22 +- .../src/__tests__/stepExamples.test.ts | 7 +- .../destinations/linkedin/src/config.ts | 19 +- .../linkedin/src/examples/step.ts | 32 +++ .../server/destinations/linkedin/src/index.ts | 4 +- .../server/destinations/linkedin/src/push.ts | 22 +- .../linkedin/src/schemas/settings.ts | 74 +++--- .../meta/src/__tests__/conformance.test.ts | 20 ++ .../meta/src/__tests__/index.test.ts | 15 +- .../meta/src/__tests__/stepExamples.test.ts | 9 +- .../server/destinations/meta/src/config.ts | 18 +- .../destinations/meta/src/examples/step.ts | 9 + .../server/destinations/meta/src/index.ts | 4 +- packages/server/destinations/meta/src/push.ts | 25 +- .../destinations/meta/src/schemas/settings.ts | 98 ++++---- .../src/__tests__/conformance.test.ts | 21 ++ .../src/__tests__/stepExamples.test.ts | 7 +- .../destinations/mixpanel/src/config.ts | 4 +- .../mixpanel/src/examples/step.ts | 46 ++++ .../server/destinations/mixpanel/src/push.ts | 110 +++++---- .../mixpanel/src/schemas/settings.ts | 127 +++++----- .../mparticle/src/__tests__/config.test.ts | 35 ++- .../src/__tests__/conformance.test.ts | 20 ++ .../mparticle/src/__tests__/push.test.ts | 2 +- .../src/__tests__/stepExamples.test.ts | 7 +- .../destinations/mparticle/src/config.ts | 22 +- .../mparticle/src/examples/step.ts | 9 + .../destinations/mparticle/src/index.ts | 4 +- .../server/destinations/mparticle/src/push.ts | 7 +- .../mparticle/src/schemas/settings.ts | 98 ++++---- .../piwikpro/src/__tests__/config.test.ts | 28 +-- .../src/__tests__/conformance.test.ts | 23 ++ .../piwikpro/src/__tests__/hits.test.ts | 46 ++-- .../piwikpro/src/__tests__/push.test.ts | 62 ++--- .../piwikpro/src/__tests__/skip.test.ts | 17 -- .../src/__tests__/stepExamples.test.ts | 16 +- .../destinations/piwikpro/src/config.ts | 15 +- .../piwikpro/src/examples/step.ts | 51 +++- .../server/destinations/piwikpro/src/hints.ts | 4 +- .../server/destinations/piwikpro/src/hits.ts | 28 ++- .../server/destinations/piwikpro/src/index.ts | 4 +- .../server/destinations/piwikpro/src/push.ts | 54 ++--- .../piwikpro/src/schemas/settings.ts | 114 ++++----- .../server/destinations/piwikpro/src/skip.ts | 22 -- .../reddit/src/__tests__/conformance.test.ts | 20 ++ .../reddit/src/__tests__/index.test.ts | 2 +- .../reddit/src/__tests__/stepExamples.test.ts | 7 +- .../server/destinations/reddit/src/config.ts | 18 +- .../destinations/reddit/src/examples/step.ts | 9 + .../server/destinations/reddit/src/index.ts | 4 +- .../server/destinations/reddit/src/push.ts | 7 +- .../reddit/src/schemas/settings.ts | 96 ++++---- .../slack/src/__tests__/conformance.test.ts | 20 ++ .../slack/src/__tests__/stepExamples.test.ts | 7 +- .../server/destinations/slack/src/config.ts | 10 +- .../destinations/slack/src/examples/step.ts | 9 + .../server/destinations/slack/src/index.ts | 6 +- .../server/destinations/slack/src/push.ts | 81 +++---- .../slack/src/schemas/settings.ts | 5 +- .../src/__tests__/conformance.test.ts | 20 ++ .../snapchat/src/__tests__/index.test.ts | 2 +- .../src/__tests__/stepExamples.test.ts | 7 +- .../destinations/snapchat/src/config.ts | 18 +- .../snapchat/src/examples/step.ts | 9 + .../server/destinations/snapchat/src/index.ts | 4 +- .../server/destinations/snapchat/src/push.ts | 7 +- .../snapchat/src/schemas/settings.ts | 82 ++++--- .../twitter/src/__tests__/conformance.test.ts | 27 +++ .../twitter/src/__tests__/index.test.ts | 12 +- .../src/__tests__/stepExamples.test.ts | 7 +- .../server/destinations/twitter/src/config.ts | 34 ++- .../destinations/twitter/src/examples/step.ts | 32 +++ .../server/destinations/twitter/src/index.ts | 4 +- .../server/destinations/twitter/src/push.ts | 15 +- .../twitter/src/schemas/settings.ts | 102 ++++---- 122 files changed, 2416 insertions(+), 1267 deletions(-) create mode 100644 packages/server/destinations/api/src/__tests__/conformance.test.ts create mode 100644 packages/server/destinations/bing/src/__tests__/conformance.test.ts create mode 100644 packages/server/destinations/criteo/src/__tests__/conformance.test.ts create mode 100644 packages/server/destinations/customerio/src/__tests__/conformance.test.ts create mode 100644 packages/server/destinations/customerio/src/__tests__/status.test.ts create mode 100644 packages/server/destinations/datamanager/src/__tests__/conformance.test.ts create mode 100644 packages/server/destinations/klaviyo/src/__tests__/conformance.test.ts create mode 100644 packages/server/destinations/klaviyo/src/__tests__/status.test.ts create mode 100644 packages/server/destinations/linkedin/src/__tests__/conformance.test.ts create mode 100644 packages/server/destinations/meta/src/__tests__/conformance.test.ts create mode 100644 packages/server/destinations/mixpanel/src/__tests__/conformance.test.ts create mode 100644 packages/server/destinations/mparticle/src/__tests__/conformance.test.ts create mode 100644 packages/server/destinations/piwikpro/src/__tests__/conformance.test.ts delete mode 100644 packages/server/destinations/piwikpro/src/__tests__/skip.test.ts delete mode 100644 packages/server/destinations/piwikpro/src/skip.ts create mode 100644 packages/server/destinations/reddit/src/__tests__/conformance.test.ts create mode 100644 packages/server/destinations/slack/src/__tests__/conformance.test.ts create mode 100644 packages/server/destinations/snapchat/src/__tests__/conformance.test.ts create mode 100644 packages/server/destinations/twitter/src/__tests__/conformance.test.ts diff --git a/packages/server/destinations/api/src/__tests__/conformance.test.ts b/packages/server/destinations/api/src/__tests__/conformance.test.ts new file mode 100644 index 000000000..9bfc4d88c --- /dev/null +++ b/packages/server/destinations/api/src/__tests__/conformance.test.ts @@ -0,0 +1,20 @@ +import path from 'path'; +import { conformance } from '@walkeros/collector/dev'; +import { destinationAPI } from '..'; +import { examples, schemas } from '../dev'; + +describe('api conformance', () => { + beforeEach(() => { + jest.useRealTimers(); + }); + + it('conforms', async () => { + await conformance('destination', destinationAPI, { + profile: 'http', + examples: examples.step, + schema: schemas.settings, + settings: { url: 'https://api.example.com/events' }, + root: path.join(__dirname, '..', '..'), + }); + }); +}); diff --git a/packages/server/destinations/api/src/__tests__/stepExamples.test.ts b/packages/server/destinations/api/src/__tests__/stepExamples.test.ts index 597d94007..fd6b09c28 100644 --- a/packages/server/destinations/api/src/__tests__/stepExamples.test.ts +++ b/packages/server/destinations/api/src/__tests__/stepExamples.test.ts @@ -22,7 +22,12 @@ describe('Step Examples', () => { mockSendServer.mockResolvedValue({}); }); - it.each(Object.entries(examples.step))('%s', async (name, example) => { + // Non-public examples (silent, failures) run in the conformance test. + const published = Object.entries(examples.step).filter( + ([, example]) => example.public !== false, + ); + + it.each(published)('%s', async (name, example) => { const event = example.in as WalkerOS.Event; const mapping = example.mapping; diff --git a/packages/server/destinations/api/src/examples/step.ts b/packages/server/destinations/api/src/examples/step.ts index 11427d081..9b69dbe80 100644 --- a/packages/server/destinations/api/src/examples/step.ts +++ b/packages/server/destinations/api/src/examples/step.ts @@ -104,3 +104,12 @@ export const customTransform: Flow.StepExample = { ], ], }; + +/** A silent rule sends nothing to the endpoint. */ +export const silentRule: Flow.StepExample = { + public: false, + description: 'A rule with silent: true sends no request to the endpoint.', + in: getEvent('page view', { timestamp: 1700000802 }), + mapping: { silent: true }, + out: [], +}; diff --git a/packages/server/destinations/api/src/index.test.ts b/packages/server/destinations/api/src/index.test.ts index 5c5d32af7..0e42bab45 100644 --- a/packages/server/destinations/api/src/index.test.ts +++ b/packages/server/destinations/api/src/index.test.ts @@ -30,16 +30,18 @@ describe('Destination API', () => { }); test('init', async () => { - // Test with no URL - should not call sendServer - await destination.push( - event, - createMockContext({ - config: {}, - env: testEnv, - logger: mockLogger, - id: 'test-api', - }), - ); + // Test with no URL - fails with config and does not call sendServer + await expect( + destination.push( + event, + createMockContext({ + config: {}, + env: testEnv, + logger: mockLogger, + id: 'test-api', + }), + ), + ).rejects.toMatchObject({ code: 'config' }); expect(mockSendServer).not.toHaveBeenCalled(); // Test with URL - should call sendServer @@ -287,16 +289,18 @@ describe('Destination API', () => { ); }); - test('sends nothing without a url', async () => { - await destination.pushBatch!( - batchOf([{ event: getEvent('product view') }]), - createMockContext({ - config: {}, - env: testEnv, - logger: mockLogger, - id: 'test-api', - }), - ); + test('fails without a url', async () => { + await expect( + destination.pushBatch!( + batchOf([{ event: getEvent('product view') }]), + createMockContext({ + config: {}, + env: testEnv, + logger: mockLogger, + id: 'test-api', + }), + ), + ).rejects.toMatchObject({ code: 'config' }); expect(mockSendServer).not.toHaveBeenCalled(); }); diff --git a/packages/server/destinations/api/src/index.ts b/packages/server/destinations/api/src/index.ts index bbfc478bc..6d54fc149 100644 --- a/packages/server/destinations/api/src/index.ts +++ b/packages/server/destinations/api/src/index.ts @@ -1,6 +1,6 @@ import type { SendDataValue } from '@walkeros/core'; import type { Settings, Destination, Env } from './types'; -import { isDefined } from '@walkeros/core'; +import { isDefined, StepError } from '@walkeros/core'; import { sendServer } from '@walkeros/server-core'; // Types @@ -17,6 +17,10 @@ async function send( await sendServerFn(url, body, { headers, method, timeout }); } +function missingUrl(): StepError { + return new StepError('Config settings url missing', { code: 'config' }); +} + export const destinationAPI: Destination = { type: 'api', @@ -24,48 +28,60 @@ export const destinationAPI: Destination = { env: { sendServer }, + async init({ config }) { + if (!config.settings?.url) throw missingUrl(); + return config; + }, + async push(event, { config, rule, data, env, logger }) { const { settings } = config; const { url, method, transform } = settings || {}; - if (!url) return; + if (!url) throw missingUrl(); - const eventData = isDefined(data) ? data : event; - const body = transform - ? transform(eventData, config, rule) // Transform event data - : JSON.stringify(eventData); + // A silent rule has no call of its own: nothing to send. + if (!rule?.silent) { + const eventData = isDefined(data) ? data : event; + const body = transform + ? transform(eventData, config, rule) // Transform event data + : JSON.stringify(eventData); - logger?.debug('API destination sending request', { - url, - method: method || 'POST', - eventName: event.name, - }); + logger?.debug('API destination sending request', { + url, + method: method || 'POST', + eventName: event.name, + }); - await send(body, { ...settings, url }, env); + await send(body, { ...settings, url }, env); + } }, async pushBatch(batch, { config, rule, env, logger }) { const { settings } = config; const { url, method, transform } = settings || {}; - if (!url) return; + if (!url) throw missingUrl(); - const items = batch.entries.map((entry) => - isDefined(entry.data) ? entry.data : entry.event, - ); + // Silent rules have no call of their own; with nothing left, send nothing. + const entries = batch.entries.filter((entry) => !entry.rule?.silent); + if (entries.length > 0) { + const items = entries.map((entry) => + isDefined(entry.data) ? entry.data : entry.event, + ); - // Transform applies per item; the request body is the array of results - const payload = transform - ? items.map((item) => transform(item, config, rule)) - : items; + // Transform applies per item; the request body is the array of results + const payload = transform + ? items.map((item) => transform(item, config, rule)) + : items; - logger?.debug('API destination sending batch', { - url, - method: method || 'POST', - events: items.length, - }); + logger?.debug('API destination sending batch', { + url, + method: method || 'POST', + events: items.length, + }); - await send(JSON.stringify(payload), { ...settings, url }, env); + await send(JSON.stringify(payload), { ...settings, url }, env); + } }, }; diff --git a/packages/server/destinations/api/src/schemas/settings.ts b/packages/server/destinations/api/src/schemas/settings.ts index fc7212113..642c6d5d8 100644 --- a/packages/server/destinations/api/src/schemas/settings.ts +++ b/packages/server/destinations/api/src/schemas/settings.ts @@ -1,23 +1,28 @@ import { z } from '@walkeros/core/dev'; -export const SettingsSchema = z.object({ - url: z.string().url().describe('The API endpoint URL to send events to'), - headers: z - .record(z.string(), z.string()) - .describe('Custom HTTP headers to include with requests') - .optional(), - method: z.string().describe('HTTP method to use (default: POST)').optional(), - transform: z - .any() - .describe( - 'Function to transform event data before sending (like (data, config, mapping) => JSON.stringify(data))', - ) - .optional(), - timeout: z - .number() - .positive() - .describe('Request timeout in milliseconds (default: 5000)') - .optional(), -}); +export const SettingsSchema = z + .object({ + url: z.string().url().describe('The API endpoint URL to send events to'), + headers: z + .record(z.string(), z.string()) + .describe('Custom HTTP headers to include with requests') + .optional(), + method: z + .string() + .describe('HTTP method to use (default: POST)') + .optional(), + transform: z + .any() + .describe( + 'Function to transform event data before sending (like (data, config, mapping) => JSON.stringify(data))', + ) + .optional(), + timeout: z + .number() + .positive() + .describe('Request timeout in milliseconds (default: 5000)') + .optional(), + }) + .strict(); export type Settings = z.infer; diff --git a/packages/server/destinations/bing/src/__tests__/conformance.test.ts b/packages/server/destinations/bing/src/__tests__/conformance.test.ts new file mode 100644 index 000000000..6c34e25ad --- /dev/null +++ b/packages/server/destinations/bing/src/__tests__/conformance.test.ts @@ -0,0 +1,20 @@ +import path from 'path'; +import { conformance } from '@walkeros/collector/dev'; +import { destinationBing } from '..'; +import { examples, schemas } from '../dev'; + +describe('bing conformance', () => { + beforeEach(() => { + jest.useRealTimers(); + }); + + it('conforms', async () => { + await conformance('destination', destinationBing, { + profile: 'http', + examples: examples.step, + schema: schemas.settings, + settings: { accessToken: 's3cr3t', tagId: 'UET-12345' }, + root: path.join(__dirname, '..', '..'), + }); + }); +}); diff --git a/packages/server/destinations/bing/src/__tests__/index.test.ts b/packages/server/destinations/bing/src/__tests__/index.test.ts index 9e2594a29..5f8ac0984 100644 --- a/packages/server/destinations/bing/src/__tests__/index.test.ts +++ b/packages/server/destinations/bing/src/__tests__/index.test.ts @@ -58,7 +58,7 @@ describe('Server Destination Bing', () => { logger: createMockLogger(), id: 'test-bing', }), - ).rejects.toThrow('Config settings accessToken missing'); + ).rejects.toThrow('Config settings accessToken, tagId missing'); }); test('init - missing tagId throws', async () => { diff --git a/packages/server/destinations/bing/src/__tests__/stepExamples.test.ts b/packages/server/destinations/bing/src/__tests__/stepExamples.test.ts index 555b7f971..d8bf87136 100644 --- a/packages/server/destinations/bing/src/__tests__/stepExamples.test.ts +++ b/packages/server/destinations/bing/src/__tests__/stepExamples.test.ts @@ -20,7 +20,12 @@ describe('Step Examples', () => { }); }); - it.each(Object.entries(examples.step))('%s', async (_name, example) => { + // Non-public examples (silent, failures) run in the conformance test. + const published = Object.entries(examples.step).filter( + ([, example]) => example.public !== false, + ); + + it.each(published)('%s', async (_name, example) => { const event = example.in as WalkerOS.Event; const mapping = example.mapping; diff --git a/packages/server/destinations/bing/src/config.ts b/packages/server/destinations/bing/src/config.ts index 5d821e92d..7fab93fa0 100644 --- a/packages/server/destinations/bing/src/config.ts +++ b/packages/server/destinations/bing/src/config.ts @@ -1,15 +1,19 @@ import type { Config, Settings, PartialConfig } from './types'; -import type { Logger } from '@walkeros/core'; +import { StepError } from '@walkeros/core'; -export function getConfig( - partialConfig: PartialConfig = {}, - logger: Logger.Instance, -): Config { +export function getConfig(partialConfig: PartialConfig = {}): Config { const settings = (partialConfig.settings || {}) as Partial; const { accessToken, tagId } = settings; - if (!accessToken) logger.throw('Config settings accessToken missing'); - if (!tagId) logger.throw('Config settings tagId missing'); + if (!accessToken || !tagId) { + const missing = [ + ...(accessToken ? [] : ['accessToken']), + ...(tagId ? [] : ['tagId']), + ]; + throw new StepError(`Config settings ${missing.join(', ')} missing`, { + code: 'config', + }); + } const settingsConfig: Settings = { url: 'https://capi.uet.microsoft.com/v1/', diff --git a/packages/server/destinations/bing/src/examples/step.ts b/packages/server/destinations/bing/src/examples/step.ts index c44d2ba0b..bf8f0c758 100644 --- a/packages/server/destinations/bing/src/examples/step.ts +++ b/packages/server/destinations/bing/src/examples/step.ts @@ -312,3 +312,12 @@ export const addToCart: Flow.StepExample = { ], ], }; + +/** A silent rule sends nothing to Bing UET CAPI. */ +export const silentRule: Flow.StepExample = { + public: false, + description: 'A rule with silent: true sends no request to Bing UET CAPI.', + in: getEvent('page view', { timestamp: 1700000990000 }), + mapping: { silent: true }, + out: [], +}; diff --git a/packages/server/destinations/bing/src/index.ts b/packages/server/destinations/bing/src/index.ts index e472ed37b..e2473ee9c 100644 --- a/packages/server/destinations/bing/src/index.ts +++ b/packages/server/destinations/bing/src/index.ts @@ -10,8 +10,8 @@ export const destinationBing: Destination = { config: {}, - async init({ config: partialConfig, logger }) { - const config = getConfig(partialConfig, logger); + async init({ config: partialConfig }) { + const config = getConfig(partialConfig); return config; }, diff --git a/packages/server/destinations/bing/src/push.ts b/packages/server/destinations/bing/src/push.ts index 83269a2da..0a9645197 100644 --- a/packages/server/destinations/bing/src/push.ts +++ b/packages/server/destinations/bing/src/push.ts @@ -13,7 +13,12 @@ import { createMappingRoot, getMappingValue, isObject } from '@walkeros/core'; import { sendServer } from '@walkeros/server-core'; import { hashUserData } from './hash'; -export const push: PushFn = async function ( +// A silent rule has no bing call of its own: nothing to send. +export const push: PushFn = async function (event, context) { + if (!context.rule?.silent) await send(event, context); +}; + +const send: PushFn = async function ( event, { config, rule, data, ingest, env, logger, collector }, ) { diff --git a/packages/server/destinations/bing/src/schemas/settings.ts b/packages/server/destinations/bing/src/schemas/settings.ts index 5fb80c249..7f125fdbd 100644 --- a/packages/server/destinations/bing/src/schemas/settings.ts +++ b/packages/server/destinations/bing/src/schemas/settings.ts @@ -5,45 +5,47 @@ const InputSchema = z.union([ z.literal(false), ]); -export const SettingsSchema = z.object({ - accessToken: z - .string() - .min(1) - .describe( - 'Long-lived Bing UET CAPI access token from Microsoft Advertising', - ), - tagId: z.string().min(1).describe('Microsoft Advertising UET tag ID'), - url: z - .string() - .url() - .describe( - 'Custom Bing UET CAPI base URL (default https://capi.uet.microsoft.com/v1/)', - ) - .optional(), - doNotHash: z - .array(z.string()) - .describe("User data fields to skip hashing (like ['em', 'ph'])") - .optional(), - user_data: z - .record(z.string(), z.string()) - .describe("Mapping for user data fields (like { em: 'user.email' })") - .optional(), - dataProvider: z - .string() - .describe('Identifier of the data source (default "walkerOS")') - .optional(), - continueOnValidationError: z - .boolean() - .describe( - 'When true, Microsoft continues to ingest events despite validation errors', - ) - .optional(), - ip: InputSchema.describe( - 'Client IP, sent as userData.clientIpAddress. Resolves against { ingest, event }. Default: ["ingest.ip", "event.user.ip"]. false switches it off. A mapped userData.clientIpAddress wins.', - ).optional(), - userAgent: InputSchema.describe( - 'Client user agent, sent as userData.clientUserAgent. Resolves against { ingest, event }. Default: ["ingest.userAgent", "event.user.userAgent"]. false switches it off. A mapped userData.clientUserAgent wins.', - ).optional(), -}); +export const SettingsSchema = z + .object({ + accessToken: z + .string() + .min(1) + .describe( + 'Long-lived Bing UET CAPI access token from Microsoft Advertising', + ), + tagId: z.string().min(1).describe('Microsoft Advertising UET tag ID'), + url: z + .string() + .url() + .describe( + 'Custom Bing UET CAPI base URL (default https://capi.uet.microsoft.com/v1/)', + ) + .optional(), + doNotHash: z + .array(z.string()) + .describe("User data fields to skip hashing (like ['em', 'ph'])") + .optional(), + user_data: z + .record(z.string(), z.string()) + .describe("Mapping for user data fields (like { em: 'user.email' })") + .optional(), + dataProvider: z + .string() + .describe('Identifier of the data source (default "walkerOS")') + .optional(), + continueOnValidationError: z + .boolean() + .describe( + 'When true, Microsoft continues to ingest events despite validation errors', + ) + .optional(), + ip: InputSchema.describe( + 'Client IP, sent as userData.clientIpAddress. Resolves against { ingest, event }. Default: ["ingest.ip", "event.user.ip"]. false switches it off. A mapped userData.clientIpAddress wins.', + ).optional(), + userAgent: InputSchema.describe( + 'Client user agent, sent as userData.clientUserAgent. Resolves against { ingest, event }. Default: ["ingest.userAgent", "event.user.userAgent"]. false switches it off. A mapped userData.clientUserAgent wins.', + ).optional(), + }) + .strict(); export type Settings = z.infer; diff --git a/packages/server/destinations/criteo/src/__tests__/conformance.test.ts b/packages/server/destinations/criteo/src/__tests__/conformance.test.ts new file mode 100644 index 000000000..0ec525358 --- /dev/null +++ b/packages/server/destinations/criteo/src/__tests__/conformance.test.ts @@ -0,0 +1,20 @@ +import path from 'path'; +import { conformance } from '@walkeros/collector/dev'; +import { destinationCriteo } from '..'; +import { examples, schemas } from '../dev'; + +describe('criteo conformance', () => { + beforeEach(() => { + jest.useRealTimers(); + }); + + it('conforms', async () => { + await conformance('destination', destinationCriteo, { + profile: 'http', + examples: examples.step, + schema: schemas.settings, + settings: { partnerId: 'PARTNER_ID', callerId: 'CALLER_ID' }, + root: path.join(__dirname, '..', '..'), + }); + }); +}); diff --git a/packages/server/destinations/criteo/src/__tests__/index.test.ts b/packages/server/destinations/criteo/src/__tests__/index.test.ts index 73ec04bd5..f5d1a8914 100644 --- a/packages/server/destinations/criteo/src/__tests__/index.test.ts +++ b/packages/server/destinations/criteo/src/__tests__/index.test.ts @@ -58,7 +58,7 @@ describe('Server Destination Criteo', () => { logger: createMockLogger(), id: 'test-criteo', }), - ).rejects.toThrow('Config settings partnerId missing'); + ).rejects.toThrow('Config settings partnerId, callerId missing'); }); test('init - missing callerId throws', async () => { diff --git a/packages/server/destinations/criteo/src/__tests__/stepExamples.test.ts b/packages/server/destinations/criteo/src/__tests__/stepExamples.test.ts index 906b3cfc1..898d88fc9 100644 --- a/packages/server/destinations/criteo/src/__tests__/stepExamples.test.ts +++ b/packages/server/destinations/criteo/src/__tests__/stepExamples.test.ts @@ -20,7 +20,12 @@ describe('Step Examples', () => { }); }); - it.each(Object.entries(examples.step))('%s', async (_name, example) => { + // Non-public examples (silent, failures) run in the conformance test. + const published = Object.entries(examples.step).filter( + ([, example]) => example.public !== false, + ); + + it.each(published)('%s', async (_name, example) => { const event = example.in as WalkerOS.Event; const mapping = example.mapping; diff --git a/packages/server/destinations/criteo/src/config.ts b/packages/server/destinations/criteo/src/config.ts index a65e46ee4..ac2f646c0 100644 --- a/packages/server/destinations/criteo/src/config.ts +++ b/packages/server/destinations/criteo/src/config.ts @@ -1,17 +1,21 @@ import type { Config, Settings, PartialConfig } from './types'; -import type { Logger } from '@walkeros/core'; +import { StepError } from '@walkeros/core'; export const DEFAULT_URL = 'https://widget.criteo.com/m/event?version=s2s_v0'; -export function getConfig( - partialConfig: PartialConfig = {}, - logger: Logger.Instance, -): Config { +export function getConfig(partialConfig: PartialConfig = {}): Config { const settings = (partialConfig.settings || {}) as Partial; const { partnerId, callerId } = settings; - if (!partnerId) logger.throw('Config settings partnerId missing'); - if (!callerId) logger.throw('Config settings callerId missing'); + if (!partnerId || !callerId) { + const missing = [ + ...(partnerId ? [] : ['partnerId']), + ...(callerId ? [] : ['callerId']), + ]; + throw new StepError(`Config settings ${missing.join(', ')} missing`, { + code: 'config', + }); + } const settingsConfig: Settings = { siteType: 'd', diff --git a/packages/server/destinations/criteo/src/examples/step.ts b/packages/server/destinations/criteo/src/examples/step.ts index 76d2f57a5..10a4f1d3f 100644 --- a/packages/server/destinations/criteo/src/examples/step.ts +++ b/packages/server/destinations/criteo/src/examples/step.ts @@ -280,3 +280,12 @@ export const pageView: Flow.StepExample = { ], ], }; + +/** A silent rule sends nothing to Criteo. */ +export const silentRule: Flow.StepExample = { + public: false, + description: 'A rule with silent: true sends no request to Criteo.', + in: getEvent('page view', { timestamp: 1700000990000 }), + mapping: { silent: true }, + out: [], +}; diff --git a/packages/server/destinations/criteo/src/index.ts b/packages/server/destinations/criteo/src/index.ts index c10f9e058..5357ef748 100644 --- a/packages/server/destinations/criteo/src/index.ts +++ b/packages/server/destinations/criteo/src/index.ts @@ -10,8 +10,8 @@ export const destinationCriteo: Destination = { config: {}, - async init({ config: partialConfig, logger }) { - const config = getConfig(partialConfig, logger); + async init({ config: partialConfig }) { + const config = getConfig(partialConfig); return config; }, diff --git a/packages/server/destinations/criteo/src/push.ts b/packages/server/destinations/criteo/src/push.ts index be70b5a93..8d9386094 100644 --- a/packages/server/destinations/criteo/src/push.ts +++ b/packages/server/destinations/criteo/src/push.ts @@ -91,7 +91,12 @@ function toItems(raw: unknown): CriteoItem[] | undefined { return items.length ? items : undefined; } -export const push: PushFn = async function ( +// A silent rule has no criteo call of its own: nothing to send. +export const push: PushFn = async function (event, context) { + if (!context.rule?.silent) await send(event, context); +}; + +const send: PushFn = async function ( event, { config, rule, data, ingest, env, logger, collector }, ) { diff --git a/packages/server/destinations/criteo/src/schemas/settings.ts b/packages/server/destinations/criteo/src/schemas/settings.ts index 0754b2493..775e410d8 100644 --- a/packages/server/destinations/criteo/src/schemas/settings.ts +++ b/packages/server/destinations/criteo/src/schemas/settings.ts @@ -5,44 +5,50 @@ const InputSchema = z.union([ z.literal(false), ]); -export const SettingsSchema = z.object({ - partnerId: z - .string() - .min(1) - .describe('Criteo Partner ID (numeric string, provided by Criteo)'), - callerId: z - .string() - .min(1) - .describe('Caller ID for user mapping (provided by Criteo)'), - siteType: z - .enum(['d', 'm', 't']) - .describe('Site type: d (desktop), m (mobile web), t (tablet)') - .optional(), - country: z - .string() - .length(2) - .describe('ISO 3166-1 alpha-2 country code') - .optional(), - language: z.string().length(2).describe('2-letter language code').optional(), - url: z - .string() - .url() - .describe( - 'Custom Events API endpoint (default https://widget.criteo.com/m/event?version=s2s_v0)', - ) - .optional(), - user_data: z - .record(z.string(), z.string()) - .describe( - "Mapping for identity fields (like { email: 'user.email', mapped_user_id: 'user.id' })", - ) - .optional(), - ip: InputSchema.describe( - 'Client IP, sent as ip. Resolves against { ingest, event }. Default: ["ingest.ip", "event.user.ip"]. false switches it off. A mapped user_data.ip wins.', - ).optional(), - userAgent: InputSchema.describe( - 'Client user agent, sent as useragent. Resolves against { ingest, event }. Default: ["ingest.userAgent", "event.user.userAgent"]. false switches it off. A mapped user_data.useragent wins.', - ).optional(), -}); +export const SettingsSchema = z + .object({ + partnerId: z + .string() + .min(1) + .describe('Criteo Partner ID (numeric string, provided by Criteo)'), + callerId: z + .string() + .min(1) + .describe('Caller ID for user mapping (provided by Criteo)'), + siteType: z + .enum(['d', 'm', 't']) + .describe('Site type: d (desktop), m (mobile web), t (tablet)') + .optional(), + country: z + .string() + .length(2) + .describe('ISO 3166-1 alpha-2 country code') + .optional(), + language: z + .string() + .length(2) + .describe('2-letter language code') + .optional(), + url: z + .string() + .url() + .describe( + 'Custom Events API endpoint (default https://widget.criteo.com/m/event?version=s2s_v0)', + ) + .optional(), + user_data: z + .record(z.string(), z.string()) + .describe( + "Mapping for identity fields (like { email: 'user.email', mapped_user_id: 'user.id' })", + ) + .optional(), + ip: InputSchema.describe( + 'Client IP, sent as ip. Resolves against { ingest, event }. Default: ["ingest.ip", "event.user.ip"]. false switches it off. A mapped user_data.ip wins.', + ).optional(), + userAgent: InputSchema.describe( + 'Client user agent, sent as useragent. Resolves against { ingest, event }. Default: ["ingest.userAgent", "event.user.userAgent"]. false switches it off. A mapped user_data.useragent wins.', + ).optional(), + }) + .strict(); export type Settings = z.infer; diff --git a/packages/server/destinations/customerio/src/__tests__/conformance.test.ts b/packages/server/destinations/customerio/src/__tests__/conformance.test.ts new file mode 100644 index 000000000..a52d25745 --- /dev/null +++ b/packages/server/destinations/customerio/src/__tests__/conformance.test.ts @@ -0,0 +1,21 @@ +import path from 'path'; +import { conformance } from '@walkeros/collector/dev'; +import { destinationCustomerIo } from '..'; +import { examples, schemas } from '../dev'; + +describe('customerio conformance', () => { + beforeEach(() => { + jest.useRealTimers(); + }); + + it('conforms', async () => { + await conformance('destination', destinationCustomerIo, { + profile: 'sdk-env', + examples: examples.step, + schema: schemas.settings, + settings: { siteId: 'site', apiKey: 'key' }, + env: { push: examples.env.push, simulation: examples.env.simulation }, + root: path.join(__dirname, '..', '..'), + }); + }); +}); diff --git a/packages/server/destinations/customerio/src/__tests__/status.test.ts b/packages/server/destinations/customerio/src/__tests__/status.test.ts new file mode 100644 index 000000000..9b3b3c25e --- /dev/null +++ b/packages/server/destinations/customerio/src/__tests__/status.test.ts @@ -0,0 +1,46 @@ +import { startFlow } from '@walkeros/collector'; +import { getEvent } from '@walkeros/core'; +import { destinationCustomerIo } from '..'; +import { examples } from '../dev'; +import type { Env } from '../types'; + +describe('customerio failure status', () => { + async function failWith(statusCode: number | undefined) { + const error = Object.assign( + new Error('request failed'), + statusCode === undefined ? {} : { statusCode }, + ); + const { trackClient } = examples.env.push; + if (!trackClient) throw new Error('mock env lacks trackClient'); + const env: Env = { + trackClient: { ...trackClient, track: () => Promise.reject(error) }, + }; + const { collector } = await startFlow({ + destinations: { + dest: { + code: { ...destinationCustomerIo, env }, + config: { settings: { siteId: 'site', apiKey: 'key' } }, + }, + }, + }); + const result = await collector.push( + getEvent('page view', { user: { id: 'u1' } }), + ); + return result.failed?.dest?.reason; + } + + it('does not retry a 400', async () => { + expect(await failWith(400)).toMatchObject({ + code: 'transport', + status: 400, + retryable: false, + }); + }); + + it('retries a 503', async () => { + expect(await failWith(503)).toMatchObject({ + status: 503, + retryable: true, + }); + }); +}); diff --git a/packages/server/destinations/customerio/src/__tests__/stepExamples.test.ts b/packages/server/destinations/customerio/src/__tests__/stepExamples.test.ts index 2351abb8d..6a6f3f5b2 100644 --- a/packages/server/destinations/customerio/src/__tests__/stepExamples.test.ts +++ b/packages/server/destinations/customerio/src/__tests__/stepExamples.test.ts @@ -113,7 +113,12 @@ function spyEnv(): { env: Env; collected: () => CallRecord[] } { } describe('customerio server destination -- step examples', () => { - it.each(Object.entries(examples.step))('%s', async (_name, rawExample) => { + // Non-public examples (silent, failures) run in the conformance test. + const published = Object.entries(examples.step).filter( + ([, example]) => example.public !== false, + ); + + it.each(published)('%s', async (_name, rawExample) => { const example = rawExample as { in?: unknown; mapping?: unknown; diff --git a/packages/server/destinations/customerio/src/config.ts b/packages/server/destinations/customerio/src/config.ts index a6484d7a3..37f0ee5a6 100644 --- a/packages/server/destinations/customerio/src/config.ts +++ b/packages/server/destinations/customerio/src/config.ts @@ -1,20 +1,24 @@ import type { Config, Settings, PartialConfig } from './types'; -import type { Logger } from '@walkeros/core'; +import { StepError } from '@walkeros/core'; -export function getConfig( - partialConfig: PartialConfig = {}, - logger: Logger.Instance, -): Config { +export function getConfig(partialConfig: PartialConfig = {}): Config { const settings = (partialConfig.settings || {}) as Partial; const { siteId, apiKey } = settings; - if (!siteId) logger.throw('Config settings siteId missing'); - if (!apiKey) logger.throw('Config settings apiKey missing'); + if (!siteId || !apiKey) { + const missing = [ + ...(siteId ? [] : ['siteId']), + ...(apiKey ? [] : ['apiKey']), + ]; + throw new StepError(`Config settings ${missing.join(', ')} missing`, { + code: 'config', + }); + } const settingsConfig: Settings = { ...settings, - siteId: siteId as string, - apiKey: apiKey as string, + siteId, + apiKey, // Default identity resolution paths customerId: settings.customerId ?? 'user.id', anonymousId: settings.anonymousId ?? 'user.session', diff --git a/packages/server/destinations/customerio/src/examples/step.ts b/packages/server/destinations/customerio/src/examples/step.ts index 4fdc35595..549b6def6 100644 --- a/packages/server/destinations/customerio/src/examples/step.ts +++ b/packages/server/destinations/customerio/src/examples/step.ts @@ -308,3 +308,52 @@ export const wildcardIgnored: CustomerIoStepExample = { mapping: { ignore: true }, out: [], }; + +/** A silent rule sends nothing to Customer.io. */ +export const silentRule: Flow.StepExample = { + public: false, + description: 'A rule with silent: true sends nothing to Customer.io.', + in: getEvent('page view', { timestamp: 1700000990000 }), + mapping: { silent: true }, + out: [], +}; + +/** + * No identity -- Customer.io needs a customerId or anonymousId for the event, so the + * push fails with missing-input instead of counting as delivered. One per + * event name above, so every example's event without identity has its + * declared outcome. + */ +function withoutIdentity(name: string, timestamp: number): Flow.StepExample { + const [entity, action] = name.split(' '); + return { + public: false, + description: `A ${name} event without a customerId or anonymousId cannot be sent to Customer.io: the push fails with missing-input instead of counting as delivered.`, + in: { name, entity, action, timestamp }, + out: [], + outcome: { status: 'failed', code: 'missing-input', retryable: false }, + }; +} + +export const noIdentityProductView = withoutIdentity( + 'product view', + 1700000991000, +); +export const noIdentityOrderComplete = withoutIdentity( + 'order complete', + 1700000991001, +); +export const noIdentityPageView = withoutIdentity('page view', 1700000991002); +export const noIdentityUserLogin = withoutIdentity('user login', 1700000991003); +export const noIdentityUserDelete = withoutIdentity( + 'user delete', + 1700000991004, +); +export const noIdentityUserSuppress = withoutIdentity( + 'user suppress', + 1700000991005, +); +export const noIdentityUserUnsuppress = withoutIdentity( + 'user unsuppress', + 1700000991006, +); diff --git a/packages/server/destinations/customerio/src/index.ts b/packages/server/destinations/customerio/src/index.ts index 63fa503cb..a7ec7562d 100644 --- a/packages/server/destinations/customerio/src/index.ts +++ b/packages/server/destinations/customerio/src/index.ts @@ -4,6 +4,7 @@ import type { CustomerIoTrackClientMock, CustomerIoApiClientMock, } from './types'; +import { StepError } from '@walkeros/core'; import { getConfig } from './config'; import { push } from './push'; @@ -15,8 +16,8 @@ export const destinationCustomerIo: Destination = { config: {}, - init({ config: partialConfig, logger, env }) { - const config = getConfig(partialConfig, logger); + init({ config: partialConfig, env }) { + const config = getConfig(partialConfig); const settings = config.settings as Settings; // Use env mocks if provided (testing), otherwise create real SDK clients @@ -40,7 +41,10 @@ export const destinationCustomerIo: Destination = { }); settings._trackClient = trackClient as CustomerIoTrackClientMock; } catch (err) { - logger.throw(`Failed to initialize Customer.io TrackClient: ${err}`); + throw new StepError('Failed to initialize Customer.io TrackClient', { + code: 'config', + cause: err, + }); } } @@ -56,7 +60,10 @@ export const destinationCustomerIo: Destination = { }); settings._apiClient = apiClient as CustomerIoApiClientMock; } catch (err) { - logger.throw(`Failed to initialize Customer.io APIClient: ${err}`); + throw new StepError('Failed to initialize Customer.io APIClient', { + code: 'config', + cause: err, + }); } } diff --git a/packages/server/destinations/customerio/src/push.ts b/packages/server/destinations/customerio/src/push.ts index 743ebc369..45606604d 100644 --- a/packages/server/destinations/customerio/src/push.ts +++ b/packages/server/destinations/customerio/src/push.ts @@ -5,11 +5,40 @@ import type { CustomerIoTrackClientMock, CustomerIoApiClientMock, } from './types'; -import { getMappingValue, isObject, isString } from '@walkeros/core'; +import { getMappingValue, isObject, isString, StepError } from '@walkeros/core'; + +/** + * One call to the SDK, awaited. A failure becomes a StepError with a fixed + * message: the SDK's own error stays on `cause`, never in the message. + */ +async function sdk( + message: string, + call: () => Promise, +): Promise { + try { + await call(); + } catch (error) { + throw new StepError(message, { + code: 'transport', + status: statusOf(error), + cause: error, + }); + } +} + +/** The HTTP status of a rejected request (CustomerIORequestError), when known. */ +function statusOf(error: unknown): number | undefined { + if (typeof error !== 'object' || error === null) return undefined; + return 'statusCode' in error && + typeof error.statusCode === 'number' && + error.statusCode > 0 + ? error.statusCode + : undefined; +} export const push: PushFn = async function ( event, - { config, rule, data, collector, env, logger }, + { config, rule, data, collector, env }, ) { const settings = config.settings as Settings; const envTyped = env as @@ -21,10 +50,10 @@ export const push: PushFn = async function ( const trackClient = envTyped?.trackClient || settings._trackClient; const apiClient = envTyped?.apiClient || settings._apiClient; - if (!trackClient) { - logger.warn('Customer.io TrackClient not initialized'); - return; - } + if (!trackClient) + throw new StepError('Customer.io TrackClient not initialized', { + code: 'not-ready', + }); const state: RuntimeState = settings._state || {}; const mappingSettings = rule?.settings || {}; @@ -46,13 +75,10 @@ export const push: PushFn = async function ( ) : undefined; - if (!customerId && !anonymousId) { - logger.warn( - 'Customer.io requires customerId or anonymousId; skipping event', - { event: event.name }, - ); - return; - } + if (!customerId && !anonymousId) + throw new StepError('Customer.io requires customerId or anonymousId', { + code: 'missing-input', + }); // 2. Identify -- rule-level overrides destination-level const identifyMapping = mappingSettings.identify ?? settings.identify; @@ -83,20 +109,28 @@ export const push: PushFn = async function ( void _u; const pageData = Object.keys(extraData).length > 0 ? extraData : undefined; - await trackClient.trackPageView(customerId, url, pageData); + await sdk('Customer.io trackPageView failed', () => + trackClient.trackPageView(customerId, url, pageData), + ); } } } // 4. Destroy / Suppress / Unsuppress if (mappingSettings.destroy === true && customerId) { - await trackClient.destroy(customerId); + await sdk('Customer.io destroy failed', () => + trackClient.destroy(customerId), + ); } if (mappingSettings.suppress === true && customerId) { - await trackClient.suppress(customerId); + await sdk('Customer.io suppress failed', () => + trackClient.suppress(customerId), + ); } if (mappingSettings.unsuppress === true && customerId) { - await trackClient.unsuppress(customerId); + await sdk('Customer.io unsuppress failed', () => + trackClient.unsuppress(customerId), + ); } // 5. Device management @@ -110,12 +144,15 @@ export const push: PushFn = async function ( platform?: unknown; data?: unknown; }; - if (isString(r.deviceId) && isString(r.platform)) { - await trackClient.addDevice( - customerId, - r.deviceId, - r.platform, - isObject(r.data) ? (r.data as Record) : undefined, + const { deviceId, platform } = r; + if (isString(deviceId) && isString(platform)) { + await sdk('Customer.io addDevice failed', () => + trackClient.addDevice( + customerId, + deviceId, + platform, + isObject(r.data) ? (r.data as Record) : undefined, + ), ); } } @@ -128,8 +165,11 @@ export const push: PushFn = async function ( ); if (isObject(resolved)) { const r = resolved as { deviceId?: unknown; platform?: unknown }; - if (isString(r.deviceId) && isString(r.platform)) { - await trackClient.deleteDevice(customerId, r.deviceId, r.platform); + const { deviceId, platform } = r; + if (isString(deviceId) && isString(platform)) { + await sdk('Customer.io deleteDevice failed', () => + trackClient.deleteDevice(customerId, deviceId, platform), + ); } } } @@ -146,17 +186,20 @@ export const push: PushFn = async function ( secondaryType?: unknown; secondaryId?: unknown; }; + const { primaryType, primaryId, secondaryType, secondaryId } = r; if ( - isString(r.primaryType) && - isString(r.primaryId) && - isString(r.secondaryType) && - isString(r.secondaryId) + isString(primaryType) && + isString(primaryId) && + isString(secondaryType) && + isString(secondaryId) ) { - await trackClient.mergeCustomers( - r.primaryType, - r.primaryId, - r.secondaryType, - r.secondaryId, + await sdk('Customer.io mergeCustomers failed', () => + trackClient.mergeCustomers( + primaryType, + primaryId, + secondaryType, + secondaryId, + ), ); } } @@ -168,7 +211,9 @@ export const push: PushFn = async function ( collector, }); if (isObject(resolved)) { - await apiClient.sendEmail(resolved); + await sdk('Customer.io sendEmail failed', () => + apiClient.sendEmail(resolved), + ); } } if (mappingSettings.sendPush !== undefined && apiClient) { @@ -176,7 +221,9 @@ export const push: PushFn = async function ( collector, }); if (isObject(resolved)) { - await apiClient.sendPush(resolved); + await sdk('Customer.io sendPush failed', () => + apiClient.sendPush(resolved), + ); } } @@ -194,9 +241,13 @@ export const push: PushFn = async function ( if (timestampSec !== undefined) eventData.timestamp = timestampSec; if (customerId) { - await trackClient.track(customerId, eventData); + await sdk('Customer.io track failed', () => + trackClient.track(customerId, eventData), + ); } else if (anonymousId) { - await trackClient.trackAnonymous(anonymousId, eventData); + await sdk('Customer.io trackAnonymous failed', () => + trackClient.trackAnonymous(anonymousId, eventData), + ); } } @@ -231,7 +282,9 @@ async function applyIdentify( if (!customerIdChanged && !attributesChanged) return; - await trackClient.identify(customerId, resolved); + await sdk('Customer.io identify failed', () => + trackClient.identify(customerId, resolved), + ); state.lastIdentity = { customerId, diff --git a/packages/server/destinations/customerio/src/schemas/settings.ts b/packages/server/destinations/customerio/src/schemas/settings.ts index 58b4215f8..1627ca061 100644 --- a/packages/server/destinations/customerio/src/schemas/settings.ts +++ b/packages/server/destinations/customerio/src/schemas/settings.ts @@ -1,54 +1,56 @@ import { z } from '@walkeros/core/dev'; -export const SettingsSchema = z.object({ - siteId: z - .string() - .min(1) - .describe( - 'Customer.io Site ID. Find it in Settings > Workspace Settings > API Credentials.', - ), - apiKey: z - .string() - .min(1) - .describe( - 'Customer.io API Key. Find it in Settings > Workspace Settings > API Credentials.', - ), - appApiKey: z - .string() - .describe( - 'App API Key for transactional messaging (sendEmail/sendPush). Find it in Settings > Workspace Settings > API Credentials > App API Keys.', - ) - .optional(), - region: z - .enum(['us', 'eu']) - .describe( - 'Data center region. Must match where your Customer.io workspace was created. Default: us.', - ) - .optional(), - timeout: z - .number() - .int() - .positive() - .describe('HTTP request timeout in milliseconds. Default: 10000.') - .optional(), - customerId: z - .string() - .describe( - 'walkerOS mapping value path to resolve customerId from each event (like user.id).', - ) - .optional(), - anonymousId: z - .string() - .describe( - 'walkerOS mapping value path to resolve anonymousId from each event (like user.session).', - ) - .optional(), - identify: z - .unknown() - .describe( - 'Destination-level identity mapping. Resolves to { email?, first_name?, ... } attributes. Fires identify() on first push and re-fires when values change.', - ) - .optional(), -}); +export const SettingsSchema = z + .object({ + siteId: z + .string() + .min(1) + .describe( + 'Customer.io Site ID. Find it in Settings > Workspace Settings > API Credentials.', + ), + apiKey: z + .string() + .min(1) + .describe( + 'Customer.io API Key. Find it in Settings > Workspace Settings > API Credentials.', + ), + appApiKey: z + .string() + .describe( + 'App API Key for transactional messaging (sendEmail/sendPush). Find it in Settings > Workspace Settings > API Credentials > App API Keys.', + ) + .optional(), + region: z + .enum(['us', 'eu']) + .describe( + 'Data center region. Must match where your Customer.io workspace was created. Default: us.', + ) + .optional(), + timeout: z + .number() + .int() + .positive() + .describe('HTTP request timeout in milliseconds. Default: 10000.') + .optional(), + customerId: z + .string() + .describe( + 'walkerOS mapping value path to resolve customerId from each event (like user.id).', + ) + .optional(), + anonymousId: z + .string() + .describe( + 'walkerOS mapping value path to resolve anonymousId from each event (like user.session).', + ) + .optional(), + identify: z + .unknown() + .describe( + 'Destination-level identity mapping. Resolves to { email?, first_name?, ... } attributes. Fires identify() on first push and re-fires when values change.', + ) + .optional(), + }) + .strict(); export type Settings = z.infer; diff --git a/packages/server/destinations/datamanager/src/__tests__/conformance.test.ts b/packages/server/destinations/datamanager/src/__tests__/conformance.test.ts new file mode 100644 index 000000000..905b10961 --- /dev/null +++ b/packages/server/destinations/datamanager/src/__tests__/conformance.test.ts @@ -0,0 +1,44 @@ +import path from 'path'; +import { conformance } from '@walkeros/collector/dev'; +import { destinationDataManager } from '..'; +import { examples, schemas } from '../dev'; + +// The runs without an injected auth client build their own: a stand-in that +// answers with a token, so no Google OAuth is attempted. +jest.mock('google-auth-library', () => ({ + __esModule: true, + GoogleAuth: class { + getClient() { + return Promise.resolve({ + getAccessToken: () => Promise.resolve({ token: 'ya29.c.test_token' }), + }); + } + }, +})); + +describe('datamanager conformance', () => { + beforeEach(() => { + jest.useRealTimers(); + }); + + it('conforms', async () => { + await conformance('destination', destinationDataManager, { + profile: 'sdk-env', + examples: examples.step, + schema: schemas.settings, + settings: { + destinations: [ + { + operatingAccount: { + accountId: '123-456-7890', + accountType: 'GOOGLE_ADS', + }, + productDestinationId: 'AW-CONVERSION-123', + }, + ], + }, + env: { push: examples.env.push, simulation: examples.env.simulation }, + root: path.join(__dirname, '..', '..'), + }); + }); +}); diff --git a/packages/server/destinations/datamanager/src/__tests__/index.test.ts b/packages/server/destinations/datamanager/src/__tests__/index.test.ts index b9a655d0f..8c087615c 100644 --- a/packages/server/destinations/datamanager/src/__tests__/index.test.ts +++ b/packages/server/destinations/datamanager/src/__tests__/index.test.ts @@ -477,10 +477,13 @@ describe('Server Destination Data Manager', () => { expect(requestBody.testEventCode).toBe('TEST12345'); }); - test('throws error on API failure', async () => { + test.each([ + [400, false], + [503, true], + ])('API failure %i has retryable %s', async (status, retryable) => { mockFetch.mockResolvedValue({ ok: false, - status: 400, + status, text: async () => 'Invalid request', }); @@ -496,7 +499,11 @@ describe('Server Destination Data Manager', () => { id: 'test-dm', }), ), - ).rejects.toThrow('Data Manager API error (400)'); + ).rejects.toMatchObject({ + message: `Data Manager API error (${status})`, + status, + retryable, + }); }); test('throws error on validation errors', async () => { @@ -526,7 +533,7 @@ describe('Server Destination Data Manager', () => { id: 'test-dm', }), ), - ).rejects.toThrow('Validation errors'); + ).rejects.toThrow('Data Manager rejected the event'); }); test('uses custom URL when provided', async () => { diff --git a/packages/server/destinations/datamanager/src/__tests__/stepExamples.test.ts b/packages/server/destinations/datamanager/src/__tests__/stepExamples.test.ts index 031f96395..bb5d97b89 100644 --- a/packages/server/destinations/datamanager/src/__tests__/stepExamples.test.ts +++ b/packages/server/destinations/datamanager/src/__tests__/stepExamples.test.ts @@ -52,7 +52,12 @@ describe('Step Examples', () => { }); }); - it.each(Object.entries(examples.step))('%s', async (_name, example) => { + // Non-public examples (silent, failures) run in the conformance test. + const published = Object.entries(examples.step).filter( + ([, example]) => example.public !== false, + ); + + it.each(published)('%s', async (_name, example) => { const event = example.in as WalkerOS.Event; const mapping = example.mapping; diff --git a/packages/server/destinations/datamanager/src/config.ts b/packages/server/destinations/datamanager/src/config.ts index b68814545..1f3531d0f 100644 --- a/packages/server/destinations/datamanager/src/config.ts +++ b/packages/server/destinations/datamanager/src/config.ts @@ -4,17 +4,16 @@ import type { PartialConfig, EventSource, } from './types'; -import type { Logger } from '@walkeros/core'; +import { StepError } from '@walkeros/core'; -export function getConfig( - partialConfig: PartialConfig = {}, - logger: Logger.Instance, -): ValidatedConfig { +export function getConfig(partialConfig: PartialConfig = {}): ValidatedConfig { const settings = (partialConfig.settings || {}) as Partial; const { destinations, eventSource = 'WEB' } = settings; if (!destinations || destinations.length === 0) - logger.throw('Config settings destinations missing or empty'); + throw new StepError('Config settings destinations missing or empty', { + code: 'config', + }); const settingsConfig: Settings & { eventSource: EventSource } = { ...settings, diff --git a/packages/server/destinations/datamanager/src/examples/step.ts b/packages/server/destinations/datamanager/src/examples/step.ts index 62c221710..2c152c0a2 100644 --- a/packages/server/destinations/datamanager/src/examples/step.ts +++ b/packages/server/destinations/datamanager/src/examples/step.ts @@ -202,3 +202,32 @@ export const ga4PageView: Flow.StepExample = { ], ], }; + +/** A silent rule sends nothing to the Data Manager API. */ +export const silentRule: Flow.StepExample = { + public: false, + description: + 'A rule with silent: true sends no request to the Data Manager API.', + in: getEvent('page view', { timestamp: 1700000990000 }), + mapping: { silent: true }, + out: [], +}; + +/** + * No transaction id -- an order complete event without data has no + * transactionId, which Data Manager requires, so the push fails with + * missing-input instead of counting as delivered. + */ +export const orderWithoutTransactionId: Flow.StepExample = { + public: false, + description: + 'An order complete event without a transaction id cannot be sent to the Data Manager API: the push fails with missing-input instead of counting as delivered.', + in: { + name: 'order complete', + entity: 'order', + action: 'complete', + timestamp: 1700000991000, + }, + out: [], + outcome: { status: 'failed', code: 'missing-input', retryable: false }, +}; diff --git a/packages/server/destinations/datamanager/src/index.ts b/packages/server/destinations/datamanager/src/index.ts index 234ad35d6..c5958013d 100644 --- a/packages/server/destinations/datamanager/src/index.ts +++ b/packages/server/destinations/datamanager/src/index.ts @@ -1,4 +1,5 @@ import type { DestinationInterface } from './types'; +import { StepError } from '@walkeros/core'; import { getConfig } from './config'; import { push } from './push'; import { createAuthClient } from './auth'; @@ -12,7 +13,7 @@ export const destinationDataManager: DestinationInterface = { async init({ config: partialConfig, env, logger }) { // getConfig validates required fields and returns ValidatedConfig - const config = getConfig(partialConfig, logger); + const config = getConfig(partialConfig); // Keep an injected client (tests, simulate); create one only when none is given. if (env?.authClient) return { ...config, env }; @@ -23,9 +24,10 @@ export const destinationDataManager: DestinationInterface = { return { ...config, env: { ...env, authClient } }; } catch (error) { - logger.throw( - `Data Manager authentication failed: ${error instanceof Error ? error.message : 'Unknown error'}`, - ); + throw new StepError('Data Manager authentication failed', { + code: 'config', + cause: error, + }); } }, diff --git a/packages/server/destinations/datamanager/src/push.ts b/packages/server/destinations/datamanager/src/push.ts index a32ef14f3..6b2f9c004 100644 --- a/packages/server/destinations/datamanager/src/push.ts +++ b/packages/server/destinations/datamanager/src/push.ts @@ -1,17 +1,27 @@ import type { PushFn } from './types'; import type { IngestEventsRequest, IngestEventsResponse } from './types'; import type { Collector, Mapping as WalkerOSMapping } from '@walkeros/core'; -import { createMappingRoot, getMappingValue, isObject } from '@walkeros/core'; +import { + createMappingRoot, + getMappingValue, + isObject, + StepError, +} from '@walkeros/core'; import { formatEvent, formatConsent } from './format'; import { getAccessToken } from './auth'; import { getConfig } from './config'; -export const push: PushFn = async function ( +// A silent rule has no Data Manager call of its own: nothing to send. +export const push: PushFn = async function (event, context) { + if (!context.rule?.silent) await send(event, context); +}; + +const send: PushFn = async function ( event, - { config, rule, data, ingest, collector, env, logger }, + { config, data, ingest, collector, env, logger }, ) { // Validate config and get typed settings - const validatedConfig = getConfig(config, logger); + const validatedConfig = getConfig(config); const { destinations, eventSource, @@ -119,7 +129,9 @@ export const push: PushFn = async function ( // Validate required fields before API call if (!dataManagerEvent.transactionId) { - logger.throw('transactionId is required'); + throw new StepError('transactionId is required', { + code: 'missing-input', + }); } // Check if any destination is GA4 (requires eventName) @@ -128,7 +140,9 @@ export const push: PushFn = async function ( ); if (hasGA4Destination && !dataManagerEvent.eventName) { - logger.throw('eventName is required for GA4 destinations'); + throw new StepError('eventName is required for GA4 destinations', { + code: 'missing-input', + }); } // Build API request. Encoding is pinned to HEX because all hashing in this @@ -155,18 +169,13 @@ export const push: PushFn = async function ( const authClient = env?.authClient; if (!authClient) { - return logger.throw( + throw new StepError( 'Auth client not initialized. Ensure init() was called successfully.', + { code: 'not-ready' }, ); } - let accessToken: string; - try { - accessToken = await getAccessToken(authClient); - } catch (error) { - logger.error('Authentication failed', { error }); - throw error; - } + const accessToken = await getAccessToken(authClient); const fetchFn = env?.fetch || fetch; const endpoint = `${url}/events:ingest`; @@ -178,18 +187,31 @@ export const push: PushFn = async function ( validateOnly, }); - const response = await fetchFn(endpoint, { - method: 'POST', - headers: { - Authorization: `Bearer ${accessToken}`, - 'Content-Type': 'application/json', - }, - body: JSON.stringify(requestBody), - }); + // A transport failure keeps the fetch error as `cause`, never in the message. + let response: Response; + try { + response = await fetchFn(endpoint, { + method: 'POST', + headers: { + Authorization: `Bearer ${accessToken}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify(requestBody), + }); + } catch (error) { + throw new StepError('Data Manager request failed', { + code: 'transport', + cause: error, + }); + } if (!response.ok) { - const errorText = await response.text(); - logger.throw(`Data Manager API error (${response.status}): ${errorText}`); + // The API's answer stays on the error body, never in the message. + throw new StepError(`Data Manager API error (${response.status})`, { + code: 'transport', + status: response.status, + body: await response.text(), + }); } const result: IngestEventsResponse = await response.json(); @@ -201,9 +223,11 @@ export const push: PushFn = async function ( // If validation errors exist, throw them if (result.validationErrors && result.validationErrors.length > 0) { - logger.throw( - `Validation errors: ${JSON.stringify(result.validationErrors)}`, - ); + throw new StepError('Data Manager rejected the event', { + code: 'transport', + retryable: false, + body: result.validationErrors, + }); } }; diff --git a/packages/server/destinations/datamanager/src/schemas/settings.ts b/packages/server/destinations/datamanager/src/schemas/settings.ts index 512a8dcda..afe76ca06 100644 --- a/packages/server/destinations/datamanager/src/schemas/settings.ts +++ b/packages/server/destinations/datamanager/src/schemas/settings.ts @@ -21,121 +21,119 @@ const CredentialsSchema = z.object({ .describe('Service account private key (PEM format)'), }); -export const SettingsSchema = z.object({ - credentials: CredentialsSchema.optional().describe( - 'Service account credentials (client_email + private_key). Recommended for serverless environments. (deprecated: use config.credentials)', - ), - keyFilename: z - .string() - .optional() - .describe( - 'Path to service account JSON file. For local development or environments with filesystem access.', - ), - scopes: z - .array(z.string()) - .optional() - .describe( - 'OAuth scopes for Data Manager API. Defaults to datamanager scope.', - ), - destinations: z - .array(DestinationSchema) - .min(1) - .max(10) - .describe( - 'Array of destination accounts and conversion actions/user lists (max 10)', - ), - eventSource: EventSourceSchema.optional() - .default('WEB') - .describe( - 'Event source for all events. Defaults to WEB. Values: WEB, APP, IN_STORE, PHONE, OTHER', +export const SettingsSchema = z + .object({ + credentials: CredentialsSchema.optional().describe( + 'Service account credentials (client_email + private_key). Recommended for serverless environments. (deprecated: use config.credentials)', ), - batchSize: z - .number() - .int() - .min(1) - .max(2000) - .describe( - 'Maximum number of events to batch before sending (max 2000, like 100)', - ) - .optional(), - batchInterval: z - .number() - .int() - .min(0) - .describe( - 'Time in milliseconds to wait before auto-flushing batch (like 5000)', - ) - .optional(), - validateOnly: z - .boolean() - .describe('If true, validate request without ingestion (testing mode)') - .optional(), - url: z - .string() - .url() - .describe( - 'Override API endpoint for testing (like https://datamanager.googleapis.com/v1)', - ) - .optional(), - consent: ConsentSchema.describe( - 'Request-level consent for all events', - ).optional(), - testEventCode: z - .string() - .describe('Test event code for debugging (like TEST12345)') - .optional(), - logLevel: z - .enum(['debug', 'info', 'warn', 'error', 'none']) - .describe('Log level for debugging (debug shows all API calls)') - .optional(), - userData: z - .record(z.string(), z.unknown()) - .describe( - "Guided helper: User data mapping for all events (like { email: 'user.id', phone: 'data.phone' })", - ) - .optional(), - userId: z - .any() - .describe( - "Guided helper: First-party user ID for all events (like 'user.id')", - ) - .optional(), - clientId: z - .any() - .describe( - "Guided helper: GA4 client ID for all events (like 'user.device')", - ) - .optional(), - appInstanceId: z - .any() - .describe( - "Guided helper: GA4 app instance ID (Firebase) for all events (like 'user.appInstanceId')", - ) - .optional(), - sessionAttributes: z - .any() - .describe( - "Guided helper: Privacy-safe attribution for all events (like 'context.sessionAttributes')", - ) - .optional(), - consentAdUserData: z - .union([z.string(), z.boolean()]) - .describe( - "Consent mapping: Field name from event.consent (like 'marketing') or static boolean value", - ) - .optional(), - consentAdPersonalization: z - .union([z.string(), z.boolean()]) - .describe( - "Consent mapping: Field name from event.consent (like 'targeting') or static boolean value", - ) - .optional(), - ip: InputSchema.describe( - 'Client IP, sent as eventDeviceInfo.ipAddress. Resolves against { ingest, event }. Default: ["ingest.ip", "event.user.ip"]. false switches it off. A mapped ipAddress wins.', - ).optional(), - userAgent: InputSchema.describe( - 'Client user agent, sent as eventDeviceInfo.userAgent. Resolves against { ingest, event }. Default: ["ingest.userAgent", "event.user.userAgent"]. false switches it off. A mapped userAgent wins.', - ).optional(), -}); + keyFilename: z + .string() + .optional() + .describe( + 'Path to service account JSON file. For local development or environments with filesystem access.', + ), + scopes: z + .array(z.string()) + .optional() + .describe( + 'OAuth scopes for Data Manager API. Defaults to datamanager scope.', + ), + destinations: z + .array(DestinationSchema) + .min(1) + .max(10) + .describe( + 'Array of destination accounts and conversion actions/user lists (max 10)', + ), + eventSource: EventSourceSchema.optional() + .default('WEB') + .describe( + 'Event source for all events. Defaults to WEB. Values: WEB, APP, IN_STORE, PHONE, OTHER', + ), + batchSize: z + .number() + .int() + .min(1) + .max(2000) + .describe( + 'Maximum number of events to batch before sending (max 2000, like 100)', + ) + .optional(), + batchInterval: z + .number() + .int() + .min(0) + .describe( + 'Time in milliseconds to wait before auto-flushing batch (like 5000)', + ) + .optional(), + validateOnly: z + .boolean() + .describe('If true, validate request without ingestion (testing mode)') + .optional(), + url: z + .string() + .url() + .describe( + 'Override API endpoint for testing (like https://datamanager.googleapis.com/v1)', + ) + .optional(), + consent: ConsentSchema.describe( + 'Request-level consent for all events', + ).optional(), + testEventCode: z + .string() + .describe('Test event code for debugging (like TEST12345)') + .optional(), + userData: z + .record(z.string(), z.unknown()) + .describe( + "Guided helper: User data mapping for all events (like { email: 'user.id', phone: 'data.phone' })", + ) + .optional(), + userId: z + .any() + .describe( + "Guided helper: First-party user ID for all events (like 'user.id')", + ) + .optional(), + clientId: z + .any() + .describe( + "Guided helper: GA4 client ID for all events (like 'user.device')", + ) + .optional(), + appInstanceId: z + .any() + .describe( + "Guided helper: GA4 app instance ID (Firebase) for all events (like 'user.appInstanceId')", + ) + .optional(), + sessionAttributes: z + .any() + .describe( + "Guided helper: Privacy-safe attribution for all events (like 'context.sessionAttributes')", + ) + .optional(), + consentAdUserData: z + .union([z.string(), z.boolean()]) + .describe( + "Consent mapping: Field name from event.consent (like 'marketing') or static boolean value", + ) + .optional(), + consentAdPersonalization: z + .union([z.string(), z.boolean()]) + .describe( + "Consent mapping: Field name from event.consent (like 'targeting') or static boolean value", + ) + .optional(), + ip: InputSchema.describe( + 'Client IP, sent as eventDeviceInfo.ipAddress. Resolves against { ingest, event }. Default: ["ingest.ip", "event.user.ip"]. false switches it off. A mapped ipAddress wins.', + ).optional(), + userAgent: InputSchema.describe( + 'Client user agent, sent as eventDeviceInfo.userAgent. Resolves against { ingest, event }. Default: ["ingest.userAgent", "event.user.userAgent"]. false switches it off. A mapped userAgent wins.', + ).optional(), + }) + .strict(); export type Settings = z.infer; diff --git a/packages/server/destinations/klaviyo/src/__tests__/conformance.test.ts b/packages/server/destinations/klaviyo/src/__tests__/conformance.test.ts new file mode 100644 index 000000000..a34fcacc0 --- /dev/null +++ b/packages/server/destinations/klaviyo/src/__tests__/conformance.test.ts @@ -0,0 +1,21 @@ +import path from 'path'; +import { conformance } from '@walkeros/collector/dev'; +import { destinationKlaviyo } from '..'; +import { examples, schemas } from '../dev'; + +describe('klaviyo conformance', () => { + beforeEach(() => { + jest.useRealTimers(); + }); + + it('conforms', async () => { + await conformance('destination', destinationKlaviyo, { + profile: 'sdk-env', + examples: examples.step, + schema: schemas.settings, + settings: { apiKey: 'pk_test_key' }, + env: { push: examples.env.push, simulation: examples.env.simulation }, + root: path.join(__dirname, '..', '..'), + }); + }); +}); diff --git a/packages/server/destinations/klaviyo/src/__tests__/status.test.ts b/packages/server/destinations/klaviyo/src/__tests__/status.test.ts new file mode 100644 index 000000000..9c3c87c49 --- /dev/null +++ b/packages/server/destinations/klaviyo/src/__tests__/status.test.ts @@ -0,0 +1,48 @@ +import { startFlow } from '@walkeros/collector'; +import { getEvent } from '@walkeros/core'; +import { destinationKlaviyo } from '..'; +import type { Env } from '../types'; + +describe('klaviyo failure status', () => { + async function failWith(status: number | undefined) { + const error = Object.assign( + new Error('request failed'), + status === undefined ? {} : { response: { status } }, + ); + const env: Env = { + eventsApi: { createEvent: () => Promise.reject(error) }, + profilesApi: { createOrUpdateProfile: () => Promise.resolve({}) }, + }; + const { collector } = await startFlow({ + destinations: { + dest: { + code: { ...destinationKlaviyo, env }, + config: { settings: { apiKey: 'pk_test_key' } }, + }, + }, + }); + const result = await collector.push( + getEvent('page view', { user: { id: 'u1', email: 'a@b.co' } }), + ); + return result.failed?.dest?.reason; + } + + it('does not retry a 400', async () => { + expect(await failWith(400)).toMatchObject({ + code: 'transport', + status: 400, + retryable: false, + }); + }); + + it('retries a 503', async () => { + expect(await failWith(503)).toMatchObject({ + status: 503, + retryable: true, + }); + }); + + it('retries a network error without a status', async () => { + expect(await failWith(undefined)).toMatchObject({ retryable: true }); + }); +}); diff --git a/packages/server/destinations/klaviyo/src/__tests__/stepExamples.test.ts b/packages/server/destinations/klaviyo/src/__tests__/stepExamples.test.ts index 01d946124..bfebb6931 100644 --- a/packages/server/destinations/klaviyo/src/__tests__/stepExamples.test.ts +++ b/packages/server/destinations/klaviyo/src/__tests__/stepExamples.test.ts @@ -59,7 +59,12 @@ function spyEnv(): { env: Env; collected: () => CallRecord[] } { } describe('klaviyo server destination -- step examples', () => { - it.each(Object.entries(examples.step))('%s', async (_name, rawExample) => { + // Non-public examples (silent, failures) run in the conformance test. + const published = Object.entries(examples.step).filter( + ([, example]) => example.public !== false, + ); + + it.each(published)('%s', async (_name, rawExample) => { const example = rawExample as { in?: unknown; mapping?: unknown; diff --git a/packages/server/destinations/klaviyo/src/config.ts b/packages/server/destinations/klaviyo/src/config.ts index 34a03cf8d..4893b81bb 100644 --- a/packages/server/destinations/klaviyo/src/config.ts +++ b/packages/server/destinations/klaviyo/src/config.ts @@ -1,14 +1,16 @@ import type { Config, Settings, PartialConfig } from './types'; -import type { Logger } from '@walkeros/core'; +import { StepError } from '@walkeros/core'; -export function getConfig( - partialConfig: PartialConfig = {}, - logger: Logger.Instance, -): Config { +export function getConfig(partialConfig: PartialConfig = {}): Config { const settings = (partialConfig.settings || {}) as Partial; const { apiKey } = settings; - if (!apiKey) logger.throw('Config settings apiKey missing'); + if (!apiKey) { + const missing = [...(apiKey ? [] : ['apiKey'])]; + throw new StepError(`Config settings ${missing.join(', ')} missing`, { + code: 'config', + }); + } const settingsConfig: Settings = { ...settings, diff --git a/packages/server/destinations/klaviyo/src/examples/step.ts b/packages/server/destinations/klaviyo/src/examples/step.ts index db2920794..87e747feb 100644 --- a/packages/server/destinations/klaviyo/src/examples/step.ts +++ b/packages/server/destinations/klaviyo/src/examples/step.ts @@ -491,3 +491,44 @@ export const numericUniqueId: KlaviyoStepExample = { ], ], }; + +/** A silent rule sends nothing to Klaviyo. */ +export const silentRule: Flow.StepExample = { + public: false, + description: 'A rule with silent: true sends nothing to Klaviyo.', + in: getEvent('page view', { timestamp: 1700000990000 }), + mapping: { silent: true }, + out: [], +}; + +/** + * No identity -- Klaviyo needs a profile identifier for the event, so the + * push fails with missing-input instead of counting as delivered. One per + * event name above, so every example's event without identity has its + * declared outcome. + */ +function withoutIdentity(name: string, timestamp: number): Flow.StepExample { + const [entity, action] = name.split(' '); + return { + public: false, + description: `A ${name} event without a profile identifier cannot be sent to Klaviyo: the push fails with missing-input instead of counting as delivered.`, + in: { name, entity, action, timestamp }, + out: [], + outcome: { status: 'failed', code: 'missing-input', retryable: false }, + }; +} + +export const noIdentityOrderComplete = withoutIdentity( + 'order complete', + 1700000991000, +); +export const noIdentityProductView = withoutIdentity( + 'product view', + 1700000991001, +); +export const noIdentityPageView = withoutIdentity('page view', 1700000991002); +export const noIdentityNewsletterSignup = withoutIdentity( + 'newsletter signup', + 1700000991003, +); +export const noIdentityUserLogin = withoutIdentity('user login', 1700000991004); diff --git a/packages/server/destinations/klaviyo/src/index.ts b/packages/server/destinations/klaviyo/src/index.ts index 3cfde0028..2f4f2118c 100644 --- a/packages/server/destinations/klaviyo/src/index.ts +++ b/packages/server/destinations/klaviyo/src/index.ts @@ -4,6 +4,7 @@ import type { KlaviyoEventsApiMock, KlaviyoProfilesApiMock, } from './types'; +import { StepError } from '@walkeros/core'; import { getConfig } from './config'; import { push } from './push'; @@ -15,8 +16,8 @@ export const destinationKlaviyo: Destination = { config: {}, - init({ config: partialConfig, logger, env }) { - const config = getConfig(partialConfig, logger); + init({ config: partialConfig, env }) { + const config = getConfig(partialConfig); const settings = config.settings as Settings; // Use env mocks if provided (testing), otherwise create real SDK instances @@ -40,7 +41,10 @@ export const destinationKlaviyo: Destination = { session, ) as KlaviyoProfilesApiMock; } catch (err) { - logger.throw(`Failed to initialize Klaviyo SDK: ${err}`); + throw new StepError('Failed to initialize Klaviyo SDK', { + code: 'config', + cause: err, + }); } } diff --git a/packages/server/destinations/klaviyo/src/push.ts b/packages/server/destinations/klaviyo/src/push.ts index b1bf40d9a..5bd92c2cf 100644 --- a/packages/server/destinations/klaviyo/src/push.ts +++ b/packages/server/destinations/klaviyo/src/push.ts @@ -11,11 +11,42 @@ import { isString, isArray, isNumber, + StepError, } from '@walkeros/core'; +/** + * One call to the SDK, awaited. A failure becomes a StepError with a fixed + * message: the SDK's own error stays on `cause`, never in the message. + */ +async function sdk( + message: string, + call: () => Promise, +): Promise { + try { + await call(); + } catch (error) { + throw new StepError(message, { + code: 'transport', + status: statusOf(error), + cause: error, + }); + } +} + +/** The HTTP status of a failed request (an Axios error), when it has one. */ +function statusOf(error: unknown): number | undefined { + if (typeof error !== 'object' || error === null || !('response' in error)) + return undefined; + const { response } = error; + if (typeof response !== 'object' || response === null) return undefined; + return 'status' in response && typeof response.status === 'number' + ? response.status + : undefined; +} + export const push: PushFn = async function ( event, - { config, rule, data, collector, env, logger }, + { config, rule, data, collector, env }, ) { const settings = config.settings as Settings; @@ -29,10 +60,10 @@ export const push: PushFn = async function ( const eventsApi = envTyped?.eventsApi || settings._eventsApi; const profilesApi = envTyped?.profilesApi || settings._profilesApi; - if (!eventsApi || !profilesApi) { - logger.warn('Klaviyo API instances not initialized'); - return; - } + if (!eventsApi || !profilesApi) + throw new StepError('Klaviyo API instances not initialized', { + code: 'not-ready', + }); const state: RuntimeState = settings._state || {}; const mappingSettings = rule?.settings || {}; @@ -53,13 +84,10 @@ export const push: PushFn = async function ( ) : undefined; - if (!email && !externalId && !phoneNumber) { - logger.warn( - 'Klaviyo requires at least one profile identifier; skipping event', - { event: event.name }, - ); - return; - } + if (!email && !externalId && !phoneNumber) + throw new StepError('Klaviyo requires at least one profile identifier', { + code: 'missing-input', + }); // Build profile identifier object const profileIdentifiers: Record = {}; @@ -154,7 +182,9 @@ export const push: PushFn = async function ( }, }; - await eventsApi.createEvent(eventBody); + await sdk('Klaviyo createEvent failed', () => + eventsApi.createEvent(eventBody), + ); } settings._state = state; @@ -244,7 +274,9 @@ async function applyIdentify( }, }; - await profilesApi.createOrUpdateProfile(body); + await sdk('Klaviyo createOrUpdateProfile failed', () => + profilesApi.createOrUpdateProfile(body), + ); state.lastIdentity = { email: profileIdentifiers.email, diff --git a/packages/server/destinations/klaviyo/src/schemas/settings.ts b/packages/server/destinations/klaviyo/src/schemas/settings.ts index 95aa32bc3..15656e03c 100644 --- a/packages/server/destinations/klaviyo/src/schemas/settings.ts +++ b/packages/server/destinations/klaviyo/src/schemas/settings.ts @@ -1,49 +1,51 @@ import { z } from '@walkeros/core/dev'; -export const SettingsSchema = z.object({ - apiKey: z - .string() - .min(1) - .describe( - 'Klaviyo private API key. Starts with pk_. Find it under Settings > API Keys in your Klaviyo account.', - ), - email: z - .string() - .describe( - 'walkerOS mapping value path to resolve email from each event (like user.email).', - ) - .optional(), - phoneNumber: z - .string() - .describe( - 'walkerOS mapping value path to resolve phone number in E.164 format from each event.', - ) - .optional(), - externalId: z - .string() - .describe( - 'walkerOS mapping value path to resolve external ID from each event (like user.id).', - ) - .optional(), - identify: z - .unknown() - .describe( - 'Destination-level identity mapping. Resolves to profile attributes { firstName?, lastName?, organization?, properties? }. Fires createOrUpdateProfile() on first push and re-fires when values change.', - ) - .optional(), - currency: z - .string() - .length(3) - .describe( - 'Default ISO 4217 currency code for revenue events (like USD, EUR). Sets valueCurrency on Klaviyo events.', - ) - .optional(), - uniqueId: z - .unknown() - .describe( - 'Destination-level dedup key mapping. Resolves to the event uniqueId (unique_id on the wire) and defaults to the walkerOS event id. Rule-level mapping.uniqueId overrides it per event.', - ) - .optional(), -}); +export const SettingsSchema = z + .object({ + apiKey: z + .string() + .min(1) + .describe( + 'Klaviyo private API key. Starts with pk_. Find it under Settings > API Keys in your Klaviyo account.', + ), + email: z + .string() + .describe( + 'walkerOS mapping value path to resolve email from each event (like user.email).', + ) + .optional(), + phoneNumber: z + .string() + .describe( + 'walkerOS mapping value path to resolve phone number in E.164 format from each event.', + ) + .optional(), + externalId: z + .string() + .describe( + 'walkerOS mapping value path to resolve external ID from each event (like user.id).', + ) + .optional(), + identify: z + .unknown() + .describe( + 'Destination-level identity mapping. Resolves to profile attributes { firstName?, lastName?, organization?, properties? }. Fires createOrUpdateProfile() on first push and re-fires when values change.', + ) + .optional(), + currency: z + .string() + .length(3) + .describe( + 'Default ISO 4217 currency code for revenue events (like USD, EUR). Sets valueCurrency on Klaviyo events.', + ) + .optional(), + uniqueId: z + .unknown() + .describe( + 'Destination-level dedup key mapping. Resolves to the event uniqueId (unique_id on the wire) and defaults to the walkerOS event id. Rule-level mapping.uniqueId overrides it per event.', + ) + .optional(), + }) + .strict(); export type Settings = z.infer; diff --git a/packages/server/destinations/linkedin/src/__tests__/conformance.test.ts b/packages/server/destinations/linkedin/src/__tests__/conformance.test.ts new file mode 100644 index 000000000..4adfc9b50 --- /dev/null +++ b/packages/server/destinations/linkedin/src/__tests__/conformance.test.ts @@ -0,0 +1,20 @@ +import path from 'path'; +import { conformance } from '@walkeros/collector/dev'; +import { destinationLinkedIn } from '..'; +import { examples, schemas } from '../dev'; + +describe('linkedin conformance', () => { + beforeEach(() => { + jest.useRealTimers(); + }); + + it('conforms', async () => { + await conformance('destination', destinationLinkedIn, { + profile: 'http', + examples: examples.step, + schema: schemas.settings, + settings: { accessToken: 's3cr3t', conversionRuleId: '12345678' }, + root: path.join(__dirname, '..', '..'), + }); + }); +}); diff --git a/packages/server/destinations/linkedin/src/__tests__/index.test.ts b/packages/server/destinations/linkedin/src/__tests__/index.test.ts index 17fe146ee..81f4916ea 100644 --- a/packages/server/destinations/linkedin/src/__tests__/index.test.ts +++ b/packages/server/destinations/linkedin/src/__tests__/index.test.ts @@ -56,7 +56,7 @@ describe('Server Destination LinkedIn', () => { logger: createMockLogger(), id: 'test-linkedin', }), - ).rejects.toThrow('Config settings accessToken missing'); + ).rejects.toThrow('Config settings accessToken, conversionRuleId missing'); }); test('init - missing conversionRuleId throws', async () => { @@ -261,7 +261,7 @@ describe('Server Destination LinkedIn', () => { expect(userIds[0].idValue).toBe('already-hashed-value'); }); - test('no userIds skips - event without email or li_fat_id does not call sendServer', async () => { + test('no userIds fails - event without email or li_fat_id is not sent', async () => { const event = createEvent({ name: 'form submit', user: { id: 'user-123' }, @@ -270,14 +270,16 @@ describe('Server Destination LinkedIn', () => { settings: { accessToken, conversionRuleId, apiVersion: '202604' }, }; - await destination.push( - event, - createMockContext({ - config, - env: testEnv, - id: 'test-linkedin', - }), - ); + await expect( + destination.push( + event, + createMockContext({ + config, + env: testEnv, + id: 'test-linkedin', + }), + ), + ).rejects.toMatchObject({ code: 'missing-input' }); expect(mockSendServer).not.toHaveBeenCalled(); }); diff --git a/packages/server/destinations/linkedin/src/__tests__/stepExamples.test.ts b/packages/server/destinations/linkedin/src/__tests__/stepExamples.test.ts index 2b311eeee..a42279ac0 100644 --- a/packages/server/destinations/linkedin/src/__tests__/stepExamples.test.ts +++ b/packages/server/destinations/linkedin/src/__tests__/stepExamples.test.ts @@ -20,7 +20,12 @@ describe('Step Examples', () => { }); }); - it.each(Object.entries(examples.step))('%s', async (_name, example) => { + // Non-public examples (silent, failures) run in the conformance test. + const published = Object.entries(examples.step).filter( + ([, example]) => example.public !== false, + ); + + it.each(published)('%s', async (_name, example) => { const event = example.in as WalkerOS.Event; const mapping = example.mapping; diff --git a/packages/server/destinations/linkedin/src/config.ts b/packages/server/destinations/linkedin/src/config.ts index c155ac972..995d51642 100644 --- a/packages/server/destinations/linkedin/src/config.ts +++ b/packages/server/destinations/linkedin/src/config.ts @@ -1,16 +1,19 @@ import type { Config, Settings, PartialConfig } from './types'; -import type { Logger } from '@walkeros/core'; +import { StepError } from '@walkeros/core'; -export function getConfig( - partialConfig: PartialConfig = {}, - logger: Logger.Instance, -): Config { +export function getConfig(partialConfig: PartialConfig = {}): Config { const settings = (partialConfig.settings || {}) as Partial; const { accessToken, conversionRuleId } = settings; - if (!accessToken) logger.throw('Config settings accessToken missing'); - if (!conversionRuleId) - logger.throw('Config settings conversionRuleId missing'); + if (!accessToken || !conversionRuleId) { + const missing = [ + ...(accessToken ? [] : ['accessToken']), + ...(conversionRuleId ? [] : ['conversionRuleId']), + ]; + throw new StepError(`Config settings ${missing.join(', ')} missing`, { + code: 'config', + }); + } const settingsConfig: Settings = { ...settings, diff --git a/packages/server/destinations/linkedin/src/examples/step.ts b/packages/server/destinations/linkedin/src/examples/step.ts index 27cda5c5a..7cc7d70b5 100644 --- a/packages/server/destinations/linkedin/src/examples/step.ts +++ b/packages/server/destinations/linkedin/src/examples/step.ts @@ -192,3 +192,35 @@ export const purchaseWithLiFatId: Flow.StepExample = { ], ], }; + +/** A silent rule sends nothing to LinkedIn. */ +export const silentRule: Flow.StepExample = { + public: false, + description: 'A rule with silent: true sends no request to LinkedIn.', + in: getEvent('page view', { timestamp: 1700000990000 }), + mapping: { silent: true }, + out: [], +}; + +/** + * No user identifier -- LinkedIn cannot match the event to a person, so the push + * fails with missing-input instead of counting as delivered. One per event + * name above, so every example's event without identifiers has its declared + * outcome. + */ +function withoutIdentifier(name: string, timestamp: number): Flow.StepExample { + const [entity, action] = name.split(' '); + return { + public: false, + description: `A ${name} event without a user identifier cannot be sent to LinkedIn: the push fails with missing-input instead of counting as delivered.`, + in: { name, entity, action, timestamp }, + out: [], + outcome: { status: 'failed', code: 'missing-input', retryable: false }, + }; +} + +export const noIdentifierOrder = withoutIdentifier( + 'order complete', + 1700000991000, +); +export const noIdentifierForm = withoutIdentifier('form submit', 1700000992000); diff --git a/packages/server/destinations/linkedin/src/index.ts b/packages/server/destinations/linkedin/src/index.ts index 123ff7e50..a2b963d59 100644 --- a/packages/server/destinations/linkedin/src/index.ts +++ b/packages/server/destinations/linkedin/src/index.ts @@ -10,8 +10,8 @@ export const destinationLinkedIn: Destination = { config: {}, - async init({ config: partialConfig, logger }) { - const config = getConfig(partialConfig, logger); + async init({ config: partialConfig }) { + const config = getConfig(partialConfig); return config; }, diff --git a/packages/server/destinations/linkedin/src/push.ts b/packages/server/destinations/linkedin/src/push.ts index 112f2619a..0c32e1373 100644 --- a/packages/server/destinations/linkedin/src/push.ts +++ b/packages/server/destinations/linkedin/src/push.ts @@ -8,11 +8,22 @@ import type { UserIdentifier, UserInfo, } from './types'; -import { getMappingValue, isArray, isObject, isString } from '@walkeros/core'; +import { + getMappingValue, + isArray, + isObject, + isString, + StepError, +} from '@walkeros/core'; import { sendServer } from '@walkeros/server-core'; import { getHashServer } from '@walkeros/server-core'; -export const push: PushFn = async function ( +// A silent rule has no linkedin call of its own: nothing to send. +export const push: PushFn = async function (event, context) { + if (!context.rule?.silent) await send(event, context); +}; + +const send: PushFn = async function ( event, { config, rule, data, env, logger, collector }, ) { @@ -76,8 +87,11 @@ export const push: PushFn = async function ( }); } - // Skip event if no user identifiers - if (userIds.length === 0) return; + // An event without a user identifier cannot be matched by LinkedIn + if (userIds.length === 0) + throw new StepError('No user identifier to send the event with', { + code: 'missing-input', + }); // Build optional userInfo let userInfo: UserInfo | undefined; diff --git a/packages/server/destinations/linkedin/src/schemas/settings.ts b/packages/server/destinations/linkedin/src/schemas/settings.ts index a3a061af2..aef61234d 100644 --- a/packages/server/destinations/linkedin/src/schemas/settings.ts +++ b/packages/server/destinations/linkedin/src/schemas/settings.ts @@ -1,41 +1,43 @@ import { z } from '@walkeros/core/dev'; import { ApiVersionSchema } from './primitives'; -export const SettingsSchema = z.object({ - accessToken: z - .string() - .min(1) - .describe( - 'LinkedIn OAuth 2.0 Bearer token for Conversions API authentication (like AQX...)', - ), - conversionRuleId: z - .string() - .regex(/^[0-9]+$/, 'Conversion rule ID must contain only digits') - .describe( - 'Default LinkedIn conversion rule ID from Campaign Manager (like 12345678)', - ), - apiVersion: ApiVersionSchema.describe( - 'Linkedin-Version header value in YYYYMM format (like 202604)', - ).optional(), - doNotHash: z - .array(z.string()) - .describe( - "Array of user data fields that should not be hashed (like ['email'])", - ) - .optional(), - url: z - .string() - .url() - .describe( - 'Custom URL for LinkedIn Conversions API endpoint (like https://api.linkedin.com/rest/)', - ) - .optional(), - user_data: z - .record(z.string(), z.string()) - .describe( - "Mapping configuration for user data fields (like { email: 'user.email', li_fat_id: 'context.li_fat_id' })", - ) - .optional(), -}); +export const SettingsSchema = z + .object({ + accessToken: z + .string() + .min(1) + .describe( + 'LinkedIn OAuth 2.0 Bearer token for Conversions API authentication (like AQX...)', + ), + conversionRuleId: z + .string() + .regex(/^[0-9]+$/, 'Conversion rule ID must contain only digits') + .describe( + 'Default LinkedIn conversion rule ID from Campaign Manager (like 12345678)', + ), + apiVersion: ApiVersionSchema.describe( + 'Linkedin-Version header value in YYYYMM format (like 202604)', + ).optional(), + doNotHash: z + .array(z.string()) + .describe( + "Array of user data fields that should not be hashed (like ['email'])", + ) + .optional(), + url: z + .string() + .url() + .describe( + 'Custom URL for LinkedIn Conversions API endpoint (like https://api.linkedin.com/rest/)', + ) + .optional(), + user_data: z + .record(z.string(), z.string()) + .describe( + "Mapping configuration for user data fields (like { email: 'user.email', li_fat_id: 'context.li_fat_id' })", + ) + .optional(), + }) + .strict(); export type Settings = z.infer; diff --git a/packages/server/destinations/meta/src/__tests__/conformance.test.ts b/packages/server/destinations/meta/src/__tests__/conformance.test.ts new file mode 100644 index 000000000..4b2de3f20 --- /dev/null +++ b/packages/server/destinations/meta/src/__tests__/conformance.test.ts @@ -0,0 +1,20 @@ +import path from 'path'; +import { conformance } from '@walkeros/collector/dev'; +import { destinationMeta } from '..'; +import { examples, schemas } from '../dev'; + +describe('meta conformance', () => { + beforeEach(() => { + jest.useRealTimers(); + }); + + it('conforms', async () => { + await conformance('destination', destinationMeta, { + profile: 'http', + examples: examples.step, + schema: schemas.settings, + settings: { accessToken: 's3cr3t', pixelId: '1234567890' }, + root: path.join(__dirname, '..', '..'), + }); + }); +}); diff --git a/packages/server/destinations/meta/src/__tests__/index.test.ts b/packages/server/destinations/meta/src/__tests__/index.test.ts index c3cf1aa2e..ea4a0518c 100644 --- a/packages/server/destinations/meta/src/__tests__/index.test.ts +++ b/packages/server/destinations/meta/src/__tests__/index.test.ts @@ -65,7 +65,7 @@ describe('Server Destination Meta', () => { logger: createMockLogger(), id: 'test-meta', }), - ).rejects.toThrow('Config settings accessToken missing'); + ).rejects.toThrow('Config settings accessToken, pixelId missing'); await expect( destination.init({ config: { settings: { accessToken, pixelId: '' } }, @@ -159,6 +159,7 @@ describe('Server Destination Meta', () => { settings: { accessToken, pixelId, user_data: { em: 'user.email' } }, }; const { collector } = await startFlow(); + const report = jest.spyOn(collector.causeLog!, 'report'); const rule: Rule = { data: { map: { @@ -182,14 +183,14 @@ describe('Server Destination Meta', () => { }), ); - expect(logger.warn).toHaveBeenCalledTimes(1); - expect(logger.warn).toHaveBeenCalledWith( + expect(report).toHaveBeenCalledTimes(1); + expect(report).toHaveBeenCalledWith( + { step: 'destination.test-meta', code: 'mapping' }, expect.stringContaining('dropped'), - { - keys: ['email'], - }, + { event: 'entity action', keys: ['email'] }, + 'warn', ); - expect(JSON.stringify(logger.warn.mock.calls)).not.toContain( + expect(JSON.stringify(report.mock.calls)).not.toContain( 'private@example.com', ); diff --git a/packages/server/destinations/meta/src/__tests__/stepExamples.test.ts b/packages/server/destinations/meta/src/__tests__/stepExamples.test.ts index 0a40af8e4..b5cccfbf1 100644 --- a/packages/server/destinations/meta/src/__tests__/stepExamples.test.ts +++ b/packages/server/destinations/meta/src/__tests__/stepExamples.test.ts @@ -70,11 +70,16 @@ describe('Step Examples', () => { ); } - it.each(Object.entries(examples.step))('%s', async (name, example) => { + // Non-public examples (silent, failures) run in the conformance test. + const published = Object.entries(examples.step).filter( + ([, example]) => example.public !== false, + ); + + it.each(published)('%s', async (name, example) => { expect(await run(example, example.mapping)).toEqual(example.out); }); - it.each(Object.entries(examples.step))( + it.each(published)( '%s works with serialized functions', async (name, example) => { expect(await run(example, serializeMapping(example.mapping))).toEqual( diff --git a/packages/server/destinations/meta/src/config.ts b/packages/server/destinations/meta/src/config.ts index 840d97a3b..7ab7143f4 100644 --- a/packages/server/destinations/meta/src/config.ts +++ b/packages/server/destinations/meta/src/config.ts @@ -1,15 +1,19 @@ import type { Config, Settings, PartialConfig } from './types'; -import type { Logger } from '@walkeros/core'; +import { StepError } from '@walkeros/core'; -export function getConfig( - partialConfig: PartialConfig = {}, - logger: Logger.Instance, -): Config { +export function getConfig(partialConfig: PartialConfig = {}): Config { const settings = (partialConfig.settings || {}) as Partial; const { accessToken, pixelId } = settings; - if (!accessToken) logger.throw('Config settings accessToken missing'); - if (!pixelId) logger.throw('Config settings pixelId missing'); + if (!accessToken || !pixelId) { + const missing = [ + ...(accessToken ? [] : ['accessToken']), + ...(pixelId ? [] : ['pixelId']), + ]; + throw new StepError(`Config settings ${missing.join(', ')} missing`, { + code: 'config', + }); + } const settingsConfig: Settings = { ...settings, diff --git a/packages/server/destinations/meta/src/examples/step.ts b/packages/server/destinations/meta/src/examples/step.ts index f0a7d3951..e5e4453d8 100644 --- a/packages/server/destinations/meta/src/examples/step.ts +++ b/packages/server/destinations/meta/src/examples/step.ts @@ -211,3 +211,12 @@ export const purchaseWithClickAttribution: Flow.StepExample = { ], ], }; + +/** A silent rule sends nothing to Meta. */ +export const silentRule: Flow.StepExample = { + public: false, + description: 'A rule with silent: true sends no request to Meta.', + in: getEvent('page view', { timestamp: 1700000990000 }), + mapping: { silent: true }, + out: [], +}; diff --git a/packages/server/destinations/meta/src/index.ts b/packages/server/destinations/meta/src/index.ts index 010cc6a09..055c12b93 100644 --- a/packages/server/destinations/meta/src/index.ts +++ b/packages/server/destinations/meta/src/index.ts @@ -10,8 +10,8 @@ export const destinationMeta: Destination = { config: {}, - async init({ config: partialConfig, logger }) { - const config = getConfig(partialConfig, logger); + async init({ config: partialConfig }) { + const config = getConfig(partialConfig); return config; }, diff --git a/packages/server/destinations/meta/src/push.ts b/packages/server/destinations/meta/src/push.ts index 31905d0e1..ca47e8e3f 100644 --- a/packages/server/destinations/meta/src/push.ts +++ b/packages/server/destinations/meta/src/push.ts @@ -6,14 +6,25 @@ import type { Env, } from './types'; import type { Collector, Mapping as WalkerOSMapping } from '@walkeros/core'; -import { createMappingRoot, getMappingValue, isObject } from '@walkeros/core'; +import { + createMappingRoot, + getMappingValue, + isObject, + reportCause, + stepId, +} from '@walkeros/core'; import { sendServer } from '@walkeros/server-core'; import { hashEvent } from './hash'; import { getUnknownUserDataKeys } from './userData'; -export const push: PushFn = async function ( +// A silent rule has no meta call of its own: nothing to send. +export const push: PushFn = async function (event, context) { + if (!context.rule?.silent) await send(event, context); +}; + +const send: PushFn = async function ( event, - { config, rule, data, ingest, collector, env, logger }, + { config, rule, data, ingest, collector, env, logger, id }, ) { const { accessToken, @@ -63,9 +74,13 @@ export const push: PushFn = async function ( // Unknown keys are never hashed, so drop them instead of sending cleartext const unknownKeys = getUnknownUserDataKeys(mappedUserData); if (unknownKeys.length) - logger.warn( + reportCause( + collector, + logger, + { step: stepId('destination', id), code: 'mapping' }, 'Unknown Meta user_data keys dropped, use parameter names such as em and ph', - { keys: unknownKeys }, + { event: event.name, keys: unknownKeys }, + 'warn', ); const userData: CustomerInformationParameters = Object.fromEntries( diff --git a/packages/server/destinations/meta/src/schemas/settings.ts b/packages/server/destinations/meta/src/schemas/settings.ts index 7320d4623..cbd8f31f1 100644 --- a/packages/server/destinations/meta/src/schemas/settings.ts +++ b/packages/server/destinations/meta/src/schemas/settings.ts @@ -7,53 +7,55 @@ const InputSchema = z.union([ z.literal(false), ]); -export const SettingsSchema = z.object({ - accessToken: z - .string() - .min(1) - .describe( - 'Meta access token for Conversions API authentication (like your_access_token)', - ), - pixelId: z - .string() - .regex(/^[0-9]+$/, 'Pixel ID must contain only digits') - .describe( - 'Meta Pixel ID from your Facebook Business account (like 1234567890)', - ), - action_source: ActionSourceSchema.describe( - 'Source of the event (website, app, phone_call, etc.) (like website)', - ).optional(), - doNotHash: z - .array(z.string()) - .describe( - "user_data keys to send without normalizing and hashing, for values that are already hashed. Only em, ph, fn, ln, db, ge, ct, st, zp, country and external_id are hashed (like ['external_id'])", - ) - .optional(), - test_event_code: z - .string() - .describe( - 'Test event code for debugging Meta Conversions API events (like TEST12345)', - ) - .optional(), - url: z - .string() - .url() - .describe( - 'Custom URL for Meta Conversions API endpoint (like https://graph.facebook.com/v22.0)', - ) - .optional(), - user_data: z - .partialRecord(z.enum(userDataKeys), schemas.MappingSchemas.ValueSchema) - .describe( - "Mapping of Meta customer information parameters to event values, applied to every event. Keys must be Meta's short names such as em and ph (like { em: 'user.email', ph: 'user.phone' })", - ) - .optional(), - ip: InputSchema.describe( - 'Client IP, sent as user_data.client_ip_address. Resolves against { ingest, event }. Default: ["ingest.ip", "event.user.ip"]. false switches it off. A mapped user_data.client_ip_address wins.', - ).optional(), - userAgent: InputSchema.describe( - 'Client user agent, sent as user_data.client_user_agent. Resolves against { ingest, event }. Default: ["ingest.userAgent", "event.user.userAgent"]. false switches it off. A mapped user_data.client_user_agent wins.', - ).optional(), -}); +export const SettingsSchema = z + .object({ + accessToken: z + .string() + .min(1) + .describe( + 'Meta access token for Conversions API authentication (like your_access_token)', + ), + pixelId: z + .string() + .regex(/^[0-9]+$/, 'Pixel ID must contain only digits') + .describe( + 'Meta Pixel ID from your Facebook Business account (like 1234567890)', + ), + action_source: ActionSourceSchema.describe( + 'Source of the event (website, app, phone_call, etc.) (like website)', + ).optional(), + doNotHash: z + .array(z.string()) + .describe( + "user_data keys to send without normalizing and hashing, for values that are already hashed. Only em, ph, fn, ln, db, ge, ct, st, zp, country and external_id are hashed (like ['external_id'])", + ) + .optional(), + test_event_code: z + .string() + .describe( + 'Test event code for debugging Meta Conversions API events (like TEST12345)', + ) + .optional(), + url: z + .string() + .url() + .describe( + 'Custom URL for Meta Conversions API endpoint (like https://graph.facebook.com/v22.0)', + ) + .optional(), + user_data: z + .partialRecord(z.enum(userDataKeys), schemas.MappingSchemas.ValueSchema) + .describe( + "Mapping of Meta customer information parameters to event values, applied to every event. Keys must be Meta's short names such as em and ph (like { em: 'user.email', ph: 'user.phone' })", + ) + .optional(), + ip: InputSchema.describe( + 'Client IP, sent as user_data.client_ip_address. Resolves against { ingest, event }. Default: ["ingest.ip", "event.user.ip"]. false switches it off. A mapped user_data.client_ip_address wins.', + ).optional(), + userAgent: InputSchema.describe( + 'Client user agent, sent as user_data.client_user_agent. Resolves against { ingest, event }. Default: ["ingest.userAgent", "event.user.userAgent"]. false switches it off. A mapped user_data.client_user_agent wins.', + ).optional(), + }) + .strict(); export type Settings = z.infer; diff --git a/packages/server/destinations/mixpanel/src/__tests__/conformance.test.ts b/packages/server/destinations/mixpanel/src/__tests__/conformance.test.ts new file mode 100644 index 000000000..718decb69 --- /dev/null +++ b/packages/server/destinations/mixpanel/src/__tests__/conformance.test.ts @@ -0,0 +1,21 @@ +import path from 'path'; +import { conformance } from '@walkeros/collector/dev'; +import { destinationMixpanel } from '..'; +import { examples, schemas } from '../dev'; + +describe('mixpanel conformance', () => { + beforeEach(() => { + jest.useRealTimers(); + }); + + it('conforms', async () => { + await conformance('destination', destinationMixpanel, { + profile: 'sdk-env', + examples: examples.step, + schema: schemas.settings, + settings: { apiKey: 'token' }, + env: { push: examples.env.push, simulation: examples.env.simulation }, + root: path.join(__dirname, '..', '..'), + }); + }); +}); diff --git a/packages/server/destinations/mixpanel/src/__tests__/stepExamples.test.ts b/packages/server/destinations/mixpanel/src/__tests__/stepExamples.test.ts index 47dec6d60..aed95b040 100644 --- a/packages/server/destinations/mixpanel/src/__tests__/stepExamples.test.ts +++ b/packages/server/destinations/mixpanel/src/__tests__/stepExamples.test.ts @@ -178,7 +178,12 @@ function spyEnv(env: Env): { env: Env; collected: () => CallRecord[] } { } describe('mixpanel server destination — step examples', () => { - it.each(Object.entries(examples.step))('%s', async (_name, rawExample) => { + // Non-public examples (silent, failures) run in the conformance test. + const published = Object.entries(examples.step).filter( + ([, example]) => example.public !== false, + ); + + it.each(published)('%s', async (_name, rawExample) => { const example = rawExample as { in?: unknown; mapping?: unknown; diff --git a/packages/server/destinations/mixpanel/src/config.ts b/packages/server/destinations/mixpanel/src/config.ts index dea41e805..40811e8c2 100644 --- a/packages/server/destinations/mixpanel/src/config.ts +++ b/packages/server/destinations/mixpanel/src/config.ts @@ -6,6 +6,7 @@ import type { Settings, } from './types'; import type { Logger } from '@walkeros/core'; +import { StepError } from '@walkeros/core'; import Mixpanel from 'mixpanel'; export function getConfig( @@ -16,7 +17,8 @@ export function getConfig( const settings = (partialConfig.settings || {}) as Partial; const { apiKey } = settings; - if (!apiKey) logger.throw('Config settings apiKey missing'); + if (!apiKey) + throw new StepError('Config settings apiKey missing', { code: 'config' }); // Build SDK init options from settings const initOptions: Record = {}; diff --git a/packages/server/destinations/mixpanel/src/examples/step.ts b/packages/server/destinations/mixpanel/src/examples/step.ts index 61dfddbe8..29c6b093c 100644 --- a/packages/server/destinations/mixpanel/src/examples/step.ts +++ b/packages/server/destinations/mixpanel/src/examples/step.ts @@ -416,3 +416,49 @@ export const wildcardIgnored: MixpanelStepExample = { mapping: { ignore: true }, out: [], }; + +/** A silent rule sends nothing to Mixpanel. */ +export const silentRule: Flow.StepExample = { + public: false, + description: 'A rule with silent: true sends nothing to Mixpanel.', + in: getEvent('page view', { timestamp: 1700000990000 }), + mapping: { silent: true }, + out: [], +}; + +/** + * The event of a people or group profile example without the data its rule + * reads: no distinct id and no group key resolve, so there is nothing to + * address. A rule that only addresses profiles has nothing else to send, so + * it returns normally; that is a declared skip, not a failure. + */ +function skipWithoutInput( + target: MixpanelStepExample, + name: string, + timestamp: number, +): MixpanelStepExample { + const [entity, action] = name.split(' '); + return { + public: false, + description: `A ${name} event without the data its rule reads has no profile or group to address: nothing is sent, and that is not a failure.`, + in: { name, entity, action, timestamp }, + mapping: target.mapping, + out: [], + }; +} + +export const userLoginPeopleSetSkip = skipWithoutInput( + userLoginPeopleSet, + 'user login', + 1700000991000, +); +export const allPeopleOperationsSkip = skipWithoutInput( + allPeopleOperations, + 'profile update', + 1700000992000, +); +export const companyGroupProfileSkip = skipWithoutInput( + companyGroupProfile, + 'company update', + 1700000993000, +); diff --git a/packages/server/destinations/mixpanel/src/push.ts b/packages/server/destinations/mixpanel/src/push.ts index 43cbcda76..af524e933 100644 --- a/packages/server/destinations/mixpanel/src/push.ts +++ b/packages/server/destinations/mixpanel/src/push.ts @@ -1,5 +1,14 @@ import type { MixpanelCallback, MixpanelClient, PushFn } from './types'; -import { getMappingValue, isArray, isObject, isString } from '@walkeros/core'; +import { + getMappingValue, + isArray, + isObject, + isString, + isThenable, + reportCause, + StepError, + stepId, +} from '@walkeros/core'; type PeopleObjectOp = | 'set' @@ -30,12 +39,23 @@ const GROUP_OBJECT_OPS: GroupObjectOp[] = [ /** * Wraps a Mixpanel SDK callback-based call in a Promise. */ -function wrapCallback(fn: (cb: MixpanelCallback) => void): Promise { +function wrapCallback(fn: (cb: MixpanelCallback) => unknown): Promise { return new Promise((resolve, reject) => { - fn((err) => { - if (err) reject(err); - else resolve(); - }); + // The SDK's own error stays on `cause`, never in the message. + const fail = (error: unknown) => + reject( + new StepError('Mixpanel request failed', { + code: 'transport', + cause: error, + }), + ); + try { + const answer = fn((err) => (err ? fail(err) : resolve())); + // An injected client may answer with a promise instead of the callback. + if (isThenable(answer)) answer.then(() => resolve(), fail); + } catch (error) { + fail(error); + } }); } @@ -54,15 +74,15 @@ async function applyPeople( const bag = resolved[op]; if (!isObject(bag) || Object.keys(bag).length === 0) continue; promises.push( - wrapCallback((cb) => { + wrapCallback((cb) => ( client.people[op] as ( id: string, props: Record, cb: MixpanelCallback, ) => void - )(distinctId, bag as Record, cb); - }), + )(distinctId, bag as Record, cb), + ), ); } @@ -71,9 +91,7 @@ async function applyPeople( if (unsetVal !== undefined) { if (isString(unsetVal)) { promises.push( - wrapCallback((cb) => { - client.people.unset(distinctId, unsetVal, cb); - }), + wrapCallback((cb) => client.people.unset(distinctId, unsetVal, cb)), ); } else if (isArray(unsetVal)) { const names = (unsetVal as unknown[]).filter((v): v is string => @@ -81,9 +99,7 @@ async function applyPeople( ); if (names.length > 0) { promises.push( - wrapCallback((cb) => { - client.people.unset(distinctId, names, cb); - }), + wrapCallback((cb) => client.people.unset(distinctId, names, cb)), ); } } @@ -92,9 +108,7 @@ async function applyPeople( // Handle delete_user if (resolved.delete_user === true) { promises.push( - wrapCallback((cb) => { - client.people.delete_user(distinctId, cb); - }), + wrapCallback((cb) => client.people.delete_user(distinctId, cb)), ); } @@ -107,9 +121,13 @@ async function applyPeople( async function applyGroupProfile( client: MixpanelClient, resolved: Record, + dropped: () => void, ): Promise { const { key, id } = resolved as { key?: unknown; id?: unknown }; - if (!isString(key) || key === '' || !isString(id) || id === '') return; + if (!isString(key) || key === '' || !isString(id) || id === '') { + dropped(); + return; + } const promises: Promise[] = []; @@ -117,7 +135,7 @@ async function applyGroupProfile( const bag = resolved[op]; if (!isObject(bag) || Object.keys(bag).length === 0) continue; promises.push( - wrapCallback((cb) => { + wrapCallback((cb) => ( client.groups[op] as ( gk: string, @@ -125,8 +143,8 @@ async function applyGroupProfile( props: Record, cb: MixpanelCallback, ) => void - )(key, id, bag as Record, cb); - }), + )(key, id, bag as Record, cb), + ), ); } @@ -135,9 +153,7 @@ async function applyGroupProfile( if (unsetVal !== undefined) { if (isString(unsetVal)) { promises.push( - wrapCallback((cb) => { - client.groups.unset(key, id, unsetVal, cb); - }), + wrapCallback((cb) => client.groups.unset(key, id, unsetVal, cb)), ); } else if (isArray(unsetVal)) { const names = (unsetVal as unknown[]).filter((v): v is string => @@ -145,9 +161,7 @@ async function applyGroupProfile( ); if (names.length > 0) { promises.push( - wrapCallback((cb) => { - client.groups.unset(key, id, names, cb); - }), + wrapCallback((cb) => client.groups.unset(key, id, names, cb)), ); } } @@ -156,9 +170,7 @@ async function applyGroupProfile( // Handle delete if (resolved.delete === true) { promises.push( - wrapCallback((cb) => { - client.groups.delete_group(key, id, cb); - }), + wrapCallback((cb) => client.groups.delete_group(key, id, cb)), ); } @@ -167,14 +179,14 @@ async function applyGroupProfile( export const push: PushFn = async function ( event, - { config, rule, data, collector, logger }, + { config, rule, data, collector, logger, id }, ) { const settings = config.settings!; const client = settings.client; - if (!client) { - logger.throw('Mixpanel client not initialized'); - return; - } + if (!client) + throw new StepError('Mixpanel client not initialized', { + code: 'not-ready', + }); const mappingSettings = rule?.settings || {}; @@ -200,9 +212,7 @@ export const push: PushFn = async function ( // 2. Alias - fire before people/track (legacy identity merge) if (distinctId && alias) { - await wrapCallback((cb) => { - client.alias(distinctId!, alias!, cb); - }); + await wrapCallback((cb) => client.alias(distinctId!, alias!, cb)); } // 3. People operations - require distinct_id @@ -245,7 +255,17 @@ export const push: PushFn = async function ( }, ); if (isObject(resolved)) { - await applyGroupProfile(client, resolved as Record); + // A group profile without key and id has nothing to address: the event + // still goes out, and the cause log reports the dropped operation once. + await applyGroupProfile(client, resolved as Record, () => + reportCause( + collector, + logger, + { step: stepId('destination', id), code: 'mapping' }, + 'Mixpanel group profile skipped: key and id must resolve to strings', + { event: event.name, mappingKey: 'groupProfile' }, + ), + ); } } @@ -272,13 +292,11 @@ export const push: PushFn = async function ( if (useImport) { const timestamp = event.timestamp || Date.now(); - await wrapCallback((cb) => { - client.import(eventName, timestamp, properties, cb); - }); + await wrapCallback((cb) => + client.import(eventName, timestamp, properties, cb), + ); } else { - await wrapCallback((cb) => { - client.track(eventName, properties, cb); - }); + await wrapCallback((cb) => client.track(eventName, properties, cb)); } } }; diff --git a/packages/server/destinations/mixpanel/src/schemas/settings.ts b/packages/server/destinations/mixpanel/src/schemas/settings.ts index 334806a02..015d1aaa8 100644 --- a/packages/server/destinations/mixpanel/src/schemas/settings.ts +++ b/packages/server/destinations/mixpanel/src/schemas/settings.ts @@ -1,65 +1,70 @@ import { z } from '@walkeros/core/dev'; -export const SettingsSchema = z.object({ - apiKey: z - .string() - .min(1) - .describe( - 'Your Mixpanel project token. Find it in Project Settings > Access Keys. Passed as the first argument to Mixpanel.init().', - ), - secret: z - .string() - .describe( - 'API secret for the /import endpoint (historical data). Required when useImport is true.', - ) - .optional(), - host: z - .string() - .describe( - "Mixpanel API host. Default: 'api.mixpanel.com' (US). Use 'api-eu.mixpanel.com' (EU) or 'api-in.mixpanel.com' (India).", - ) - .optional(), - protocol: z - .string() - .describe("Protocol for API requests. Default: 'https'.") - .optional(), - keepAlive: z - .boolean() - .describe('Reuse HTTP connections. Default: true.') - .optional(), - geolocate: z - .boolean() - .describe( - 'Parse IP for geolocation. Default: false. Server IP caveat: all users map to server location unless $ip is overridden.', - ) - .optional(), - debug: z - .boolean() - .describe('Enable SDK debug logging. Default: false.') - .optional(), - verbose: z - .boolean() - .describe('Enable verbose request logging. Default: false.') - .optional(), - test: z.boolean().describe('Enable dry-run mode. Default: false.').optional(), - useImport: z - .boolean() - .describe( - 'Use /import endpoint instead of /track. Accepts events of any age (no 5-day limit). Requires secret for authentication.', - ) - .optional(), - identify: z - .unknown() - .describe( - 'walkerOS mapping value resolving to { distinctId, alias? }. distinctId is passed as distinct_id on every SDK call.', - ) - .optional(), - include: z - .unknown() - .describe( - "Event data sections to flatten into track() properties. Example: ['data', 'globals']. Sections are prefixed (data_, globals_, etc.).", - ) - .optional(), -}); +export const SettingsSchema = z + .object({ + apiKey: z + .string() + .min(1) + .describe( + 'Your Mixpanel project token. Find it in Project Settings > Access Keys. Passed as the first argument to Mixpanel.init().', + ), + secret: z + .string() + .describe( + 'API secret for the /import endpoint (historical data). Required when useImport is true.', + ) + .optional(), + host: z + .string() + .describe( + "Mixpanel API host. Default: 'api.mixpanel.com' (US). Use 'api-eu.mixpanel.com' (EU) or 'api-in.mixpanel.com' (India).", + ) + .optional(), + protocol: z + .string() + .describe("Protocol for API requests. Default: 'https'.") + .optional(), + keepAlive: z + .boolean() + .describe('Reuse HTTP connections. Default: true.') + .optional(), + geolocate: z + .boolean() + .describe( + 'Parse IP for geolocation. Default: false. Server IP caveat: all users map to server location unless $ip is overridden.', + ) + .optional(), + debug: z + .boolean() + .describe('Enable SDK debug logging. Default: false.') + .optional(), + verbose: z + .boolean() + .describe('Enable verbose request logging. Default: false.') + .optional(), + test: z + .boolean() + .describe('Enable dry-run mode. Default: false.') + .optional(), + useImport: z + .boolean() + .describe( + 'Use /import endpoint instead of /track. Accepts events of any age (no 5-day limit). Requires secret for authentication.', + ) + .optional(), + identify: z + .unknown() + .describe( + 'walkerOS mapping value resolving to { distinctId, alias? }. distinctId is passed as distinct_id on every SDK call.', + ) + .optional(), + include: z + .unknown() + .describe( + "Event data sections to flatten into track() properties. Example: ['data', 'globals']. Sections are prefixed (data_, globals_, etc.).", + ) + .optional(), + }) + .strict(); export type Settings = z.infer; diff --git a/packages/server/destinations/mparticle/src/__tests__/config.test.ts b/packages/server/destinations/mparticle/src/__tests__/config.test.ts index 84f66c8df..546de0d03 100644 --- a/packages/server/destinations/mparticle/src/__tests__/config.test.ts +++ b/packages/server/destinations/mparticle/src/__tests__/config.test.ts @@ -1,24 +1,22 @@ -import { createMockLogger } from '@walkeros/core'; import { getConfig } from '../config'; describe('getConfig', () => { test('throws when apiKey is missing', () => { - expect(() => getConfig({}, createMockLogger())).toThrow( - 'Config settings apiKey missing', + expect(() => getConfig({})).toThrow( + 'Config settings apiKey, apiSecret missing', ); }); test('throws when apiSecret is missing', () => { - expect(() => - getConfig({ settings: { apiKey: 'key' } }, createMockLogger()), - ).toThrow('Config settings apiSecret missing'); + expect(() => getConfig({ settings: { apiKey: 'key' } })).toThrow( + 'Config settings apiSecret missing', + ); }); test('applies defaults for pod and environment', () => { - const config = getConfig( - { settings: { apiKey: 'key', apiSecret: 'secret' } }, - createMockLogger(), - ); + const config = getConfig({ + settings: { apiKey: 'key', apiSecret: 'secret' }, + }); expect(config.settings).toEqual({ apiKey: 'key', apiSecret: 'secret', @@ -28,17 +26,14 @@ describe('getConfig', () => { }); test('preserves custom pod and environment', () => { - const config = getConfig( - { - settings: { - apiKey: 'key', - apiSecret: 'secret', - pod: 'eu1', - environment: 'development', - }, + const config = getConfig({ + settings: { + apiKey: 'key', + apiSecret: 'secret', + pod: 'eu1', + environment: 'development', }, - createMockLogger(), - ); + }); expect(config.settings.pod).toBe('eu1'); expect(config.settings.environment).toBe('development'); }); diff --git a/packages/server/destinations/mparticle/src/__tests__/conformance.test.ts b/packages/server/destinations/mparticle/src/__tests__/conformance.test.ts new file mode 100644 index 000000000..239149a85 --- /dev/null +++ b/packages/server/destinations/mparticle/src/__tests__/conformance.test.ts @@ -0,0 +1,20 @@ +import path from 'path'; +import { conformance } from '@walkeros/collector/dev'; +import { destinationMParticle } from '..'; +import { examples, schemas } from '../dev'; + +describe('mparticle conformance', () => { + beforeEach(() => { + jest.useRealTimers(); + }); + + it('conforms', async () => { + await conformance('destination', destinationMParticle, { + profile: 'http', + examples: examples.step, + schema: schemas.settings, + settings: { apiKey: 'key', apiSecret: 'secret' }, + root: path.join(__dirname, '..', '..'), + }); + }); +}); diff --git a/packages/server/destinations/mparticle/src/__tests__/push.test.ts b/packages/server/destinations/mparticle/src/__tests__/push.test.ts index 3e0533cbb..2e5a616f0 100644 --- a/packages/server/destinations/mparticle/src/__tests__/push.test.ts +++ b/packages/server/destinations/mparticle/src/__tests__/push.test.ts @@ -227,6 +227,6 @@ describe('push', () => { logger: createMockLogger(), id: 'test-mparticle', }), - ).rejects.toThrow('Config settings apiKey missing'); + ).rejects.toThrow('Config settings apiKey, apiSecret missing'); }); }); diff --git a/packages/server/destinations/mparticle/src/__tests__/stepExamples.test.ts b/packages/server/destinations/mparticle/src/__tests__/stepExamples.test.ts index aea1d0fda..aa2c27cd3 100644 --- a/packages/server/destinations/mparticle/src/__tests__/stepExamples.test.ts +++ b/packages/server/destinations/mparticle/src/__tests__/stepExamples.test.ts @@ -22,7 +22,12 @@ describe('Step Examples', () => { mockSendServer.mockResolvedValue({ data: {} }); }); - it.each(Object.entries(examples.step))('%s', async (_name, example) => { + // Non-public examples (silent, failures) run in the conformance test. + const published = Object.entries(examples.step).filter( + ([, example]) => example.public !== false, + ); + + it.each(published)('%s', async (_name, example) => { const event = example.in as WalkerOS.Event; const mapping = example.mapping; diff --git a/packages/server/destinations/mparticle/src/config.ts b/packages/server/destinations/mparticle/src/config.ts index fefc0d894..4b54b61da 100644 --- a/packages/server/destinations/mparticle/src/config.ts +++ b/packages/server/destinations/mparticle/src/config.ts @@ -1,22 +1,26 @@ import type { Config, PartialConfig, Settings } from './types'; -import type { Logger } from '@walkeros/core'; +import { StepError } from '@walkeros/core'; -export function getConfig( - partialConfig: PartialConfig = {}, - logger: Logger.Instance, -): Config { +export function getConfig(partialConfig: PartialConfig = {}): Config { const settings = (partialConfig.settings || {}) as Partial; const { apiKey, apiSecret } = settings; - if (!apiKey) logger.throw('Config settings apiKey missing'); - if (!apiSecret) logger.throw('Config settings apiSecret missing'); + if (!apiKey || !apiSecret) { + const missing = [ + ...(apiKey ? [] : ['apiKey']), + ...(apiSecret ? [] : ['apiSecret']), + ]; + throw new StepError(`Config settings ${missing.join(', ')} missing`, { + code: 'config', + }); + } const settingsConfig: Settings = { pod: 'us1', environment: 'production', ...settings, - apiKey: apiKey as string, - apiSecret: apiSecret as string, + apiKey, + apiSecret, }; return { ...partialConfig, settings: settingsConfig }; diff --git a/packages/server/destinations/mparticle/src/examples/step.ts b/packages/server/destinations/mparticle/src/examples/step.ts index 6eee5880b..cdf622484 100644 --- a/packages/server/destinations/mparticle/src/examples/step.ts +++ b/packages/server/destinations/mparticle/src/examples/step.ts @@ -232,3 +232,12 @@ export const identityAndAttributes: Flow.StepExample = { ], ], }; + +/** A silent rule sends nothing to mParticle. */ +export const silentRule: Flow.StepExample = { + public: false, + description: 'A rule with silent: true sends no request to mParticle.', + in: getEvent('page view', { timestamp: 1700000990000 }), + mapping: { silent: true }, + out: [], +}; diff --git a/packages/server/destinations/mparticle/src/index.ts b/packages/server/destinations/mparticle/src/index.ts index 5da89ae19..bfd39df9e 100644 --- a/packages/server/destinations/mparticle/src/index.ts +++ b/packages/server/destinations/mparticle/src/index.ts @@ -10,8 +10,8 @@ export const destinationMParticle: Destination = { config: {}, - async init({ config: partialConfig, logger }) { - const config = getConfig(partialConfig, logger); + async init({ config: partialConfig }) { + const config = getConfig(partialConfig); return config; }, diff --git a/packages/server/destinations/mparticle/src/push.ts b/packages/server/destinations/mparticle/src/push.ts index 70951098e..a4b6cbb6e 100644 --- a/packages/server/destinations/mparticle/src/push.ts +++ b/packages/server/destinations/mparticle/src/push.ts @@ -23,7 +23,12 @@ import { buildScreenViewEvent, } from './batch'; -export const push: PushFn = async function ( +// A silent rule has no mparticle call of its own: nothing to send. +export const push: PushFn = async function (event, context) { + if (!context.rule?.silent) await send(event, context); +}; + +const send: PushFn = async function ( event, { config, rule, data, env, logger, collector }, ) { diff --git a/packages/server/destinations/mparticle/src/schemas/settings.ts b/packages/server/destinations/mparticle/src/schemas/settings.ts index e74566594..6aaf1c611 100644 --- a/packages/server/destinations/mparticle/src/schemas/settings.ts +++ b/packages/server/destinations/mparticle/src/schemas/settings.ts @@ -1,53 +1,55 @@ import { z } from '@walkeros/core/dev'; import { EnvironmentSchema, PodSchema } from './primitives'; -export const SettingsSchema = z.object({ - apiKey: z - .string() - .min(1) - .describe( - 'mParticle input feed API key from the mParticle dashboard (Setup > Inputs > Feeds).', - ), - apiSecret: z - .string() - .min(1) - .describe( - 'mParticle input feed API secret paired with apiKey. Used for HTTP Basic auth.', - ), - pod: PodSchema.describe( - "mParticle data pod selecting the regional endpoint. Default: 'us1'.", - ).optional(), - environment: EnvironmentSchema.describe( - "Environment the batch targets. Default: 'production'.", - ).optional(), - userIdentities: z - .record(z.string(), z.unknown()) - .describe( - 'Mapping that resolves to user_identities per batch. Keys are mParticle identity types (like customer_id, email); values are walkerOS mapping values.', - ) - .optional(), - userAttributes: z - .unknown() - .describe( - 'Mapping value that resolves to the user_attributes object placed on the batch.', - ) - .optional(), - consent: z - .record(z.string(), z.unknown()) - .describe( - 'Static consent_state envelope forwarded verbatim on the batch. See mParticle consent_state docs.', - ) - .optional(), - ip: z - .unknown() - .describe('Mapping value resolving to the client IP for the batch.') - .optional(), - sourceRequestId: z - .unknown() - .describe( - 'Mapping value resolving to the source_request_id for the batch. Falls back to event.id when unset.', - ) - .optional(), -}); +export const SettingsSchema = z + .object({ + apiKey: z + .string() + .min(1) + .describe( + 'mParticle input feed API key from the mParticle dashboard (Setup > Inputs > Feeds).', + ), + apiSecret: z + .string() + .min(1) + .describe( + 'mParticle input feed API secret paired with apiKey. Used for HTTP Basic auth.', + ), + pod: PodSchema.describe( + "mParticle data pod selecting the regional endpoint. Default: 'us1'.", + ).optional(), + environment: EnvironmentSchema.describe( + "Environment the batch targets. Default: 'production'.", + ).optional(), + userIdentities: z + .record(z.string(), z.unknown()) + .describe( + 'Mapping that resolves to user_identities per batch. Keys are mParticle identity types (like customer_id, email); values are walkerOS mapping values.', + ) + .optional(), + userAttributes: z + .unknown() + .describe( + 'Mapping value that resolves to the user_attributes object placed on the batch.', + ) + .optional(), + consent: z + .record(z.string(), z.unknown()) + .describe( + 'Static consent_state envelope forwarded verbatim on the batch. See mParticle consent_state docs.', + ) + .optional(), + ip: z + .unknown() + .describe('Mapping value resolving to the client IP for the batch.') + .optional(), + sourceRequestId: z + .unknown() + .describe( + 'Mapping value resolving to the source_request_id for the batch. Falls back to event.id when unset.', + ) + .optional(), + }) + .strict(); export type Settings = z.infer; diff --git a/packages/server/destinations/piwikpro/src/__tests__/config.test.ts b/packages/server/destinations/piwikpro/src/__tests__/config.test.ts index d76808455..174727d02 100644 --- a/packages/server/destinations/piwikpro/src/__tests__/config.test.ts +++ b/packages/server/destinations/piwikpro/src/__tests__/config.test.ts @@ -1,4 +1,3 @@ -import { createMockLogger } from '@walkeros/core'; import { getConfig, normalizeUrl } from '../config'; describe('getConfig', () => { @@ -6,28 +5,23 @@ describe('getConfig', () => { ['url', { appId: 'site-1' }], ['appId', { url: 'https://acc.piwik.pro' }], ])('throws when %s is missing', (name, settings) => { - const logger = createMockLogger(); - expect(() => getConfig({ settings }, logger)).toThrow( + expect(() => getConfig({ settings })).toThrow( `Config settings ${name} missing`, ); }); it('normalizes url and keeps every other setting unchanged', () => { - const logger = createMockLogger(); - const config = getConfig( - { - settings: { - url: 'https://acc.piwik.pro', - appId: 'site-1', - timeout: 2000, - identified: { marketing: true }, - ip: false, - customDimensions: { '1': 'data.size' }, - }, - consent: { analytics: true }, + const config = getConfig({ + settings: { + url: 'https://acc.piwik.pro', + appId: 'site-1', + timeout: 2000, + identified: { marketing: true }, + ip: false, + customDimensions: { '1': 'data.size' }, }, - logger, - ); + consent: { analytics: true }, + }); expect(config).toEqual({ settings: { diff --git a/packages/server/destinations/piwikpro/src/__tests__/conformance.test.ts b/packages/server/destinations/piwikpro/src/__tests__/conformance.test.ts new file mode 100644 index 000000000..633c38b61 --- /dev/null +++ b/packages/server/destinations/piwikpro/src/__tests__/conformance.test.ts @@ -0,0 +1,23 @@ +import path from 'path'; +import { conformance } from '@walkeros/collector/dev'; +import { destinationPiwikPro } from '..'; +import { examples, schemas } from '../dev'; + +describe('piwikpro conformance', () => { + beforeEach(() => { + jest.useRealTimers(); + }); + + it('conforms', async () => { + await conformance('destination', destinationPiwikPro, { + profile: 'http', + examples: examples.step, + schema: schemas.settings, + settings: { + url: 'https://your_account_name.piwik.pro/', + appId: 'e8f3a1c2-0b4d-4e5f-9a6b-7c8d9e0f1a2b', + }, + root: path.join(__dirname, '..', '..'), + }); + }); +}); diff --git a/packages/server/destinations/piwikpro/src/__tests__/hits.test.ts b/packages/server/destinations/piwikpro/src/__tests__/hits.test.ts index dd2a12bf8..0d6d0e72e 100644 --- a/packages/server/destinations/piwikpro/src/__tests__/hits.test.ts +++ b/packages/server/destinations/piwikpro/src/__tests__/hits.test.ts @@ -48,7 +48,7 @@ describe('buildHits', () => { /** The hits of a successful build, as parameter maps. */ function params(result: BuildResult): Array> { - if ('skip' in result) throw new Error(`unexpected skip: ${result.skip}`); + if (!('hits' in result)) throw new Error('unexpected unsent result'); return result.hits.map((hit) => new Map(hit)); } @@ -95,38 +95,50 @@ describe('buildHits', () => { }); it.each<[string, WalkerOS.Event, Rule | undefined, string]>([ - ['an unmapped event', event('promotion visible'), undefined, 'unmapped'], [ - 'an unnamed rule without a goal', - event('promotion visible'), - { settings: { customDimensions: { '1': 'data.name' } } }, - 'unmapped', + 'silent without a goal', + otherEvent('trackEvent'), + { name: 'trackEvent', silent: true }, + 'silent', ], + ])('skips %s', async (_, event, rule, skip) => { + expect(await buildHits({ event, rule }, settings, collector)).toEqual({ + skip, + }); + }); + + it.each<[string, WalkerOS.Event, Rule | undefined, string]>([ [ 'a method outside the table', otherEvent('setUserId'), { name: 'setUserId' }, 'unknown method setUserId', ], - [ - 'silent without a goal', - otherEvent('trackEvent'), - { name: 'trackEvent', silent: true }, - 'silent', - ], [ 'a missing page url', event('page view', { source: { type: 'collector' } }), undefined, 'missing url', ], - ])('skips %s', async (_, event, rule, skip) => { + [ + 'an unmapped event', + event('promotion visible'), + undefined, + 'no rule maps the event', + ], + [ + 'an unnamed rule without a goal', + event('promotion visible'), + { settings: { customDimensions: { '1': 'data.name' } } }, + 'no rule maps the event', + ], + ])('rejects %s', async (_, event, rule, invalid) => { expect(await buildHits({ event, rule }, settings, collector)).toEqual({ - skip, + invalid, }); }); - it('skips an invalid method call with its reason', async () => { + it('rejects an invalid method call with its reason', async () => { expect( await buildHits( { @@ -137,7 +149,7 @@ describe('buildHits', () => { settings, collector, ), - ).toEqual({ skip: 'trackEvent: category missing' }); + ).toEqual({ invalid: 'trackEvent: category missing' }); }); const goalRule: Rule = { @@ -207,7 +219,7 @@ describe('buildHits', () => { settings, collector, ); - if ('skip' in result) throw new Error(result.skip); + if (!('hits' in result)) throw new Error('unexpected unsent result'); const [hit] = result.hits; expect(hit.filter(([key]) => key === 'url')).toEqual([ ['url', 'https://example.com/a.pdf'], diff --git a/packages/server/destinations/piwikpro/src/__tests__/push.test.ts b/packages/server/destinations/piwikpro/src/__tests__/push.test.ts index 04aad172c..144f7e7ad 100644 --- a/packages/server/destinations/piwikpro/src/__tests__/push.test.ts +++ b/packages/server/destinations/piwikpro/src/__tests__/push.test.ts @@ -1,18 +1,11 @@ import type { Collector, - Logger, SendResponse, StandardEventName, WalkerOS, } from '@walkeros/core'; import type { PartialConfig } from '../types'; -import { - createEvent, - createIngest, - getEvent, - Level, - StepError, -} from '@walkeros/core'; +import { createEvent, createIngest, getEvent, StepError } from '@walkeros/core'; import { startFlow } from '@walkeros/collector'; import { destinationPiwikPro } from '..'; @@ -21,21 +14,15 @@ const SOURCE = { type: 'collector', url: 'https://www.example.com/docs/' }; describe('push', () => { const sendServer = jest.fn, unknown[]>(); - let logs: Array<[Level, string]>; let collector: Collector.Instance; beforeEach(() => { sendServer.mockReset(); sendServer.mockResolvedValue({ data: '' }); - logs = []; }); async function setup(config: PartialConfig = {}) { - const handler: Logger.Handler = (level, message) => { - logs.push([level, message]); - }; ({ collector } = await startFlow({ - logger: { level: 'DEBUG', handler }, destinations: { piwik: { code: { ...destinationPiwikPro, env: { sendServer } }, @@ -111,7 +98,7 @@ describe('push', () => { expect(requests()[0][0]).toContain('&action_name=M%C3%BCller+Shop&'); }); - it('sends one request per batch, with goal hits and per-entry ingest, and drops skips', async () => { + it('sends one request per batch, with goal hits and per-entry ingest, and fails the unmapped one', async () => { await setup({ batch: { size: 3, wait: 60000 }, mapping: { @@ -148,13 +135,13 @@ describe('push', () => { expect(goal.get('idgoal')).toBe('g1'); }); - it('sends nothing for a batch of skipped entries', async () => { + it('sends nothing for a batch of unmapped entries', async () => { await setup({ batch: { size: 2, wait: 60000 } }); await collector.push(event('product view')); await collector.push(event('product add')); await settle(); expect(sendServer).not.toHaveBeenCalled(); - expect(collector.status.destinations.piwik.failed).toBe(0); + expect(collector.status.destinations.piwik.failed).toBe(2); }); it('fails a push on a transport error', async () => { @@ -192,28 +179,41 @@ describe('push', () => { }); }); - it('warns once per skip reason, then logs at debug', async () => { + it('fails an event the rule cannot send, with missing-input', async () => { await setup(); - // An event name no other test pushes, so the warned key is this test's own - await collector.push( - createEvent({ - name: 'skip probe', - timestamp: 1700000300000, - source: SOURCE, - }), + const result = await collector.push( + event('page view', { source: { type: 'collector' } }), ); - await collector.push( + expect(result.failed?.piwik?.reason).toMatchObject({ + code: 'missing-input', + retryable: false, + }); + expect(sendServer).not.toHaveBeenCalled(); + }); + + it('fails only the unsendable entry of a batch', async () => { + await setup({ batch: { size: 2, wait: 60000 } }); + await collector.push(event('page view')); + await collector.push(event('page view', { source: { type: 'collector' } })); + await settle(); + expect(requests()[0]).toHaveLength(1); + expect(collector.status.destinations.piwik.count).toBe(1); + expect(collector.status.destinations.piwik.failed).toBe(1); + }); + + it('fails an unmapped event with missing-input', async () => { + await setup(); + const result = await collector.push( createEvent({ name: 'skip probe', timestamp: 1700000300000, source: SOURCE, }), ); - const skips = logs.filter(([, message]) => message.includes('skipped')); - expect(skips).toEqual([ - [Level.WARN, 'Piwik PRO skipped an event: unmapped'], - [Level.DEBUG, 'Piwik PRO skipped an event: unmapped'], - ]); + expect(result.failed?.piwik?.reason).toMatchObject({ + code: 'missing-input', + retryable: false, + }); expect(sendServer).not.toHaveBeenCalled(); }); }); diff --git a/packages/server/destinations/piwikpro/src/__tests__/skip.test.ts b/packages/server/destinations/piwikpro/src/__tests__/skip.test.ts deleted file mode 100644 index b645eefd1..000000000 --- a/packages/server/destinations/piwikpro/src/__tests__/skip.test.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { createMockLogger } from '@walkeros/core'; -import { logSkip, SKIP_KEY_CAP } from '../skip'; - -// Its own file: jest gives each test file a fresh module registry, so filling -// the key set here cannot reach the warn-once assertions in push.test.ts. -describe('logSkip', () => { - it('logs at debug once the key cap is reached', () => { - const logger = createMockLogger(); - for (let i = 0; i < SKIP_KEY_CAP; i++) - logSkip(logger, `cap|${i}`, 'skipped', {}); - logger.warn.mockClear(); - - logSkip(logger, 'cap|new', 'skipped', {}); - expect(logger.warn).not.toHaveBeenCalled(); - expect(logger.debug).toHaveBeenCalledWith('skipped', {}); - }); -}); diff --git a/packages/server/destinations/piwikpro/src/__tests__/stepExamples.test.ts b/packages/server/destinations/piwikpro/src/__tests__/stepExamples.test.ts index 53d2ae555..f57095eed 100644 --- a/packages/server/destinations/piwikpro/src/__tests__/stepExamples.test.ts +++ b/packages/server/destinations/piwikpro/src/__tests__/stepExamples.test.ts @@ -31,6 +31,11 @@ function asPublished(rule: Rule | undefined): Rule | undefined { * calls to filter. */ describe('Step Examples', () => { + // Failure examples (with an outcome) run in the conformance test. + const published = Object.entries(examples.step).filter( + ([, example]) => example.outcome === undefined, + ); + const sendServer = jest.fn, unknown[]>(); beforeEach(() => { @@ -66,14 +71,11 @@ describe('Step Examples', () => { ); } - it.each<[string, StepExample]>(Object.entries(examples.step))( - '%s', - async (_, example) => { - expect(await run(example, example.mapping)).toEqual(example.out); - }, - ); + it.each<[string, StepExample]>(published)('%s', async (_, example) => { + expect(await run(example, example.mapping)).toEqual(example.out); + }); - it.each<[string, StepExample]>(Object.entries(examples.step))( + it.each<[string, StepExample]>(published)( '%s as published', async (_, example) => { expect(await run(example, asPublished(example.mapping))).toEqual( diff --git a/packages/server/destinations/piwikpro/src/config.ts b/packages/server/destinations/piwikpro/src/config.ts index 00fb0aaba..80299d913 100644 --- a/packages/server/destinations/piwikpro/src/config.ts +++ b/packages/server/destinations/piwikpro/src/config.ts @@ -1,20 +1,21 @@ -import type { Logger } from '@walkeros/core'; import type { Config, PartialConfig } from './types'; +import { StepError } from '@walkeros/core'; /** Ends the account URL with exactly one slash, so `${url}ppms.php` is valid. */ export function normalizeUrl(url: string): string { return `${url.replace(/\/+$/, '')}/`; } -export function getConfig( - partialConfig: PartialConfig = {}, - logger: Logger.Instance, -): Config { +export function getConfig(partialConfig: PartialConfig = {}): Config { const settings = partialConfig.settings || {}; const { url, appId } = settings; - if (!url) logger.throw('Config settings url missing'); - if (!appId) logger.throw('Config settings appId missing'); + if (!url || !appId) { + const missing = [...(url ? [] : ['url']), ...(appId ? [] : ['appId'])]; + throw new StepError(`Config settings ${missing.join(', ')} missing`, { + code: 'config', + }); + } return { ...partialConfig, diff --git a/packages/server/destinations/piwikpro/src/examples/step.ts b/packages/server/destinations/piwikpro/src/examples/step.ts index d38bf28ce..51203200e 100644 --- a/packages/server/destinations/piwikpro/src/examples/step.ts +++ b/packages/server/destinations/piwikpro/src/examples/step.ts @@ -392,11 +392,58 @@ export const anonymous: StepExample = { ]), }; -export const unmappedSkip: StepExample = { +export const unmappedEvent: StepExample = { title: 'Unmapped event', description: - 'An event without a rule name that is not a page view is skipped, so nothing is sent.', + 'An event without a rule name that is not a page view has nothing to send: the push fails with missing-input. To keep such events out without a failure, ignore them with a rule.', public: false, in: event('promotion visible'), out: [], + outcome: { status: 'failed', code: 'missing-input', retryable: false }, }; + +export const silentRule: StepExample = { + title: 'Silent rule', + description: + 'A rule with silent: true and no goal sends no request: nothing is left to send, and that is not a failure.', + public: false, + in: event('page view'), + mapping: { silent: true }, + out: [], +}; + +export const missingUrl: StepExample = { + title: 'Event without a page url', + description: + 'An event without a page url cannot become a Tracking API request: the push fails with missing-input instead of counting as delivered.', + public: false, + in: event('page view', { source: { type: 'collector' } }), + out: [], + outcome: { status: 'failed', code: 'missing-input', retryable: false }, +}; + +/** + * The event of a mapped example without the data its rule reads: Piwik PRO + * cannot build the request, so the push fails with missing-input instead of + * counting as delivered. + */ +function withoutInput(target: StepExample): StepExample { + const { name } = target.in; + return { + title: `${target.title ?? name} without input`, + description: `A ${name} event without the data its rule reads cannot become a Tracking API request: the push fails with missing-input instead of counting as delivered.`, + public: false, + in: { ...target.in, data: {}, user: {}, globals: {}, context: {} }, + mapping: target.mapping, + out: [], + outcome: { status: 'failed', code: 'missing-input', retryable: false }, + }; +} + +export const siteSearchWithoutInput = withoutInput(siteSearch); +export const trackLinkWithoutInput = withoutInput(trackLink); +export const productViewWithoutInput = withoutInput(ecommerceProductDetailView); +export const addToCartWithoutInput = withoutInput(ecommerceAddToCart); +export const removeFromCartWithoutInput = withoutInput(ecommerceRemoveFromCart); +export const cartUpdateWithoutInput = withoutInput(ecommerceCartUpdate); +export const orderWithoutInput = withoutInput(ecommerceOrder); diff --git a/packages/server/destinations/piwikpro/src/hints.ts b/packages/server/destinations/piwikpro/src/hints.ts index 4dcd1352c..f90edf6c3 100644 --- a/packages/server/destinations/piwikpro/src/hints.ts +++ b/packages/server/destinations/piwikpro/src/hints.ts @@ -52,10 +52,10 @@ export const hints: Hint.Hints = { text: 'customDimensions is keyed by bare dimension id, like { "1": "data.size" }, and each value is a mapping value resolved per event. Every hit carries dimension{id}=value. Precedence per key: destination settings, then rule settings, then a dimensions object passed as the method argument (trackEvent, trackGoal, trackSiteSearch and trackLink take one).', }, 'portable-methods': { - text: 'rule.name is a Piwik PRO JavaScript method and data resolves to its positional arguments (use set for several). The server builds hits for 13 methods: trackPageView, trackEvent, trackGoal, trackSiteSearch, trackLink, trackContentImpression, trackContentInteraction, ecommerceProductDetailView, ecommerceAddToCart, ecommerceRemoveFromCart, ecommerceCartUpdate, ecommerceOrder and ping. A web destination mapping copied to the server produces the same hits. State setters such as setUserId are web only; any other method is skipped with a warning.', + text: 'rule.name is a Piwik PRO JavaScript method and data resolves to its positional arguments (use set for several). The server builds hits for 13 methods: trackPageView, trackEvent, trackGoal, trackSiteSearch, trackLink, trackContentImpression, trackContentInteraction, ecommerceProductDetailView, ecommerceAddToCart, ecommerceRemoveFromCart, ecommerceCartUpdate, ecommerceOrder and ping. A web destination mapping copied to the server produces the same hits. State setters such as setUserId are web only; any other method fails the push with missing-input.', }, 'unmapped-events': { - text: 'A page view without rule.name becomes trackPageView. Any other event without rule.name sends only its goal when the rule has settings.goalId, and is otherwise skipped with a warning (once per event name, then at debug level). A named rule with silent: true also sends only its goal. To keep other events out of the destination entirely, ignore everything with a wildcard and map what you need: { "*": { "*": { "ignore": true } }, "page": { "view": {} } }.', + text: 'A page view without rule.name becomes trackPageView. Any other event without rule.name sends only its goal when the rule has settings.goalId, and otherwise fails the push with missing-input. A named rule with silent: true also sends only its goal. To keep other events out of the destination without a failure, ignore everything with a wildcard and map what you need: { "*": { "*": { "ignore": true } }, "page": { "view": {} } }.', }, 'visitor-id': { text: 'visitorId defaults to event.user.device and is sent as _id. A 16 character hex value passes through, anything else is hashed to 16 hex characters, so walkerOS device ids always hash. For cookieless tracking, run the fingerprint transformer with output user.hash and length 16, and set visitorId: "event.user.hash".', diff --git a/packages/server/destinations/piwikpro/src/hits.ts b/packages/server/destinations/piwikpro/src/hits.ts index b88fb8124..a3ad142f3 100644 --- a/packages/server/destinations/piwikpro/src/hits.ts +++ b/packages/server/destinations/piwikpro/src/hits.ts @@ -15,7 +15,17 @@ export interface BuildInput { ingest?: Record; } -export type BuildResult = { hits: Hit[] } | { skip: string }; +/** + * `skip` is a deliberate non-send (a silent rule with nothing else to send; + * an `ignore` rule never reaches the destination); `invalid` is an event the + * rule cannot turn into a request, which fails the push. + */ +export type BuildResult = + | { hits: Hit[] } + | { skip: 'silent' } + | { invalid: string }; + +type Unsent = Exclude; /** * The method a rule selects; an explicit rule name already renamed the event. @@ -24,15 +34,17 @@ export type BuildResult = { hits: Hit[] } | { skip: string }; function selectMethod( event: WalkerOS.Event, rule: Rule | undefined, -): { method?: MethodName } | { skip: string } { +): { method?: MethodName } | Unsent { const name = rule?.name ? event.name : event.name === 'page view' ? 'trackPageView' : undefined; if (name === undefined) - return rule?.settings?.goalId === undefined ? { skip: 'unmapped' } : {}; - if (!isMethodName(name)) return { skip: `unknown method ${name}` }; + return rule?.settings?.goalId === undefined + ? { invalid: 'no rule maps the event' } + : {}; + if (!isMethodName(name)) return { invalid: `unknown method ${name}` }; return { method: name }; } @@ -85,7 +97,7 @@ export async function buildHits( const { event, rule, ingest } = input; const selected = selectMethod(event, rule); - if ('skip' in selected) return selected; + if ('skip' in selected || 'invalid' in selected) return selected; const { method } = selected; const identified = isIdentified(settings.identified, collector, event); @@ -108,7 +120,7 @@ export async function buildHits( if (method !== undefined && rule?.silent !== true) { const args = await methodArgs(input, collector); const result = methodParams(method, args); - if ('invalid' in result) return { skip: result.invalid }; + if ('invalid' in result) return { invalid: result.invalid }; const index = DIMENSIONS_ARG[method]; const argument = index === undefined ? undefined : args[index]; hits.push( @@ -130,7 +142,7 @@ export async function buildHits( ? undefined : await getMappingValue(event, goalValue, { collector }); const result = methodParams('trackGoal', [goalId, value]); - if ('invalid' in result) return { skip: result.invalid }; + if ('invalid' in result) return { invalid: result.invalid }; hits.push( assemble( settings, @@ -144,7 +156,7 @@ export async function buildHits( if (!hits.length) return { skip: 'silent' }; if (hits.some((hit) => !hit.some(([key]) => key === 'url'))) - return { skip: 'missing url' }; + return { invalid: 'missing url' }; return { hits }; } diff --git a/packages/server/destinations/piwikpro/src/index.ts b/packages/server/destinations/piwikpro/src/index.ts index 00b51f04f..6cb98a780 100644 --- a/packages/server/destinations/piwikpro/src/index.ts +++ b/packages/server/destinations/piwikpro/src/index.ts @@ -13,8 +13,8 @@ export const destinationPiwikPro: Destination = { env: { sendServer }, - async init({ config, logger }) { - return getConfig(config, logger); + async init({ config }) { + return getConfig(config); }, push, diff --git a/packages/server/destinations/piwikpro/src/push.ts b/packages/server/destinations/piwikpro/src/push.ts index 3e58fffaa..cffdaa795 100644 --- a/packages/server/destinations/piwikpro/src/push.ts +++ b/packages/server/destinations/piwikpro/src/push.ts @@ -1,71 +1,67 @@ -import type { Logger, WalkerOS } from '@walkeros/core'; import type { Hit, PushBatchFn, PushFn, Settings } from './types'; +import { StepError } from '@walkeros/core'; import { normalizeUrl } from './config'; import { buildHits } from './hits'; import { sendHits } from './send'; -import { logSkip } from './skip'; /** * init returns validated settings, but config.init: true skips it, so the url * is normalized here as well (idempotent). */ -function getSettings( - settings: Partial | undefined, - logger: Logger.Instance, -): Settings { +function getSettings(settings: Partial | undefined): Settings { const { url, appId } = settings || {}; - if (!url || !appId) logger.throw('Config settings missing, init() not run'); + if (!url || !appId) + throw new StepError('Config settings missing, init() not run', { + code: 'config', + }); return { ...settings, url: normalizeUrl(url), appId }; } -function skip( - logger: Logger.Instance, - id: string, - event: WalkerOS.Event, - reason: string, -): void { - logSkip( - logger, - `${id}|${event.name}|${reason}`, - `Piwik PRO skipped an event: ${reason}`, - { event: event.name, id: event.id }, - ); +/** An event the rule cannot turn into a request. */ +function invalid(reason: string): StepError { + return new StepError(`Piwik PRO cannot send the event: ${reason}`, { + code: 'missing-input', + }); } export const push: PushFn = async function ( event, - { config, rule, data, ingest, collector, env, logger, id }, + { config, rule, data, ingest, collector, env, logger }, ) { - const settings = getSettings(config.settings, logger); + const settings = getSettings(config.settings); const result = await buildHits( { event, rule, data, ingest }, settings, collector, ); - if ('skip' in result) return skip(logger, id, event, result.skip); + if ('invalid' in result) throw invalid(result.invalid); - await sendHits(result.hits, settings, env, logger); + // A skip is deliberate (no rule for the event, a silent rule): no request. + if ('hits' in result) await sendHits(result.hits, settings, env, logger); }; export const pushBatch: PushBatchFn = async function ( batch, - { config, collector, env, logger, id }, + { config, collector, env, logger }, ) { - const settings = getSettings(config.settings, logger); + const settings = getSettings(config.settings); const results = await Promise.all( batch.entries.map(({ event, rule, data, ingest }) => buildHits({ event, rule, data, ingest }, settings, collector), ), ); - // Skips are dropped and counted delivered: a retry cannot fix them + // An event the rule cannot send fails on its own; the rest still goes out. const hits: Hit[] = []; + const failed: Array<{ index: number; error: StepError }> = []; results.forEach((result, index) => { - if ('skip' in result) - skip(logger, id, batch.entries[index].event, result.skip); - else hits.push(...result.hits); + if ('invalid' in result) + failed.push({ index, error: invalid(result.invalid) }); + else if ('hits' in result) hits.push(...result.hits); }); if (hits.length) await sendHits(hits, settings, env, logger); + + return failed.length ? { failed } : undefined; }; diff --git a/packages/server/destinations/piwikpro/src/schemas/settings.ts b/packages/server/destinations/piwikpro/src/schemas/settings.ts index 9dc86f784..f2b4e85db 100644 --- a/packages/server/destinations/piwikpro/src/schemas/settings.ts +++ b/packages/server/destinations/piwikpro/src/schemas/settings.ts @@ -9,59 +9,61 @@ export const CustomDimensionsSchema = z.record( ValueSchema, ); -export const SettingsSchema = z.object({ - url: z - .string() - .url() - .describe( - 'Piwik PRO account URL, normalized to one trailing slash. Hits go to ppms.php (like https://your_account_name.piwik.pro/)', - ), - appId: z - .string() - .min(1) - .describe( - 'Piwik PRO site or app id, sent as idsite (like XXX-XXX-XXX-XXX-XXX)', - ), - timeout: z - .number() - .int() - .positive() - .describe('Request timeout in milliseconds. Default: 5000') - .optional(), - identified: z - .union([z.boolean(), z.record(z.string(), z.boolean())]) - .describe( - 'Identified or anonymous tracking. true (default) identifies every hit, false makes every hit anonymous (uia=1, dda=1, no _id, no uid). A consent object like { marketing: true } identifies a hit when any listed state is granted. Do not repeat these states in config.consent: a destination needs only one of its config.consent states granted, so next to another state the repeated one lets events through on its own, and as the only state it keeps events without it in the queue, so no anonymous hit is sent.', - ) - .optional(), - customDimensions: CustomDimensionsSchema.describe( - 'Custom dimensions sent as dimension{id} on every hit, keyed by bare dimension id. Values are mapping values resolved against the event (like { "1": "data.size" }). Rule-level customDimensions win per key.', - ).optional(), - ip: InputSchema.describe( - 'Visitor IP, sent as cip. Resolves against { ingest, event }. Default: ["ingest.ip", "event.user.ip"]. false switches it off. Kept on anonymous hits for country-level geolocation.', - ).optional(), - userAgent: InputSchema.describe( - 'User agent, sent as ua. Resolves against { ingest, event }. Default: ["ingest.userAgent", "event.user.userAgent"]. false switches it off.', - ).optional(), - language: InputSchema.describe( - 'Accept-Language value, sent as lang. Resolves against { ingest, event }. Default: ["ingest.language", "event.user.language"]. false switches it off.', - ).optional(), - pageUrl: InputSchema.describe( - 'Page URL, sent as url (required by the Tracking API, a hit without it is skipped). Resolves against { ingest, event }. Default: "event.source.url". false switches it off.', - ).optional(), - referrer: InputSchema.describe( - 'Referrer URL, sent as urlref. Resolves against { ingest, event }. Default: "event.source.referrer". false switches it off.', - ).optional(), - visitorId: InputSchema.describe( - 'Visitor id, sent as _id on identified hits. A 16-character hex value passes through, anything else is hashed to 16 hex (sha256). Resolves against { ingest, event }. Default: "event.user.device". false switches it off.', - ).optional(), - userId: InputSchema.describe( - 'User id, sent as uid on identified hits. Resolves against { ingest, event }. Default: "event.user.id". false switches it off.', - ).optional(), - pageViewId: InputSchema.describe( - 'Page view id, sent as pv_id (first 6 hex characters, else a 6-character hash). Resolves against { ingest, event }. Default: "event.source.trace", only for web events whose trace differs from the server collector trace. false switches it off.', - ).optional(), - timestamp: InputSchema.describe( - 'Event time in milliseconds, sent as cdt in UNIX seconds. Resolves against { ingest, event }. Default: "event.timestamp". false lets Piwik PRO use the receive time.', - ).optional(), -}); +export const SettingsSchema = z + .object({ + url: z + .string() + .url() + .describe( + 'Piwik PRO account URL, normalized to one trailing slash. Hits go to ppms.php (like https://your_account_name.piwik.pro/)', + ), + appId: z + .string() + .min(1) + .describe( + 'Piwik PRO site or app id, sent as idsite (like XXX-XXX-XXX-XXX-XXX)', + ), + timeout: z + .number() + .int() + .positive() + .describe('Request timeout in milliseconds. Default: 5000') + .optional(), + identified: z + .union([z.boolean(), z.record(z.string(), z.boolean())]) + .describe( + 'Identified or anonymous tracking. true (default) identifies every hit, false makes every hit anonymous (uia=1, dda=1, no _id, no uid). A consent object like { marketing: true } identifies a hit when any listed state is granted. Do not repeat these states in config.consent: a destination needs only one of its config.consent states granted, so next to another state the repeated one lets events through on its own, and as the only state it keeps events without it in the queue, so no anonymous hit is sent.', + ) + .optional(), + customDimensions: CustomDimensionsSchema.describe( + 'Custom dimensions sent as dimension{id} on every hit, keyed by bare dimension id. Values are mapping values resolved against the event (like { "1": "data.size" }). Rule-level customDimensions win per key.', + ).optional(), + ip: InputSchema.describe( + 'Visitor IP, sent as cip. Resolves against { ingest, event }. Default: ["ingest.ip", "event.user.ip"]. false switches it off. Kept on anonymous hits for country-level geolocation.', + ).optional(), + userAgent: InputSchema.describe( + 'User agent, sent as ua. Resolves against { ingest, event }. Default: ["ingest.userAgent", "event.user.userAgent"]. false switches it off.', + ).optional(), + language: InputSchema.describe( + 'Accept-Language value, sent as lang. Resolves against { ingest, event }. Default: ["ingest.language", "event.user.language"]. false switches it off.', + ).optional(), + pageUrl: InputSchema.describe( + 'Page URL, sent as url (required by the Tracking API, a hit without it is skipped). Resolves against { ingest, event }. Default: "event.source.url". false switches it off.', + ).optional(), + referrer: InputSchema.describe( + 'Referrer URL, sent as urlref. Resolves against { ingest, event }. Default: "event.source.referrer". false switches it off.', + ).optional(), + visitorId: InputSchema.describe( + 'Visitor id, sent as _id on identified hits. A 16-character hex value passes through, anything else is hashed to 16 hex (sha256). Resolves against { ingest, event }. Default: "event.user.device". false switches it off.', + ).optional(), + userId: InputSchema.describe( + 'User id, sent as uid on identified hits. Resolves against { ingest, event }. Default: "event.user.id". false switches it off.', + ).optional(), + pageViewId: InputSchema.describe( + 'Page view id, sent as pv_id (first 6 hex characters, else a 6-character hash). Resolves against { ingest, event }. Default: "event.source.trace", only for web events whose trace differs from the server collector trace. false switches it off.', + ).optional(), + timestamp: InputSchema.describe( + 'Event time in milliseconds, sent as cdt in UNIX seconds. Resolves against { ingest, event }. Default: "event.timestamp". false lets Piwik PRO use the receive time.', + ).optional(), + }) + .strict(); diff --git a/packages/server/destinations/piwikpro/src/skip.ts b/packages/server/destinations/piwikpro/src/skip.ts deleted file mode 100644 index 567463f65..000000000 --- a/packages/server/destinations/piwikpro/src/skip.ts +++ /dev/null @@ -1,22 +0,0 @@ -import type { Logger } from '@walkeros/core'; - -/** Warned keys are capped, so unbounded event names cannot grow memory. */ -export const SKIP_KEY_CAP = 1000; - -const warned = new Set(); - -/** One warn per key, debug afterwards; every key beyond the cap logs at debug. */ -export function logSkip( - logger: Logger.Instance, - key: string, - message: string, - meta: Record, -): void { - if (warned.has(key) || warned.size >= SKIP_KEY_CAP) { - logger.debug(message, meta); - return; - } - - warned.add(key); - logger.warn(message, meta); -} diff --git a/packages/server/destinations/reddit/src/__tests__/conformance.test.ts b/packages/server/destinations/reddit/src/__tests__/conformance.test.ts new file mode 100644 index 000000000..5edd71c59 --- /dev/null +++ b/packages/server/destinations/reddit/src/__tests__/conformance.test.ts @@ -0,0 +1,20 @@ +import path from 'path'; +import { conformance } from '@walkeros/collector/dev'; +import { destinationReddit } from '..'; +import { examples, schemas } from '../dev'; + +describe('reddit conformance', () => { + beforeEach(() => { + jest.useRealTimers(); + }); + + it('conforms', async () => { + await conformance('destination', destinationReddit, { + profile: 'http', + examples: examples.step, + schema: schemas.settings, + settings: { accessToken: 's3cr3t', pixelId: 'a2_abcdef123456' }, + root: path.join(__dirname, '..', '..'), + }); + }); +}); diff --git a/packages/server/destinations/reddit/src/__tests__/index.test.ts b/packages/server/destinations/reddit/src/__tests__/index.test.ts index dbfa8e370..da40d0c6a 100644 --- a/packages/server/destinations/reddit/src/__tests__/index.test.ts +++ b/packages/server/destinations/reddit/src/__tests__/index.test.ts @@ -57,7 +57,7 @@ describe('Server Destination Reddit', () => { logger: createMockLogger(), id: 'test-reddit', }), - ).rejects.toThrow('Config settings accessToken missing'); + ).rejects.toThrow('Config settings accessToken, pixelId missing'); await expect( destination.init({ config: { settings: { accessToken, pixelId: '' } }, diff --git a/packages/server/destinations/reddit/src/__tests__/stepExamples.test.ts b/packages/server/destinations/reddit/src/__tests__/stepExamples.test.ts index 90577b696..6c6c41033 100644 --- a/packages/server/destinations/reddit/src/__tests__/stepExamples.test.ts +++ b/packages/server/destinations/reddit/src/__tests__/stepExamples.test.ts @@ -20,7 +20,12 @@ describe('Step Examples', () => { }); }); - it.each(Object.entries(examples.step))('%s', async (_name, example) => { + // Non-public examples (silent, failures) run in the conformance test. + const published = Object.entries(examples.step).filter( + ([, example]) => example.public !== false, + ); + + it.each(published)('%s', async (_name, example) => { const event = example.in as WalkerOS.Event; const mapping = example.mapping; diff --git a/packages/server/destinations/reddit/src/config.ts b/packages/server/destinations/reddit/src/config.ts index 840d97a3b..7ab7143f4 100644 --- a/packages/server/destinations/reddit/src/config.ts +++ b/packages/server/destinations/reddit/src/config.ts @@ -1,15 +1,19 @@ import type { Config, Settings, PartialConfig } from './types'; -import type { Logger } from '@walkeros/core'; +import { StepError } from '@walkeros/core'; -export function getConfig( - partialConfig: PartialConfig = {}, - logger: Logger.Instance, -): Config { +export function getConfig(partialConfig: PartialConfig = {}): Config { const settings = (partialConfig.settings || {}) as Partial; const { accessToken, pixelId } = settings; - if (!accessToken) logger.throw('Config settings accessToken missing'); - if (!pixelId) logger.throw('Config settings pixelId missing'); + if (!accessToken || !pixelId) { + const missing = [ + ...(accessToken ? [] : ['accessToken']), + ...(pixelId ? [] : ['pixelId']), + ]; + throw new StepError(`Config settings ${missing.join(', ')} missing`, { + code: 'config', + }); + } const settingsConfig: Settings = { ...settings, diff --git a/packages/server/destinations/reddit/src/examples/step.ts b/packages/server/destinations/reddit/src/examples/step.ts index d4e9d40de..de3f182eb 100644 --- a/packages/server/destinations/reddit/src/examples/step.ts +++ b/packages/server/destinations/reddit/src/examples/step.ts @@ -384,3 +384,12 @@ export const search: Flow.StepExample = { ], ], }; + +/** A silent rule sends nothing to Reddit. */ +export const silentRule: Flow.StepExample = { + public: false, + description: 'A rule with silent: true sends no request to Reddit.', + in: getEvent('page view', { timestamp: 1700000990000 }), + mapping: { silent: true }, + out: [], +}; diff --git a/packages/server/destinations/reddit/src/index.ts b/packages/server/destinations/reddit/src/index.ts index e2c10e435..8044b24d6 100644 --- a/packages/server/destinations/reddit/src/index.ts +++ b/packages/server/destinations/reddit/src/index.ts @@ -10,8 +10,8 @@ export const destinationReddit: Destination = { config: {}, - async init({ config: partialConfig, logger }) { - const config = getConfig(partialConfig, logger); + async init({ config: partialConfig }) { + const config = getConfig(partialConfig); return config; }, diff --git a/packages/server/destinations/reddit/src/push.ts b/packages/server/destinations/reddit/src/push.ts index 73f3b6cb8..fdfc49955 100644 --- a/packages/server/destinations/reddit/src/push.ts +++ b/packages/server/destinations/reddit/src/push.ts @@ -35,7 +35,12 @@ function buildEventType(name: string): EventType { return { tracking_type: 'Custom', custom_event_name: name }; } -export const push: PushFn = async function ( +// A silent rule has no reddit call of its own: nothing to send. +export const push: PushFn = async function (event, context) { + if (!context.rule?.silent) await send(event, context); +}; + +const send: PushFn = async function ( event, { config, data, ingest, env, logger, collector }, ) { diff --git a/packages/server/destinations/reddit/src/schemas/settings.ts b/packages/server/destinations/reddit/src/schemas/settings.ts index 10eb172b4..3c6fe0466 100644 --- a/packages/server/destinations/reddit/src/schemas/settings.ts +++ b/packages/server/destinations/reddit/src/schemas/settings.ts @@ -6,51 +6,55 @@ const InputSchema = z.union([ z.literal(false), ]); -export const SettingsSchema = z.object({ - accessToken: z - .string() - .min(1) - .describe( - 'Reddit Conversion Access Token for Bearer authentication (like rdt_ABC123...)', - ), - pixelId: z - .string() - .min(1) - .describe( - 'Reddit Pixel ID used as the API path parameter (like a2_abcdef123456)', - ), - action_source: ActionSourceSchema.describe( - 'Source of the event (WEBSITE, APP, PHYSICAL_STORE) (like WEBSITE)', - ).optional(), - doNotHash: z - .array(z.string()) - .describe("Array of user fields that should not be hashed (like ['email'])") - .optional(), - test_mode: z - .boolean() - .describe( - 'Enable test mode by sending test_mode: true in the request body (like true)', - ) - .optional(), - url: z - .string() - .url() - .describe( - 'Custom URL for Reddit Conversions API endpoint (like https://ads-api.reddit.com/api/v2.0/conversions/events/)', - ) - .optional(), - user_data: z - .record(z.string(), z.string()) - .describe( - "Mapping configuration for user fields (like { email: 'user.email', external_id: 'user.id' })", - ) - .optional(), - ip: InputSchema.describe( - 'Client IP, sent as user.ip_address (hashed). Resolves against { ingest, event }. Default: ["ingest.ip", "event.user.ip"]. false switches it off. A mapped user.ip_address wins.', - ).optional(), - userAgent: InputSchema.describe( - 'Client user agent, sent as user.user_agent (hashed). Resolves against { ingest, event }. Default: ["ingest.userAgent", "event.user.userAgent"]. false switches it off. A mapped user.user_agent wins.', - ).optional(), -}); +export const SettingsSchema = z + .object({ + accessToken: z + .string() + .min(1) + .describe( + 'Reddit Conversion Access Token for Bearer authentication (like rdt_ABC123...)', + ), + pixelId: z + .string() + .min(1) + .describe( + 'Reddit Pixel ID used as the API path parameter (like a2_abcdef123456)', + ), + action_source: ActionSourceSchema.describe( + 'Source of the event (WEBSITE, APP, PHYSICAL_STORE) (like WEBSITE)', + ).optional(), + doNotHash: z + .array(z.string()) + .describe( + "Array of user fields that should not be hashed (like ['email'])", + ) + .optional(), + test_mode: z + .boolean() + .describe( + 'Enable test mode by sending test_mode: true in the request body (like true)', + ) + .optional(), + url: z + .string() + .url() + .describe( + 'Custom URL for Reddit Conversions API endpoint (like https://ads-api.reddit.com/api/v2.0/conversions/events/)', + ) + .optional(), + user_data: z + .record(z.string(), z.string()) + .describe( + "Mapping configuration for user fields (like { email: 'user.email', external_id: 'user.id' })", + ) + .optional(), + ip: InputSchema.describe( + 'Client IP, sent as user.ip_address (hashed). Resolves against { ingest, event }. Default: ["ingest.ip", "event.user.ip"]. false switches it off. A mapped user.ip_address wins.', + ).optional(), + userAgent: InputSchema.describe( + 'Client user agent, sent as user.user_agent (hashed). Resolves against { ingest, event }. Default: ["ingest.userAgent", "event.user.userAgent"]. false switches it off. A mapped user.user_agent wins.', + ).optional(), + }) + .strict(); export type Settings = z.infer; diff --git a/packages/server/destinations/slack/src/__tests__/conformance.test.ts b/packages/server/destinations/slack/src/__tests__/conformance.test.ts new file mode 100644 index 000000000..30061df68 --- /dev/null +++ b/packages/server/destinations/slack/src/__tests__/conformance.test.ts @@ -0,0 +1,20 @@ +import path from 'path'; +import { conformance } from '@walkeros/collector/dev'; +import { destinationSlack } from '..'; +import { examples, schemas } from '../dev'; + +describe('slack conformance', () => { + beforeEach(() => { + jest.useRealTimers(); + }); + + it('conforms', async () => { + await conformance('destination', destinationSlack, { + profile: 'http', + examples: examples.step, + schema: schemas.settings, + settings: { webhookUrl: 'https://hooks.slack.com/services/x/y/z' }, + root: path.join(__dirname, '..', '..'), + }); + }); +}); diff --git a/packages/server/destinations/slack/src/__tests__/stepExamples.test.ts b/packages/server/destinations/slack/src/__tests__/stepExamples.test.ts index 6dad8c033..f9642ee2b 100644 --- a/packages/server/destinations/slack/src/__tests__/stepExamples.test.ts +++ b/packages/server/destinations/slack/src/__tests__/stepExamples.test.ts @@ -59,7 +59,12 @@ function spyEnv(): { env: Env; collected: () => Captured[] } { } describe('slack server destination -- step examples', () => { - it.each(Object.entries(examples.step))('%s', async (_name, rawExample) => { + // Non-public examples (silent, failures) run in the conformance test. + const published = Object.entries(examples.step).filter( + ([, example]) => example.public !== false, + ); + + it.each(published)('%s', async (_name, rawExample) => { const example = rawExample as { in?: unknown; mapping?: unknown; diff --git a/packages/server/destinations/slack/src/config.ts b/packages/server/destinations/slack/src/config.ts index d89d427be..05bf62ac2 100644 --- a/packages/server/destinations/slack/src/config.ts +++ b/packages/server/destinations/slack/src/config.ts @@ -1,5 +1,6 @@ import type { Logger } from '@walkeros/core'; import type { Config, PartialConfig, Settings } from './types'; +import { StepError } from '@walkeros/core'; export function getConfig( partialConfig: PartialConfig = {}, @@ -9,11 +10,16 @@ export function getConfig( const { token, webhookUrl, channel } = settings; if (!token && !webhookUrl) - logger.throw('Slack destination requires either token or webhookUrl'); + throw new StepError( + 'Slack destination requires either token or webhookUrl', + { code: 'config' }, + ); if (token && webhookUrl) - logger.throw( + throw new StepError( 'Slack destination accepts either token or webhookUrl, not both', + { code: 'config' }, ); + // An advisory, not a failure: a push without a channel fails with config. if (token && !channel) logger.warn( 'Slack destination has no default channel; every mapping rule must provide one', diff --git a/packages/server/destinations/slack/src/examples/step.ts b/packages/server/destinations/slack/src/examples/step.ts index 7b11b662e..5519a3dba 100644 --- a/packages/server/destinations/slack/src/examples/step.ts +++ b/packages/server/destinations/slack/src/examples/step.ts @@ -293,3 +293,12 @@ export const deployNotification: SlackStepExample = { ], ], }; + +/** A silent rule posts nothing to Slack. */ +export const silentRule: SlackStepExample = { + public: false, + description: 'A rule with silent: true posts no message to Slack.', + in: getEvent('page view', { timestamp: 1700000990000 }), + mapping: { silent: true }, + out: [], +}; diff --git a/packages/server/destinations/slack/src/index.ts b/packages/server/destinations/slack/src/index.ts index 1cc4bdc7b..2b379ca45 100644 --- a/packages/server/destinations/slack/src/index.ts +++ b/packages/server/destinations/slack/src/index.ts @@ -1,4 +1,5 @@ import type { Destination, Settings, SlackClientMock } from './types'; +import { StepError } from '@walkeros/core'; import { getConfig } from './config'; import { push } from './push'; @@ -37,7 +38,10 @@ export const destinationSlack: Destination = { ...(retryConfig !== undefined ? { retryConfig } : {}), }); } catch (err) { - logger.throw(`Failed to initialize Slack WebClient: ${err}`); + throw new StepError('Failed to initialize Slack WebClient', { + code: 'config', + cause: err, + }); } return config; diff --git a/packages/server/destinations/slack/src/push.ts b/packages/server/destinations/slack/src/push.ts index 7ff808c23..a124e4027 100644 --- a/packages/server/destinations/slack/src/push.ts +++ b/packages/server/destinations/slack/src/push.ts @@ -1,34 +1,40 @@ +import { StepError } from '@walkeros/core'; import { sendServer } from '@walkeros/server-core'; import type { Mapping, PushFn, Settings, SlackClientMock } from './types'; import { buildMessage } from './message'; -export const push: PushFn = async function (event, ctx) { - const { config, rule, env, logger } = ctx; +// A silent rule has no Slack message of its own: nothing to send. +export const push: PushFn = async function (event, context) { + if (!context.rule?.silent) await send(event, context); +}; + +const send: PushFn = async function (event, ctx) { + const { config, rule, env } = ctx; const settings = config.settings as Settings; const ruleSettings = (rule?.settings || {}) as Partial; const message = buildMessage(event, settings, ruleSettings); + // A message without text and blocks has nothing to show in Slack. + if (!message.text && !message.blocks) + throw new StepError('Slack message has no text or blocks', { + code: 'missing-input', + }); + // Webhook mode if (settings.webhookUrl) { - return await pushWebhook(message, settings, env, logger); + await pushWebhook(message, settings, env); + } else { + // Web API mode + await pushWebApi(message, settings, ruleSettings, env); } - - // Web API mode - return await pushWebApi(message, settings, ruleSettings, env, logger); }; async function pushWebhook( message: { text?: string; blocks?: unknown[] }, settings: Settings, env: { sendServer?: typeof sendServer } | undefined, - logger: import('@walkeros/core').Logger.Instance, ): Promise { - if (!message.text && !message.blocks) { - logger.warn('Slack message has no text or blocks; skipping'); - return; - } - const body = { ...(message.text ? { text: message.text } : {}), ...(message.blocks ? { blocks: message.blocks } : {}), @@ -46,50 +52,44 @@ async function pushWebApi( settings: Settings, rule: Partial, env: { slackClient?: SlackClientMock } | undefined, - logger: import('@walkeros/core').Logger.Instance, ): Promise { const client = (env?.slackClient as SlackClientMock | undefined) || (settings._client as unknown as SlackClientMock | undefined); - if (!client) { - logger.warn('Slack WebClient not initialized'); - return; - } - - if (!message.text && !message.blocks) { - logger.warn('Slack message has no text or blocks; skipping'); - return; - } + if (!client) + throw new StepError('Slack WebClient not initialized', { + code: 'not-ready', + }); // Resolve target channel let channel = rule.channel ?? settings.channel; // DM path: open DM channel first if (rule.dm) { - if (!rule.user) { - logger.throw('Slack DM requires `mapping.settings.user`'); - return; - } + if (!rule.user) + throw new StepError('Slack DM requires `mapping.settings.user`', { + code: 'config', + }); const opened = await client.conversations.open({ users: rule.user }); const dmChannel = ( opened as unknown as { channel?: { id?: string }; } ).channel?.id; - if (!dmChannel) { - logger.warn('Slack conversations.open returned no channel id'); - return; - } + if (!dmChannel) + throw new StepError('Slack conversations.open returned no channel id', { + code: 'transport', + retryable: false, + }); channel = dmChannel; } - if (!channel) { - logger.throw( + if (!channel) + throw new StepError( 'Slack push requires a channel (set settings.channel or mapping.settings.channel)', + { code: 'config' }, ); - return; - } const baseArgs: Record = { channel, @@ -107,16 +107,15 @@ async function pushWebApi( // Ephemeral path if (rule.ephemeral) { - if (!rule.user) { - logger.throw('Slack ephemeral requires `mapping.settings.user`'); - return; - } + if (!rule.user) + throw new StepError('Slack ephemeral requires `mapping.settings.user`', { + code: 'config', + }); await client.chat.postEphemeral({ ...baseArgs, user: rule.user, } as never); - return; + } else { + await client.chat.postMessage(baseArgs as never); } - - await client.chat.postMessage(baseArgs as never); } diff --git a/packages/server/destinations/slack/src/schemas/settings.ts b/packages/server/destinations/slack/src/schemas/settings.ts index 747b4cea0..c254679e6 100644 --- a/packages/server/destinations/slack/src/schemas/settings.ts +++ b/packages/server/destinations/slack/src/schemas/settings.ts @@ -65,8 +65,11 @@ export const SettingsSchema = z .describe('Retry policy passed to WebClient. Default: "default".') .optional(), }) + .strict() .refine((v) => Boolean(v.token) !== Boolean(v.webhookUrl), { message: 'Provide exactly one of `token` or `webhookUrl`.', - }); + }) + // The same rule in JSON Schema: each alternative requires one credential. + .meta({ oneOf: [{ required: ['token'] }, { required: ['webhookUrl'] }] }); export type Settings = z.infer; diff --git a/packages/server/destinations/snapchat/src/__tests__/conformance.test.ts b/packages/server/destinations/snapchat/src/__tests__/conformance.test.ts new file mode 100644 index 000000000..790d0451d --- /dev/null +++ b/packages/server/destinations/snapchat/src/__tests__/conformance.test.ts @@ -0,0 +1,20 @@ +import path from 'path'; +import { conformance } from '@walkeros/collector/dev'; +import { destinationSnapchat } from '..'; +import { examples, schemas } from '../dev'; + +describe('snapchat conformance', () => { + beforeEach(() => { + jest.useRealTimers(); + }); + + it('conforms', async () => { + await conformance('destination', destinationSnapchat, { + profile: 'http', + examples: examples.step, + schema: schemas.settings, + settings: { accessToken: 's3cr3t', pixelId: 'p1x3l1d' }, + root: path.join(__dirname, '..', '..'), + }); + }); +}); diff --git a/packages/server/destinations/snapchat/src/__tests__/index.test.ts b/packages/server/destinations/snapchat/src/__tests__/index.test.ts index 904f46e38..31c3b1277 100644 --- a/packages/server/destinations/snapchat/src/__tests__/index.test.ts +++ b/packages/server/destinations/snapchat/src/__tests__/index.test.ts @@ -58,7 +58,7 @@ describe('Server Destination Snapchat', () => { logger: createMockLogger(), id: 'test-snapchat', }), - ).rejects.toThrow('Config settings accessToken missing'); + ).rejects.toThrow('Config settings accessToken, pixelId missing'); }); test('init - missing pixelId throws', async () => { diff --git a/packages/server/destinations/snapchat/src/__tests__/stepExamples.test.ts b/packages/server/destinations/snapchat/src/__tests__/stepExamples.test.ts index 5982d86c2..457d9b60a 100644 --- a/packages/server/destinations/snapchat/src/__tests__/stepExamples.test.ts +++ b/packages/server/destinations/snapchat/src/__tests__/stepExamples.test.ts @@ -21,7 +21,12 @@ describe('Step Examples', () => { }); }); - it.each(Object.entries(examples.step))('%s', async (_name, example) => { + // Non-public examples (silent, failures) run in the conformance test. + const published = Object.entries(examples.step).filter( + ([, example]) => example.public !== false, + ); + + it.each(published)('%s', async (_name, example) => { const event = example.in as WalkerOS.Event; const mapping = example.mapping; diff --git a/packages/server/destinations/snapchat/src/config.ts b/packages/server/destinations/snapchat/src/config.ts index 89f3c3db0..d1aa786bd 100644 --- a/packages/server/destinations/snapchat/src/config.ts +++ b/packages/server/destinations/snapchat/src/config.ts @@ -1,15 +1,19 @@ import type { Config, Settings, PartialConfig } from './types'; -import type { Logger } from '@walkeros/core'; +import { StepError } from '@walkeros/core'; -export function getConfig( - partialConfig: PartialConfig = {}, - logger: Logger.Instance, -): Config { +export function getConfig(partialConfig: PartialConfig = {}): Config { const settings = (partialConfig.settings || {}) as Partial; const { accessToken, pixelId } = settings; - if (!accessToken) logger.throw('Config settings accessToken missing'); - if (!pixelId) logger.throw('Config settings pixelId missing'); + if (!accessToken || !pixelId) { + const missing = [ + ...(accessToken ? [] : ['accessToken']), + ...(pixelId ? [] : ['pixelId']), + ]; + throw new StepError(`Config settings ${missing.join(', ')} missing`, { + code: 'config', + }); + } const settingsConfig: Settings = { action_source: 'WEB', diff --git a/packages/server/destinations/snapchat/src/examples/step.ts b/packages/server/destinations/snapchat/src/examples/step.ts index acabc4ecc..9f20e1925 100644 --- a/packages/server/destinations/snapchat/src/examples/step.ts +++ b/packages/server/destinations/snapchat/src/examples/step.ts @@ -284,3 +284,12 @@ export const lead: Flow.StepExample = { ], ], }; + +/** A silent rule sends nothing to Snapchat. */ +export const silentRule: Flow.StepExample = { + public: false, + description: 'A rule with silent: true sends no request to Snapchat.', + in: getEvent('page view', { timestamp: 1700000990000 }), + mapping: { silent: true }, + out: [], +}; diff --git a/packages/server/destinations/snapchat/src/index.ts b/packages/server/destinations/snapchat/src/index.ts index 0e40b7211..92168f316 100644 --- a/packages/server/destinations/snapchat/src/index.ts +++ b/packages/server/destinations/snapchat/src/index.ts @@ -10,8 +10,8 @@ export const destinationSnapchat: Destination = { config: {}, - async init({ config: partialConfig, logger }) { - const config = getConfig(partialConfig, logger); + async init({ config: partialConfig }) { + const config = getConfig(partialConfig); return config; }, diff --git a/packages/server/destinations/snapchat/src/push.ts b/packages/server/destinations/snapchat/src/push.ts index e23d0c201..ea3468bfa 100644 --- a/packages/server/destinations/snapchat/src/push.ts +++ b/packages/server/destinations/snapchat/src/push.ts @@ -12,7 +12,12 @@ import { createMappingRoot, getMappingValue, isObject } from '@walkeros/core'; import { sendServer } from '@walkeros/server-core'; import { hashUserData } from './hash'; -export const push: PushFn = async function ( +// A silent rule has no snapchat call of its own: nothing to send. +export const push: PushFn = async function (event, context) { + if (!context.rule?.silent) await send(event, context); +}; + +const send: PushFn = async function ( event, { config, rule, data, ingest, collector, env, logger }, ) { diff --git a/packages/server/destinations/snapchat/src/schemas/settings.ts b/packages/server/destinations/snapchat/src/schemas/settings.ts index 73e4ef8ba..b67906214 100644 --- a/packages/server/destinations/snapchat/src/schemas/settings.ts +++ b/packages/server/destinations/snapchat/src/schemas/settings.ts @@ -5,45 +5,47 @@ const InputSchema = z.union([ z.literal(false), ]); -export const SettingsSchema = z.object({ - accessToken: z - .string() - .min(1) - .describe( - 'Long-lived Conversions API access token from Snapchat Ads Manager', - ), - pixelId: z.string().min(1).describe('Snap Pixel ID'), - url: z - .string() - .url() - .describe( - 'Custom Conversions API base URL (default https://tr.snapchat.com/v3/)', - ) - .optional(), - action_source: z - .enum(['WEB', 'MOBILE_APP', 'OFFLINE']) - .describe('Event action source (default WEB)') - .optional(), - doNotHash: z - .array(z.string()) - .describe("User data fields to skip hashing (like ['em', 'ph'])") - .optional(), - user_data: z - .record(z.string(), z.string()) - .describe("Mapping for user data fields (like { em: 'user.email' })") - .optional(), - testMode: z - .boolean() - .describe( - 'When true, sends events to /events/validate instead of /events for testing', - ) - .optional(), - ip: InputSchema.describe( - 'Client IP, sent as user_data.client_ip_address. Resolves against { ingest, event }. Default: ["ingest.ip", "event.user.ip"]. false switches it off. A mapped user_data.client_ip_address wins.', - ).optional(), - userAgent: InputSchema.describe( - 'Client user agent, sent as user_data.client_user_agent. Resolves against { ingest, event }. Default: ["ingest.userAgent", "event.user.userAgent"]. false switches it off. A mapped user_data.client_user_agent wins.', - ).optional(), -}); +export const SettingsSchema = z + .object({ + accessToken: z + .string() + .min(1) + .describe( + 'Long-lived Conversions API access token from Snapchat Ads Manager', + ), + pixelId: z.string().min(1).describe('Snap Pixel ID'), + url: z + .string() + .url() + .describe( + 'Custom Conversions API base URL (default https://tr.snapchat.com/v3/)', + ) + .optional(), + action_source: z + .enum(['WEB', 'MOBILE_APP', 'OFFLINE']) + .describe('Event action source (default WEB)') + .optional(), + doNotHash: z + .array(z.string()) + .describe("User data fields to skip hashing (like ['em', 'ph'])") + .optional(), + user_data: z + .record(z.string(), z.string()) + .describe("Mapping for user data fields (like { em: 'user.email' })") + .optional(), + testMode: z + .boolean() + .describe( + 'When true, sends events to /events/validate instead of /events for testing', + ) + .optional(), + ip: InputSchema.describe( + 'Client IP, sent as user_data.client_ip_address. Resolves against { ingest, event }. Default: ["ingest.ip", "event.user.ip"]. false switches it off. A mapped user_data.client_ip_address wins.', + ).optional(), + userAgent: InputSchema.describe( + 'Client user agent, sent as user_data.client_user_agent. Resolves against { ingest, event }. Default: ["ingest.userAgent", "event.user.userAgent"]. false switches it off. A mapped user_data.client_user_agent wins.', + ).optional(), + }) + .strict(); export type Settings = z.infer; diff --git a/packages/server/destinations/twitter/src/__tests__/conformance.test.ts b/packages/server/destinations/twitter/src/__tests__/conformance.test.ts new file mode 100644 index 000000000..fb01150f1 --- /dev/null +++ b/packages/server/destinations/twitter/src/__tests__/conformance.test.ts @@ -0,0 +1,27 @@ +import path from 'path'; +import { conformance } from '@walkeros/collector/dev'; +import { destinationTwitter } from '..'; +import { examples, schemas } from '../dev'; + +describe('twitter conformance', () => { + beforeEach(() => { + jest.useRealTimers(); + }); + + it('conforms', async () => { + await conformance('destination', destinationTwitter, { + profile: 'http', + examples: examples.step, + schema: schemas.settings, + settings: { + pixelId: 'o8z6j', + eventId: 'tw-o8z6j-o8z21', + consumerKey: 'consumer-key', + consumerSecret: 'consumer-secret', + accessToken: 'access-token', + accessTokenSecret: 'access-token-secret', + }, + root: path.join(__dirname, '..', '..'), + }); + }); +}); diff --git a/packages/server/destinations/twitter/src/__tests__/index.test.ts b/packages/server/destinations/twitter/src/__tests__/index.test.ts index c51adc499..f60ba3c66 100644 --- a/packages/server/destinations/twitter/src/__tests__/index.test.ts +++ b/packages/server/destinations/twitter/src/__tests__/index.test.ts @@ -364,17 +364,19 @@ describe('Server Destination Twitter', () => { expect(twclidId).toEqual({ twclid: 'raw-twclid-value' }); }); - test('no primary identifier - event is skipped', async () => { + test('no primary identifier - event fails with missing-input', async () => { const event = createEvent({ name: 'form submit', user: { id: 'user-123' }, }); const config: Config = { settings: baseSettings }; - await destination.push( - event, - createMockContext({ config, env: testEnv, id: 'test-twitter' }), - ); + await expect( + destination.push( + event, + createMockContext({ config, env: testEnv, id: 'test-twitter' }), + ), + ).rejects.toMatchObject({ code: 'missing-input' }); expect(mockSendServer).not.toHaveBeenCalled(); }); diff --git a/packages/server/destinations/twitter/src/__tests__/stepExamples.test.ts b/packages/server/destinations/twitter/src/__tests__/stepExamples.test.ts index f780e53d4..d8ab0515b 100644 --- a/packages/server/destinations/twitter/src/__tests__/stepExamples.test.ts +++ b/packages/server/destinations/twitter/src/__tests__/stepExamples.test.ts @@ -23,7 +23,12 @@ describe('Step Examples', () => { }); }); - it.each(Object.entries(examples.step))('%s', async (_name, example) => { + // Non-public examples (silent, failures) run in the conformance test. + const published = Object.entries(examples.step).filter( + ([, example]) => example.public !== false, + ); + + it.each(published)('%s', async (_name, example) => { const event = example.in as WalkerOS.Event; const mapping = example.mapping; diff --git a/packages/server/destinations/twitter/src/config.ts b/packages/server/destinations/twitter/src/config.ts index 2348478a0..417de0881 100644 --- a/packages/server/destinations/twitter/src/config.ts +++ b/packages/server/destinations/twitter/src/config.ts @@ -1,10 +1,7 @@ import type { Config, Settings, PartialConfig } from './types'; -import type { Logger } from '@walkeros/core'; +import { StepError } from '@walkeros/core'; -export function getConfig( - partialConfig: PartialConfig = {}, - logger: Logger.Instance, -): Config { +export function getConfig(partialConfig: PartialConfig = {}): Config { const settings = (partialConfig.settings || {}) as Partial; const { pixelId, @@ -15,13 +12,26 @@ export function getConfig( accessTokenSecret, } = settings; - if (!pixelId) logger.throw('Config settings pixelId missing'); - if (!eventId) logger.throw('Config settings eventId missing'); - if (!consumerKey) logger.throw('Config settings consumerKey missing'); - if (!consumerSecret) logger.throw('Config settings consumerSecret missing'); - if (!accessToken) logger.throw('Config settings accessToken missing'); - if (!accessTokenSecret) - logger.throw('Config settings accessTokenSecret missing'); + if ( + !pixelId || + !eventId || + !consumerKey || + !consumerSecret || + !accessToken || + !accessTokenSecret + ) { + const missing = [ + ...(pixelId ? [] : ['pixelId']), + ...(eventId ? [] : ['eventId']), + ...(consumerKey ? [] : ['consumerKey']), + ...(consumerSecret ? [] : ['consumerSecret']), + ...(accessToken ? [] : ['accessToken']), + ...(accessTokenSecret ? [] : ['accessTokenSecret']), + ]; + throw new StepError(`Config settings ${missing.join(', ')} missing`, { + code: 'config', + }); + } const settingsConfig: Settings = { ...settings, diff --git a/packages/server/destinations/twitter/src/examples/step.ts b/packages/server/destinations/twitter/src/examples/step.ts index 01bcaa6a6..88915f6d2 100644 --- a/packages/server/destinations/twitter/src/examples/step.ts +++ b/packages/server/destinations/twitter/src/examples/step.ts @@ -158,3 +158,35 @@ export const purchaseWithTwclid: Flow.StepExample = { ], ], }; + +/** A silent rule sends nothing to X. */ +export const silentRule: Flow.StepExample = { + public: false, + description: 'A rule with silent: true sends no request to X.', + in: getEvent('page view', { timestamp: 1700000990000 }), + mapping: { silent: true }, + out: [], +}; + +/** + * No user identifier -- X cannot match the event to a person, so the push + * fails with missing-input instead of counting as delivered. One per event + * name above, so every example's event without identifiers has its declared + * outcome. + */ +function withoutIdentifier(name: string, timestamp: number): Flow.StepExample { + const [entity, action] = name.split(' '); + return { + public: false, + description: `A ${name} event without a user identifier cannot be sent to X: the push fails with missing-input instead of counting as delivered.`, + in: { name, entity, action, timestamp }, + out: [], + outcome: { status: 'failed', code: 'missing-input', retryable: false }, + }; +} + +export const noIdentifierOrder = withoutIdentifier( + 'order complete', + 1700000991000, +); +export const noIdentifierForm = withoutIdentifier('form submit', 1700000992000); diff --git a/packages/server/destinations/twitter/src/index.ts b/packages/server/destinations/twitter/src/index.ts index 5a896789d..4bde616c1 100644 --- a/packages/server/destinations/twitter/src/index.ts +++ b/packages/server/destinations/twitter/src/index.ts @@ -10,8 +10,8 @@ export const destinationTwitter: Destination = { config: {}, - async init({ config: partialConfig, logger }) { - const config = getConfig(partialConfig, logger); + async init({ config: partialConfig }) { + const config = getConfig(partialConfig); return config; }, diff --git a/packages/server/destinations/twitter/src/push.ts b/packages/server/destinations/twitter/src/push.ts index f6bfd6b47..a3ac167b2 100644 --- a/packages/server/destinations/twitter/src/push.ts +++ b/packages/server/destinations/twitter/src/push.ts @@ -15,6 +15,7 @@ import { isNumber, isObject, isString, + StepError, } from '@walkeros/core'; import { sendServer, getHashServer } from '@walkeros/server-core'; import OAuth from 'oauth-1.0a'; @@ -27,7 +28,12 @@ function normalizeString(value: unknown): string | undefined { return undefined; } -export const push: PushFn = async function ( +// A silent rule has no twitter call of its own: nothing to send. +export const push: PushFn = async function (event, context) { + if (!context.rule?.silent) await send(event, context); +}; + +const send: PushFn = async function ( event, { config, rule, data, ingest, env, logger, collector }, ) { @@ -121,12 +127,15 @@ export const push: PushFn = async function ( identifiers.push({ user_agent: userAgent }); } - // Skip event if no primary identifier is present + // An event without a primary identifier cannot be matched by X const hasPrimary = identifiers.some( (id) => 'twclid' in id || 'hashed_email' in id || 'hashed_phone_number' in id, ); - if (!hasPrimary) return; + if (!hasPrimary) + throw new StepError('No primary identifier to send the event with', { + code: 'missing-input', + }); // Resolve per-event mapping settings const mappingSettings = (rule?.settings || {}) as Mapping; diff --git a/packages/server/destinations/twitter/src/schemas/settings.ts b/packages/server/destinations/twitter/src/schemas/settings.ts index fc8215120..e373059a5 100644 --- a/packages/server/destinations/twitter/src/schemas/settings.ts +++ b/packages/server/destinations/twitter/src/schemas/settings.ts @@ -6,55 +6,57 @@ const InputSchema = z.union([ z.literal(false), ]); -export const SettingsSchema = z.object({ - pixelId: z - .string() - .min(1) - .describe('X Pixel ID used in the Conversions API endpoint URL'), - eventId: EventIdSchema.describe( - 'Default pre-registered conversion event ID (like tw-xxxxx-xxxxx)', - ), - consumerKey: z - .string() - .min(1) - .describe('OAuth 1.0a API Key (Consumer Key) for X Ads API'), - consumerSecret: z - .string() - .min(1) - .describe('OAuth 1.0a API Key Secret (Consumer Secret)'), - accessToken: z.string().min(1).describe('OAuth 1.0a User Access Token'), - accessTokenSecret: z - .string() - .min(1) - .describe('OAuth 1.0a User Access Token Secret'), - apiVersion: ApiVersionSchema.describe( - 'X Ads API version number (like "12")', - ).optional(), - doNotHash: z - .array(z.string()) - .describe( - "Array of user data fields that should not be hashed (like ['email'])", - ) - .optional(), - url: z - .string() - .url() - .describe( - 'Custom base URL for the X Conversions API endpoint (like https://ads-api.x.com/)', - ) - .optional(), - user_data: z - .record(z.string(), z.string()) - .describe( - "Mapping configuration for user identifiers (like { email: 'user.email', twclid: 'context.twclid' })", - ) - .optional(), - ip: InputSchema.describe( - 'Client IP, sent as the ip_address identifier. Resolves against { ingest, event }. Default: ["ingest.ip", "event.user.ip"]. false switches it off. A mapped user_data.ip_address wins.', - ).optional(), - userAgent: InputSchema.describe( - 'Client user agent, sent as the user_agent identifier. Resolves against { ingest, event }. Default: ["ingest.userAgent", "event.user.userAgent"]. false switches it off. A mapped user_data.user_agent wins.', - ).optional(), -}); +export const SettingsSchema = z + .object({ + pixelId: z + .string() + .min(1) + .describe('X Pixel ID used in the Conversions API endpoint URL'), + eventId: EventIdSchema.describe( + 'Default pre-registered conversion event ID (like tw-xxxxx-xxxxx)', + ), + consumerKey: z + .string() + .min(1) + .describe('OAuth 1.0a API Key (Consumer Key) for X Ads API'), + consumerSecret: z + .string() + .min(1) + .describe('OAuth 1.0a API Key Secret (Consumer Secret)'), + accessToken: z.string().min(1).describe('OAuth 1.0a User Access Token'), + accessTokenSecret: z + .string() + .min(1) + .describe('OAuth 1.0a User Access Token Secret'), + apiVersion: ApiVersionSchema.describe( + 'X Ads API version number (like "12")', + ).optional(), + doNotHash: z + .array(z.string()) + .describe( + "Array of user data fields that should not be hashed (like ['email'])", + ) + .optional(), + url: z + .string() + .url() + .describe( + 'Custom base URL for the X Conversions API endpoint (like https://ads-api.x.com/)', + ) + .optional(), + user_data: z + .record(z.string(), z.string()) + .describe( + "Mapping configuration for user identifiers (like { email: 'user.email', twclid: 'context.twclid' })", + ) + .optional(), + ip: InputSchema.describe( + 'Client IP, sent as the ip_address identifier. Resolves against { ingest, event }. Default: ["ingest.ip", "event.user.ip"]. false switches it off. A mapped user_data.ip_address wins.', + ).optional(), + userAgent: InputSchema.describe( + 'Client user agent, sent as the user_agent identifier. Resolves against { ingest, event }. Default: ["ingest.userAgent", "event.user.userAgent"]. false switches it off. A mapped user_data.user_agent wins.', + ).optional(), + }) + .strict(); export type Settings = z.infer; From e51bc29642e0bbd4778918785b56a8de8b9a6e2c Mon Sep 17 00:00:00 2001 From: alexanderkirtzel Date: Wed, 21 Oct 2026 06:50:59 +0000 Subject: [PATCH 20/41] aws errors --- .../aws/src/__tests__/sdk.test.ts | 12 ++- .../firehose/__tests__/conformance.test.ts | 22 ++++ .../src/firehose/__tests__/deliver.test.ts | 5 +- .../src/firehose/__tests__/firehose.test.ts | 6 +- .../aws/src/firehose/__tests__/push.test.ts | 5 +- .../src/firehose/__tests__/pushBatch.test.ts | 9 +- .../destinations/aws/src/firehose/config.ts | 8 +- .../aws/src/firehose/examples/step.ts | 9 ++ .../aws/src/firehose/lib/deliver.ts | 25 ++--- .../aws/src/firehose/lib/records.ts | 2 +- .../destinations/aws/src/firehose/push.ts | 13 ++- .../aws/src/firehose/pushBatch.ts | 30 ++++-- .../aws/src/firehose/schemas/settings.ts | 51 ++++++--- packages/server/destinations/aws/src/hints.ts | 2 +- .../aws/src/lib/__tests__/credentials.test.ts | 4 +- .../aws/src/lib/__tests__/errors.test.ts | 38 ++++--- .../server/destinations/aws/src/lib/errors.ts | 102 +++++++++--------- .../aws/src/sns/__tests__/conformance.test.ts | 24 +++++ .../aws/src/sns/__tests__/sns.test.ts | 12 +-- .../server/destinations/aws/src/sns/config.ts | 5 +- .../destinations/aws/src/sns/examples/step.ts | 9 ++ .../destinations/aws/src/sns/lib/sns.ts | 7 +- .../server/destinations/aws/src/sns/push.ts | 9 +- .../aws/src/sns/schemas/settings.ts | 76 +++++++------ 24 files changed, 306 insertions(+), 179 deletions(-) create mode 100644 packages/server/destinations/aws/src/firehose/__tests__/conformance.test.ts create mode 100644 packages/server/destinations/aws/src/sns/__tests__/conformance.test.ts diff --git a/packages/server/destinations/aws/src/__tests__/sdk.test.ts b/packages/server/destinations/aws/src/__tests__/sdk.test.ts index ecf1be091..2262d94e7 100644 --- a/packages/server/destinations/aws/src/__tests__/sdk.test.ts +++ b/packages/server/destinations/aws/src/__tests__/sdk.test.ts @@ -10,10 +10,14 @@ import type { Socket } from 'node:net'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import type { Destination } from '@walkeros/core'; -import { createEvent, createMockContext, isObject } from '@walkeros/core'; +import { + StepError, + createEvent, + createMockContext, + isObject, +} from '@walkeros/core'; import firehose, { destinationSNS } from '../index'; import type { DestinationFirehose } from '../index'; -import { AwsDeliveryError } from '../lib/errors'; interface FakeRequest { target: string; @@ -234,7 +238,7 @@ describe('Firehose with the real SDK', () => { ).catch((e: unknown) => e); expect(requests).toHaveLength(3); - expect(error).toBeInstanceOf(AwsDeliveryError); + expect(error).toBeInstanceOf(StepError); expect(error).toMatchObject({ code: 'ServiceUnavailableException', status: 500, @@ -276,7 +280,7 @@ describe('Firehose with the real SDK', () => { ).catch((e: unknown) => e); expect(requests).toEqual([]); - expect(error).toBeInstanceOf(AwsDeliveryError); + expect(error).toBeInstanceOf(StepError); expect(String(error)).toContain('config.credentials'); }, 15_000); }); diff --git a/packages/server/destinations/aws/src/firehose/__tests__/conformance.test.ts b/packages/server/destinations/aws/src/firehose/__tests__/conformance.test.ts new file mode 100644 index 000000000..d471f1712 --- /dev/null +++ b/packages/server/destinations/aws/src/firehose/__tests__/conformance.test.ts @@ -0,0 +1,22 @@ +import path from 'path'; +import { conformance } from '@walkeros/collector/dev'; +import { destinationFirehose } from '..'; +import { schemas } from '../../dev'; +import * as examples from '../examples'; + +describe('aws firehose conformance', () => { + beforeEach(() => { + jest.useRealTimers(); + }); + + it('conforms', async () => { + await conformance('destination', destinationFirehose, { + profile: 'sdk-env', + examples: examples.step, + schema: schemas.settings, + settings: { streamName: 'walkeros-events', region: 'eu-central-1' }, + env: { push: examples.env.push, simulation: examples.env.simulation }, + root: path.join(__dirname, '..', '..', '..'), + }); + }); +}); diff --git a/packages/server/destinations/aws/src/firehose/__tests__/deliver.test.ts b/packages/server/destinations/aws/src/firehose/__tests__/deliver.test.ts index ac2f61d13..ff1928ecc 100644 --- a/packages/server/destinations/aws/src/firehose/__tests__/deliver.test.ts +++ b/packages/server/destinations/aws/src/firehose/__tests__/deliver.test.ts @@ -1,7 +1,6 @@ import { ServiceUnavailableException } from '@aws-sdk/client-firehose'; -import { createMockLogger } from '@walkeros/core'; +import { StepError, createMockLogger } from '@walkeros/core'; import { deliver } from '../lib/deliver'; -import { AwsDeliveryError } from '../../lib/errors'; import type { Answer } from './support'; import { STREAM, accepted, events, initialized, recordingEnv } from './support'; import { resolveSettings } from '../config'; @@ -122,7 +121,7 @@ describe('deliver', () => { createMockLogger(), ); - expect(failure.error).toBeInstanceOf(AwsDeliveryError); + expect(failure.error).toBeInstanceOf(StepError); expect(failure.error).toMatchObject({ code: 'ServiceUnavailableException', status: 500, diff --git a/packages/server/destinations/aws/src/firehose/__tests__/firehose.test.ts b/packages/server/destinations/aws/src/firehose/__tests__/firehose.test.ts index eea66f13a..f3f64e4db 100644 --- a/packages/server/destinations/aws/src/firehose/__tests__/firehose.test.ts +++ b/packages/server/destinations/aws/src/firehose/__tests__/firehose.test.ts @@ -1,11 +1,11 @@ import { FirehoseClient } from '@aws-sdk/client-firehose'; import { + StepError, createEvent, createMockContext, createMockLogger, } from '@walkeros/core'; import destination, { destinationFirehose } from '../'; -import { AwsDeliveryError } from '../../lib/errors'; import type { PartialConfig, SendClient } from '../types'; import { STREAM, id, initialized, isRecord, recordingEnv } from './support'; @@ -170,8 +170,8 @@ describe('settings', () => { const error = await initialized({ settings }, recordingEnv().env).catch( (e: unknown) => e, ); - expect(error).toBeInstanceOf(AwsDeliveryError); - expect(error).toMatchObject({ code: 'InvalidConfig', retryable: false }); + expect(error).toBeInstanceOf(StepError); + expect(error).toMatchObject({ code: 'config', retryable: false }); expect(String(error)).toContain('settings.streamName'); }); diff --git a/packages/server/destinations/aws/src/firehose/__tests__/push.test.ts b/packages/server/destinations/aws/src/firehose/__tests__/push.test.ts index 938924689..6e916683f 100644 --- a/packages/server/destinations/aws/src/firehose/__tests__/push.test.ts +++ b/packages/server/destinations/aws/src/firehose/__tests__/push.test.ts @@ -25,7 +25,10 @@ describe('Firehose push', () => { setTimeout(() => reject(new Error('later failure')), 5), ), ); - await expect(pushing).rejects.toThrow('later failure'); + await expect(pushing).rejects.toHaveProperty( + 'cause.message', + 'later failure', + ); }); test('does not resolve before its send resolves', async () => { diff --git a/packages/server/destinations/aws/src/firehose/__tests__/pushBatch.test.ts b/packages/server/destinations/aws/src/firehose/__tests__/pushBatch.test.ts index 3575815da..6608021e4 100644 --- a/packages/server/destinations/aws/src/firehose/__tests__/pushBatch.test.ts +++ b/packages/server/destinations/aws/src/firehose/__tests__/pushBatch.test.ts @@ -2,9 +2,8 @@ import { ResourceNotFoundException, ServiceUnavailableException, } from '@aws-sdk/client-firehose'; -import { createMockContext, createMockLogger } from '@walkeros/core'; +import { StepError, createMockContext, createMockLogger } from '@walkeros/core'; import destination from '../'; -import { AwsDeliveryError } from '../../lib/errors'; import type { Answer } from './support'; import { STREAM, @@ -59,8 +58,8 @@ describe('Firehose pushBatch outcomes', () => { const outcome = await result; expect(outcome).toEqual({ failed: [ - { index: 502, error: expect.any(AwsDeliveryError) }, - { index: 507, error: expect.any(AwsDeliveryError) }, + { index: 502, error: expect.any(StepError) }, + { index: 507, error: expect.any(StepError) }, ], }); }); @@ -88,7 +87,7 @@ describe('Firehose pushBatch outcomes', () => { () => undefined, (e: unknown) => e, ); - expect(error).toBeInstanceOf(AwsDeliveryError); + expect(error).toBeInstanceOf(StepError); expect(error).toMatchObject({ code: 'ServiceUnavailableException', retryable: false, diff --git a/packages/server/destinations/aws/src/firehose/config.ts b/packages/server/destinations/aws/src/firehose/config.ts index 63c6b53c0..0e2830c61 100644 --- a/packages/server/destinations/aws/src/firehose/config.ts +++ b/packages/server/destinations/aws/src/firehose/config.ts @@ -1,6 +1,6 @@ import type { Logger } from '@walkeros/core'; import { FirehoseClient } from '@aws-sdk/client-firehose'; -import { isObject } from '@walkeros/core'; +import { StepError, isObject } from '@walkeros/core'; import type { Config, CredentialSource, @@ -80,7 +80,7 @@ export async function getConfig( if (!streamName) throw configError( - 'Firehose: settings.streamName is missing. Set it to the name of your Firehose stream.', + 'Firehose: settings.streamName is missing. Set it to the name of your Firehose stream (settings.firehose.streamName is the deprecated form).', ); if (!STREAM_NAME_PATTERN.test(streamName)) throw configError( @@ -140,7 +140,9 @@ export async function getConfig( export function resolveSettings(settings: InitSettings | undefined): Settings { const { streamName, region, client, runtime } = settings ?? {}; if (!streamName || !region || !client || !runtime) - throw configError('Firehose: the destination was not initialized.'); + throw new StepError('Firehose: the destination was not initialized.', { + code: 'not-ready', + }); return { streamName, region, diff --git a/packages/server/destinations/aws/src/firehose/examples/step.ts b/packages/server/destinations/aws/src/firehose/examples/step.ts index ba6ea64fc..1ebcda897 100644 --- a/packages/server/destinations/aws/src/firehose/examples/step.ts +++ b/packages/server/destinations/aws/src/firehose/examples/step.ts @@ -127,3 +127,12 @@ export const mappedRecord: Flow.StepExample = { total: 99.99, }), }; + +/** A silent rule sends no record to Firehose. */ +export const silentRule: Flow.StepExample = { + public: false, + description: 'A rule with silent: true sends no record to Firehose.', + in: pageEvent, + mapping: { silent: true }, + out: [], +}; diff --git a/packages/server/destinations/aws/src/firehose/lib/deliver.ts b/packages/server/destinations/aws/src/firehose/lib/deliver.ts index ba782df8e..e8e2707cc 100644 --- a/packages/server/destinations/aws/src/firehose/lib/deliver.ts +++ b/packages/server/destinations/aws/src/firehose/lib/deliver.ts @@ -1,13 +1,9 @@ import type { Destination, Logger, WalkerOS } from '@walkeros/core'; +import { StepError } from '@walkeros/core'; import { PutRecordBatchCommand } from '@aws-sdk/client-firehose'; import type { Env, Settings } from '../types'; import type { DeliveryTarget } from '../../lib/errors'; -import { - AwsDeliveryError, - fromSdkError, - localError, - recordError, -} from '../../lib/errors'; +import { fromSdkError, localError, recordError } from '../../lib/errors'; import type { PlannedRecord } from './records'; import { planChunks, toPayload } from './records'; @@ -20,7 +16,7 @@ export interface DeliveryItem { /** A record that did not land, by its index in the delivery. */ export interface DeliveryFailure { index: number; - error: AwsDeliveryError; + error: StepError; } /** The per-record answer of a `PutRecordBatch` call, as far as it is read. */ @@ -99,12 +95,7 @@ function chunkFailures( if (!entry.errorCode) return; failures.push({ index: chunk[local].index, - error: recordError( - target, - entry.errorCode, - entry.errorMessage ?? 'Record rejected.', - status, - ), + error: recordError(target, entry.errorCode, status, entry), }); }); // Entries that do not account for the reported count cannot place the @@ -205,13 +196,11 @@ export async function deliver( * mixed causes give the most frequent code, retryable only if every record * was, with the counts per code in the message. */ -export function nothingLanded( - failures: readonly DeliveryFailure[], -): AwsDeliveryError { +export function nothingLanded(failures: readonly DeliveryFailure[]): StepError { const first = failures[0].error; if (failures.every(({ error }) => error === first)) return first; - const byCode = new Map(); + const byCode = new Map(); for (const { error } of failures) { const entry = byCode.get(error.code); if (entry) entry.count += 1; @@ -226,7 +215,7 @@ export function nothingLanded( .map(([code, { count }]) => `${count} ${code}`) .join(', '); - return new AwsDeliveryError( + return new StepError( `${dominant.sample.message} (all ${failures.length} records failed: ${counts})`, { code: dominant.sample.code, diff --git a/packages/server/destinations/aws/src/firehose/lib/records.ts b/packages/server/destinations/aws/src/firehose/lib/records.ts index 9a987a1a9..68d5b72f8 100644 --- a/packages/server/destinations/aws/src/firehose/lib/records.ts +++ b/packages/server/destinations/aws/src/firehose/lib/records.ts @@ -62,7 +62,7 @@ export function planChunks( refused.push({ index, code: 'InvalidRecord', - detail: `Record ${index} does not serialize to JSON: ${error instanceof Error ? error.message : String(error)}`, + detail: `Record ${index} does not serialize to JSON.`, cause: error, }); return; diff --git a/packages/server/destinations/aws/src/firehose/push.ts b/packages/server/destinations/aws/src/firehose/push.ts index 3ddfb82d5..72f1f9685 100644 --- a/packages/server/destinations/aws/src/firehose/push.ts +++ b/packages/server/destinations/aws/src/firehose/push.ts @@ -2,11 +2,16 @@ import type { PushFn } from './types'; import { resolveSettings } from './config'; import { deliver } from './lib/deliver'; -/** Sends one event as one record. Resolves once AWS confirmed it, else throws. */ +/** + * Sends one event as one record. Resolves once AWS confirmed it, else throws. + * A silent rule has no record of its own: nothing to send. + */ export const push: PushFn = async function (event, context) { - const { config, data, env, logger } = context; + const { config, data, env, logger, rule } = context; const settings = resolveSettings(config.settings); - const [failure] = await deliver([{ event, data }], settings, env, logger); - if (failure) throw failure.error; + if (!rule?.silent) { + const [failure] = await deliver([{ event, data }], settings, env, logger); + if (failure) throw failure.error; + } }; diff --git a/packages/server/destinations/aws/src/firehose/pushBatch.ts b/packages/server/destinations/aws/src/firehose/pushBatch.ts index 8139180d4..4db7b5c8f 100644 --- a/packages/server/destinations/aws/src/firehose/pushBatch.ts +++ b/packages/server/destinations/aws/src/firehose/pushBatch.ts @@ -4,19 +4,31 @@ import { deliver, toOutcome } from './lib/deliver'; /** * Sends a flushed batch. Resolves when every record landed, returns the - * failed indices when some did, and throws when none did. + * failed indices when some did, and throws when none did. Silent rules have + * no record of their own: they are not sent and count as delivered. */ export const pushBatch: PushBatchFn = async (batch, context) => { const { config, env, logger } = context; - if (batch.entries.length === 0) return; const settings = resolveSettings(config.settings); - const failures = await deliver( - batch.entries.map(({ event, data }) => ({ event, data })), - settings, - env, - logger, - ); + // Each record keeps its index in the flushed batch for the outcome. + const sending = batch.entries + .map((entry, index) => ({ entry, index })) + .filter(({ entry }) => !entry.rule?.silent); + const failures = sending.length + ? await deliver( + sending.map(({ entry: { event, data } }) => ({ event, data })), + settings, + env, + logger, + ) + : []; - return toOutcome(batch.entries.length, failures); + return toOutcome( + batch.entries.length, + failures.map(({ index, error }) => ({ + index: sending[index].index, + error, + })), + ); }; diff --git a/packages/server/destinations/aws/src/firehose/schemas/settings.ts b/packages/server/destinations/aws/src/firehose/schemas/settings.ts index 0094ad02d..eef0d3de5 100644 --- a/packages/server/destinations/aws/src/firehose/schemas/settings.ts +++ b/packages/server/destinations/aws/src/firehose/schemas/settings.ts @@ -32,22 +32,39 @@ export const FirehoseConfigSchema = z.object({ config: config.optional(), }); -export const SettingsSchema = z.object({ - // Optional in the schema only so the deprecated alias still validates; - // init throws when neither form names a stream. - streamName, - region: region.optional(), - newline: z - .boolean() - .describe( - 'Append a newline to every record (default true). Set false when the stream adds its own new line delimiter.', - ) - .optional(), - config: config.optional(), - client: client.optional(), - firehose: FirehoseConfigSchema.describe( - 'Deprecated: use the flat fields streamName, region, config and client.', - ).optional(), -}); +export const SettingsSchema = z + .object({ + // Optional in the schema only so the deprecated alias still validates; + // init throws when neither form names a stream. + streamName, + region: region.optional(), + newline: z + .boolean() + .describe( + 'Append a newline to every record (default true). Set false when the stream adds its own new line delimiter.', + ) + .optional(), + config: config.optional(), + client: client.optional(), + firehose: FirehoseConfigSchema.describe( + 'Deprecated: use the flat fields streamName, region, config and client.', + ).optional(), + }) + .refine( + (settings) => + settings.streamName !== undefined || + settings.firehose?.streamName !== undefined, + { message: 'Set settings.streamName' }, + ) + // The same rule in JSON Schema: the stream is named in one of both forms. + .meta({ + anyOf: [ + { required: ['streamName'] }, + { + required: ['firehose'], + properties: { firehose: { required: ['streamName'] } }, + }, + ], + }); export type Settings = z.infer; diff --git a/packages/server/destinations/aws/src/hints.ts b/packages/server/destinations/aws/src/hints.ts index 3c256e422..46defd9a9 100644 --- a/packages/server/destinations/aws/src/hints.ts +++ b/packages/server/destinations/aws/src/hints.ts @@ -30,6 +30,6 @@ export const hints: Hint.Hints = { text: 'Delivery is at least once: deduplicate downstream on event.id, and keep id in mapped data. A record AWS rejects is reported as failed with code, status and retryable. There is no retry beyond the built-in AWS SDK retry: a rejected record goes to the in-memory dead letter queue of the collector, which nothing replays yet. On SNS FIFO topics MessageDeduplicationId defaults to event.id.', }, 'troubleshoot-errors': { - text: 'Every failure carries code, status and retryable, and its message names the stream or topic and the region. ResourceNotFoundException: wrong settings.streamName or region. AccessDeniedException / AuthorizationErrorException: the credentials lack firehose:PutRecordBatch or sns:Publish. UnrecognizedClientException, InvalidSignatureException, ExpiredTokenException or no credentials found: check config.credentials or the environment. SNS NotFoundException: the topic does not exist; run walkeros setup or set settings.topicArn. NULL rows in Athena: the stream adds its own new line delimiter too, so set newline: false. InvalidConfig at start: a missing or conflicting setting, named in the message.', + text: 'Every failure carries code, status and retryable, and its message names the stream or topic and the region; the AWS error itself stays on the error as its cause. ResourceNotFoundException: wrong settings.streamName or region. AccessDeniedException / AuthorizationErrorException: the credentials lack firehose:PutRecordBatch or sns:Publish. UnrecognizedClientException, InvalidSignatureException, ExpiredTokenException or no credentials found: check config.credentials or the environment. SNS NotFoundException: the topic does not exist; run walkeros setup or set settings.topicArn. NULL rows in Athena: the stream adds its own new line delimiter too, so set newline: false. Code config at start: a missing or conflicting setting, named in the message.', }, }; diff --git a/packages/server/destinations/aws/src/lib/__tests__/credentials.test.ts b/packages/server/destinations/aws/src/lib/__tests__/credentials.test.ts index 33241bd51..b15df5d14 100644 --- a/packages/server/destinations/aws/src/lib/__tests__/credentials.test.ts +++ b/packages/server/destinations/aws/src/lib/__tests__/credentials.test.ts @@ -1,4 +1,4 @@ -import { AwsDeliveryError } from '../errors'; +import { StepError } from '@walkeros/core'; import { parseCredentials } from '../credentials'; const SECRET = 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYSECRETVALUE'; @@ -8,7 +8,7 @@ function messageOf(fn: () => unknown): string { try { fn(); } catch (error) { - expect(error).toBeInstanceOf(AwsDeliveryError); + expect(error).toBeInstanceOf(StepError); return error instanceof Error ? error.message : String(error); } throw new Error('expected a throw'); diff --git a/packages/server/destinations/aws/src/lib/__tests__/errors.test.ts b/packages/server/destinations/aws/src/lib/__tests__/errors.test.ts index 878886001..42f403fe9 100644 --- a/packages/server/destinations/aws/src/lib/__tests__/errors.test.ts +++ b/packages/server/destinations/aws/src/lib/__tests__/errors.test.ts @@ -6,14 +6,9 @@ import { } from '@aws-sdk/client-firehose'; import { NotFoundException, ThrottledException } from '@aws-sdk/client-sns'; import { CredentialsProviderError } from '@smithy/core/config'; +import { StepError } from '@walkeros/core'; import type { DeliveryTarget } from '../errors'; -import { - AwsDeliveryError, - configError, - fromSdkError, - localError, - recordError, -} from '../errors'; +import { configError, fromSdkError, localError, recordError } from '../errors'; const firehose: DeliveryTarget = { service: 'Firehose', @@ -66,6 +61,11 @@ describe('fromSdkError', () => { namedError('Error', { code: 'ECONNRESET' }), { code: 'ECONNRESET', status: undefined, retryable: true }, ], + [ + 'an error that never reached AWS', + new Error('Connection refused'), + { code: 'Error', status: undefined, retryable: true }, + ], [ 'a 400 ResourceNotFoundException', new ResourceNotFoundException({ @@ -84,7 +84,7 @@ describe('fromSdkError', () => { ])('classifies %s with the SDK classifier', (_name, sdkError, expected) => { const error = fromSdkError(sdkError, firehose); - expect(error).toBeInstanceOf(AwsDeliveryError); + expect(error).toBeInstanceOf(StepError); expect({ code: error.code, status: error.status, @@ -94,9 +94,17 @@ describe('fromSdkError', () => { expect(error.message).toContain('walkeros-events'); expect(error.message).toContain('eu-west-1'); expect(error.message).toContain(expected.code); + expect(error.message).not.toContain(sdkError.message); expect(error).not.toHaveProperty('maybeDelivered'); }); + it('keeps the text of anything thrown out of the message', () => { + const error = fromSdkError('user@example.com', firehose); + expect(error).toMatchObject({ code: 'UnknownError', retryable: false }); + expect(error.message).not.toContain('user@example.com'); + expect(error.cause).toBe('user@example.com'); + }); + it.each([ ['ResourceNotFoundException', firehose, 'settings.streamName'], ['ResourceNotFoundException', sns, 'settings.topicArn'], @@ -118,7 +126,7 @@ describe('fromSdkError', () => { ])('passes %s through without a fix', (name, target) => { const error = fromSdkError(namedError(name), target); expect(error.message).toBe( - `${target.service} ${target.service === 'SNS' ? 'topic' : 'stream'} "${target.resource}" (${target.region}): ${name}: ${name} happened`, + `${target.service} ${target.service === 'SNS' ? 'topic' : 'stream'} "${target.resource}" (${target.region}): ${name}`, ); }); @@ -132,7 +140,7 @@ describe('fromSdkError', () => { expect(error.cause).toBe(sdkError); }); - it('returns an AwsDeliveryError unchanged', () => { + it('returns a StepError unchanged', () => { const error = configError('bad'); expect(fromSdkError(error, firehose)).toBe(error); }); @@ -140,17 +148,23 @@ describe('fromSdkError', () => { describe('package errors', () => { it('marks a record rejected inside a 200 as retryable', () => { + const answer = { + errorCode: 'ServiceUnavailableException', + errorMessage: 'Slow down.', + }; const error = recordError( firehose, 'ServiceUnavailableException', - 'Slow down.', 200, + answer, ); expect(error).toMatchObject({ code: 'ServiceUnavailableException', status: 200, retryable: true, + body: answer, }); + expect(error.message).not.toContain('Slow down.'); }); it.each([ @@ -165,7 +179,7 @@ describe('package errors', () => { it('configuration errors are not retryable', () => { expect(configError('fix it')).toMatchObject({ - code: 'InvalidConfig', + code: 'config', retryable: false, message: 'fix it', }); diff --git a/packages/server/destinations/aws/src/lib/errors.ts b/packages/server/destinations/aws/src/lib/errors.ts index 3bd9dff81..9a796c35f 100644 --- a/packages/server/destinations/aws/src/lib/errors.ts +++ b/packages/server/destinations/aws/src/lib/errors.ts @@ -3,6 +3,7 @@ import { isThrottlingError, isTransientError, } from '@smithy/core/retry'; +import { StepError } from '@walkeros/core'; /** The AWS service a delivery goes to, as it appears in every message. */ export type AwsService = 'Firehose' | 'SNS'; @@ -21,33 +22,14 @@ export interface DeliveryTarget { id?: string; } -export interface AwsDeliveryErrorOptions { - code: string; - status?: number; - retryable: boolean; - cause?: unknown; -} - -/** - * Every failure this package reports or throws. `code` is the AWS error name - * or one of the package's own codes (`RecordTooLarge`, `InvalidRecord`, - * `InvalidResponse`, `InvalidConfig`), `status` the HTTP status when AWS - * answered, and `retryable` whether sending the same record again can succeed. - * The SDK error stays reachable as `cause`. +/* + * Every failure this package throws is a `StepError`. `code` is the AWS error + * name or one of the package's own codes (`RecordTooLarge`, `InvalidRecord`, + * `InvalidResponse`, `config`), `status` the HTTP status when AWS answered, + * and `retryable` whether sending the same record again can succeed. The + * message is the package's own: AWS's text can quote the request, so it stays + * on the SDK error, kept as `cause`, or on a rejected record's `body`. */ -export class AwsDeliveryError extends Error { - readonly code: string; - readonly status: number | undefined; - readonly retryable: boolean; - - constructor(message: string, options: AwsDeliveryErrorOptions) { - super(message, { cause: options.cause }); - this.name = 'AwsDeliveryError'; - this.code = options.code; - this.status = options.status; - this.retryable = options.retryable; - } -} /** Codes whose fix is a configuration change, each with that change. */ function hintFor(code: string, target: DeliveryTarget): string | undefined { @@ -80,14 +62,16 @@ function describe(target: DeliveryTarget): string { return `${target.service} ${kind} "${target.resource}" (${target.region})`; } -/** Builds the message: target, code, the AWS text and, if listed, the fix. */ +/** Builds the message: target, code, the package's detail and, if listed, the fix. */ export function formatMessage( target: DeliveryTarget, code: string, - text: string, + detail?: string, ): string { const hint = hintFor(code, target); - const base = `${describe(target)}: ${code}: ${text}`; + const base = detail + ? `${describe(target)}: ${code}: ${detail}` + : `${describe(target)}: ${code}`; return hint ? `${base} ${hint}` : base; } @@ -106,31 +90,48 @@ function readCode(error: SdkError): string { return typeof code === 'string' && code.length > 0 ? code : error.name; } +/** + * Errors of the providers that resolve credentials before a request goes + * out: a setup problem, never fixed by sending again. + */ +const PROVIDER_ERRORS = [ + 'ProviderError', + 'CredentialsProviderError', + 'TokenProviderError', +]; + /** * Wraps an error a whole call threw. `retryable` is the SDK's own public * classifier, the one its standard retry strategy uses, so this package keeps - * no table of its own. + * no table of its own. An error it does not know that carries no AWS answer + * never reached AWS (a refused connection) and is retryable like any + * transport failure, unless no credentials could be found. */ export function fromSdkError( error: unknown, target: DeliveryTarget, -): AwsDeliveryError { - if (error instanceof AwsDeliveryError) return error; +): StepError { + if (error instanceof StepError) return error; if (!isSdkError(error)) { - return new AwsDeliveryError( - formatMessage(target, 'UnknownError', String(error)), - { code: 'UnknownError', retryable: false, cause: error }, - ); + return new StepError(formatMessage(target, 'UnknownError'), { + code: 'UnknownError', + retryable: false, + cause: error, + }); } const code = readCode(error); + const status = error.$metadata?.httpStatusCode; const retryable = - isThrottlingError(error) || isTransientError(error) || isServerError(error); + isThrottlingError(error) || + isTransientError(error) || + isServerError(error) || + (status === undefined && !PROVIDER_ERRORS.includes(code)); - return new AwsDeliveryError(formatMessage(target, code, error.message), { + return new StepError(formatMessage(target, code), { code, - status: error.$metadata?.httpStatusCode, + status, retryable, cause: error, }); @@ -139,18 +140,20 @@ export function fromSdkError( /** * A record AWS rejected inside an accepted call. Both per-record codes AWS * documents for PutRecordBatch (`ServiceUnavailableException`, - * `InternalFailure`) are retryable, so every such rejection is. + * `InternalFailure`) are retryable, so every such rejection is. AWS's answer + * for the record stays on the error as `body`. */ export function recordError( target: DeliveryTarget, code: string, - text: string, status?: number, -): AwsDeliveryError { - return new AwsDeliveryError(formatMessage(target, code, text), { + body?: unknown, +): StepError { + return new StepError(formatMessage(target, code), { code, status, retryable: true, + body, }); } @@ -161,10 +164,10 @@ export function recordError( export function localError( target: DeliveryTarget, code: 'RecordTooLarge' | 'InvalidRecord' | 'InvalidResponse', - text: string, + detail: string, cause?: unknown, -): AwsDeliveryError { - return new AwsDeliveryError(formatMessage(target, code, text), { +): StepError { + return new StepError(formatMessage(target, code, detail), { code, retryable: code === 'InvalidResponse', cause, @@ -172,9 +175,6 @@ export function localError( } /** A configuration error found offline in `init` or `setup`. */ -export function configError(message: string): AwsDeliveryError { - return new AwsDeliveryError(message, { - code: 'InvalidConfig', - retryable: false, - }); +export function configError(message: string): StepError { + return new StepError(message, { code: 'config' }); } diff --git a/packages/server/destinations/aws/src/sns/__tests__/conformance.test.ts b/packages/server/destinations/aws/src/sns/__tests__/conformance.test.ts new file mode 100644 index 000000000..84e89797b --- /dev/null +++ b/packages/server/destinations/aws/src/sns/__tests__/conformance.test.ts @@ -0,0 +1,24 @@ +import path from 'path'; +import { conformance } from '@walkeros/collector/dev'; +import { destinationSNS } from '..'; +import { schemas } from '../../dev'; +import * as examples from '../examples'; + +describe('aws sns conformance', () => { + beforeEach(() => { + jest.useRealTimers(); + }); + + it('conforms', async () => { + await conformance('destination', destinationSNS, { + profile: 'sdk-env', + examples: examples.step, + schema: schemas.snsSettings, + settings: { + topicArn: 'arn:aws:sns:eu-central-1:123456789012:walkeros-events', + }, + env: { push: examples.env.push, simulation: examples.env.simulation }, + root: path.join(__dirname, '..', '..', '..'), + }); + }); +}); diff --git a/packages/server/destinations/aws/src/sns/__tests__/sns.test.ts b/packages/server/destinations/aws/src/sns/__tests__/sns.test.ts index 37ef63e67..d883f037e 100644 --- a/packages/server/destinations/aws/src/sns/__tests__/sns.test.ts +++ b/packages/server/destinations/aws/src/sns/__tests__/sns.test.ts @@ -19,6 +19,7 @@ import { __setStsHarness, } from '@aws-sdk/client-sts'; import { + StepError, createEvent, createMockContext, createMockLogger, @@ -26,7 +27,6 @@ import { import type { MockLogger } from '@walkeros/core'; import type { Config, Env, PartialConfig, SendClient } from '../types'; import { expectSimulationResolves } from '@walkeros/core/dev'; -import { AwsDeliveryError } from '../../lib/errors'; import destination from '../'; import * as snsExamples from '../examples'; @@ -155,8 +155,8 @@ describe('SNS init', () => { ['neither ARN nor name', {}, 'settings.topicArn'], ])('%s throws', async (_case, settings, fragment) => { const error = await init({ settings }).catch((e: unknown) => e); - expect(error).toBeInstanceOf(AwsDeliveryError); - expect(error).toMatchObject({ code: 'InvalidConfig' }); + expect(error).toBeInstanceOf(StepError); + expect(error).toMatchObject({ code: 'config' }); expect(String(error)).toContain(fragment); }); @@ -295,14 +295,14 @@ describe('SNS push', () => { __setStsHarness({ error: new Error('network down') }); const config = await init({ settings: { topicName: 'walkeros-events' } }); - await expect(publish(config)).rejects.toBeInstanceOf(AwsDeliveryError); + await expect(publish(config)).rejects.toBeInstanceOf(StepError); await publish(config); expect(publishInput()).toMatchObject({ TopicArn: 'arn:aws:sns:eu-central-1:000000000000:walkeros-events', }); }); - test('a publish failure throws an AwsDeliveryError and logs no error', async () => { + test('a publish failure throws an StepError and logs no error', async () => { const logger = createMockLogger(); const config = await init({ settings: { topicArn: ARN } }); __setHarness({ @@ -317,7 +317,7 @@ describe('SNS push', () => { (e: unknown) => e, ); - expect(error).toBeInstanceOf(AwsDeliveryError); + expect(error).toBeInstanceOf(StepError); expect(error).toMatchObject({ code: 'NotFoundException', status: 404, diff --git a/packages/server/destinations/aws/src/sns/config.ts b/packages/server/destinations/aws/src/sns/config.ts index 0e4d14dd7..c71e38fa8 100644 --- a/packages/server/destinations/aws/src/sns/config.ts +++ b/packages/server/destinations/aws/src/sns/config.ts @@ -1,4 +1,5 @@ import type { Logger } from '@walkeros/core'; +import { StepError } from '@walkeros/core'; import type { Config, Env, @@ -155,7 +156,9 @@ function topicArnResolver( export function resolveSettings(settings: InitSettings | undefined): Settings { const { region, client, runtime } = settings ?? {}; if (!region || !client || !runtime) - throw configError('SNS: the destination was not initialized.'); + throw new StepError('SNS: the destination was not initialized.', { + code: 'not-ready', + }); return { topicArn: settings?.topicArn, topicName: settings?.topicName, diff --git a/packages/server/destinations/aws/src/sns/examples/step.ts b/packages/server/destinations/aws/src/sns/examples/step.ts index 8906f6ab1..6e80f043c 100644 --- a/packages/server/destinations/aws/src/sns/examples/step.ts +++ b/packages/server/destinations/aws/src/sns/examples/step.ts @@ -158,3 +158,12 @@ export const mappedMessage: Flow.StepExample = { ], ], }; + +/** A silent rule publishes no message to SNS. */ +export const silentRule: Flow.StepExample = { + public: false, + description: 'A rule with silent: true publishes no message to SNS.', + in: pageEvent, + mapping: { silent: true }, + out: [], +}; diff --git a/packages/server/destinations/aws/src/sns/lib/sns.ts b/packages/server/destinations/aws/src/sns/lib/sns.ts index 75fb2b54f..a4f8fa4f7 100644 --- a/packages/server/destinations/aws/src/sns/lib/sns.ts +++ b/packages/server/destinations/aws/src/sns/lib/sns.ts @@ -8,7 +8,7 @@ import type { SendClient } from '../../lib/client'; import type { DeliveryTarget } from '../../lib/errors'; import type { Env } from '../types'; import { isHandlerInstance, requestHandlerOptions } from '../../lib/client'; -import { fromSdkError } from '../../lib/errors'; +import { fromSdkError, localError } from '../../lib/errors'; export interface ClientOptions { region: string; @@ -94,9 +94,10 @@ export async function getCallerIdentity( } if (!isObject(response) || typeof response.Account !== 'string') - throw fromSdkError( - new Error('STS GetCallerIdentity returned no Account.'), + throw localError( target, + 'InvalidResponse', + 'STS GetCallerIdentity returned no Account.', ); // The caller ARN carries the partition (aws, aws-cn, aws-us-gov). diff --git a/packages/server/destinations/aws/src/sns/push.ts b/packages/server/destinations/aws/src/sns/push.ts index e41e61383..d0043311b 100644 --- a/packages/server/destinations/aws/src/sns/push.ts +++ b/packages/server/destinations/aws/src/sns/push.ts @@ -59,9 +59,14 @@ async function resolveStringMappingValue( /** * Publishes one event. Resolves once SNS accepted the message, else throws an - * `AwsDeliveryError`; the collector logs it, so SNS logs no error of its own. + * `StepError`; the collector logs it, so SNS logs no error of its own. */ export const push: PushFn = async function (event, context) { + // A silent rule has no message of its own: nothing to publish. + if (!context.rule?.silent) await publish(event, context); +}; + +const publish: PushFn = async function (event, context) { const { config, data, env, rule, collector, logger, id } = context; const settings = resolveSettings(config.settings); const { runtime } = settings; @@ -113,7 +118,7 @@ export const push: PushFn = async function (event, context) { throw localError( target, 'InvalidRecord', - `The message does not serialize to JSON: ${error instanceof Error ? error.message : String(error)}`, + 'The message does not serialize to JSON.', error, ); } diff --git a/packages/server/destinations/aws/src/sns/schemas/settings.ts b/packages/server/destinations/aws/src/sns/schemas/settings.ts index b51e20fff..50bad8c06 100644 --- a/packages/server/destinations/aws/src/sns/schemas/settings.ts +++ b/packages/server/destinations/aws/src/sns/schemas/settings.ts @@ -1,37 +1,47 @@ import { z } from '@walkeros/core/dev'; -export const SettingsSchema = z.object({ - topicArn: z - .string() - .describe( - 'Topic ARN (like arn:aws:sns:eu-central-1:123456789012:walkeros-events). Region and topic name derive from it. Required unless topicName is set.', - ) - .optional(), - topicName: z - .string() - .min(1) - .describe( - 'Topic name (like walkeros-events). Used by walkeros setup; without topicArn the ARN is completed with the account id at the first publish.', - ) - .optional(), - region: z - .string() - .describe( - 'AWS region (like eu-central-1). Default: the region in topicArn, AWS_REGION or the active profile, else eu-central-1.', - ) - .optional(), - client: z - .any() - .describe( - 'An SNSClient of your own. Used as is and never closed by the destination.', - ) - .optional(), - config: z - .any() - .describe( - 'Raw SNSClient options passed to the SDK. Put keys in config.credentials instead.', - ) - .optional(), -}); +export const SettingsSchema = z + .object({ + topicArn: z + .string() + .describe( + 'Topic ARN (like arn:aws:sns:eu-central-1:123456789012:walkeros-events). Region and topic name derive from it. Required unless topicName is set.', + ) + .optional(), + topicName: z + .string() + .min(1) + .describe( + 'Topic name (like walkeros-events). Used by walkeros setup; without topicArn the ARN is completed with the account id at the first publish.', + ) + .optional(), + region: z + .string() + .describe( + 'AWS region (like eu-central-1). Default: the region in topicArn, AWS_REGION or the active profile, else eu-central-1.', + ) + .optional(), + client: z + .any() + .describe( + 'An SNSClient of your own. Used as is and never closed by the destination.', + ) + .optional(), + config: z + .any() + .describe( + 'Raw SNSClient options passed to the SDK. Put keys in config.credentials instead.', + ) + .optional(), + }) + .refine( + (settings) => + settings.topicArn !== undefined || settings.topicName !== undefined, + { message: 'Set settings.topicArn or settings.topicName' }, + ) + // The same rule in JSON Schema: one of both names the topic. + .meta({ + anyOf: [{ required: ['topicArn'] }, { required: ['topicName'] }], + }); export type Settings = z.infer; From ceb3682d7fc2215d6fe9be5a786a6c48872ac3e9 Mon Sep 17 00:00:00 2001 From: alexanderkirtzel Date: Wed, 21 Oct 2026 07:00:51 +0000 Subject: [PATCH 21/41] guards flip --- packages/collector/src/__tests__/init-contract.test.ts | 1 + .../src/__tests__/transformer-init-error.test.ts | 1 + packages/collector/src/destination.ts | 3 ++- packages/collector/src/outcome.ts | 8 ++++++++ packages/collector/src/transformer.ts | 4 ++-- packages/config/eslint/index.mjs | 6 +++--- packages/core/src/types/destination.ts | 2 +- packages/core/src/types/store.ts | 2 +- packages/core/src/types/transformer.ts | 4 ++-- 9 files changed, 21 insertions(+), 10 deletions(-) diff --git a/packages/collector/src/__tests__/init-contract.test.ts b/packages/collector/src/__tests__/init-contract.test.ts index a68d3a76a..6366557c2 100644 --- a/packages/collector/src/__tests__/init-contract.test.ts +++ b/packages/collector/src/__tests__/init-contract.test.ts @@ -29,6 +29,7 @@ describe('init contract', () => { const legacy: Destination.Instance = { type: 'legacy', config: {}, + // @ts-expect-error init may no longer return false: the type rejects it, the runtime guard reports it init: initFalse, push: jest.fn(), }; diff --git a/packages/collector/src/__tests__/transformer-init-error.test.ts b/packages/collector/src/__tests__/transformer-init-error.test.ts index 5c100001f..7d74048b2 100644 --- a/packages/collector/src/__tests__/transformer-init-error.test.ts +++ b/packages/collector/src/__tests__/transformer-init-error.test.ts @@ -149,6 +149,7 @@ describe('transformer init contract (C7)', () => { const enrich: Transformer.Instance = { type: 'enrich', config: {}, + // @ts-expect-error init may no longer return false: the type rejects it, the runtime guard reports it init: initFalse, push: (event) => ({ event }), }; diff --git a/packages/collector/src/destination.ts b/packages/collector/src/destination.ts index 7201b539b..1f8c2ff3b 100644 --- a/packages/collector/src/destination.ts +++ b/packages/collector/src/destination.ts @@ -48,6 +48,7 @@ import { isTransportFailure, recordFailure, recordSkip, + returnedFalse, } from './outcome'; import { BATCH_WAIT, @@ -1597,7 +1598,7 @@ async function runDestinationInit( // Runtime guard: bundled code is not typechecked, so an init may still // return false. Without this, `{ ...(false || config), init: true }` // would install a destination with its settings missing. - if (configResult === false) throw initReturnedFalse(); + if (returnedFalse(configResult)) throw initReturnedFalse(); // The config to install, marked initialized; the caller installs it. return { diff --git a/packages/collector/src/outcome.ts b/packages/collector/src/outcome.ts index 5e6b18fa8..888fce021 100644 --- a/packages/collector/src/outcome.ts +++ b/packages/collector/src/outcome.ts @@ -38,6 +38,14 @@ export function isTransportFailure(reason: FailureReason): boolean { return reason.retryable && !GATE_CODES.includes(reason.code); } +/** + * Whether an `init` returned `false`. Its type forbids it, but bundled code is + * not typechecked, so the guard reads the value as it arrives. + */ +export function returnedFalse(result: unknown): boolean { + return result === false; +} + /** The error of an `init` that returned `false` (bundled code is not typechecked). */ export function initReturnedFalse(): StepError { return new StepError( diff --git a/packages/collector/src/transformer.ts b/packages/collector/src/transformer.ts index 8a128c480..b857ff03c 100644 --- a/packages/collector/src/transformer.ts +++ b/packages/collector/src/transformer.ts @@ -67,7 +67,7 @@ import { journeyFields, observerStepError, stepState } from './observerEmit'; import { getCacheStore } from './cache'; import { buildReportError, errorMeta } from './report-error'; import { compileStepState, runStepState, warnStateOff } from './state'; -import { asStepError, initReturnedFalse } from './outcome'; +import { asStepError, initReturnedFalse, returnedFalse } from './outcome'; import { resolveDestinationTimeout, withTimeout } from './timeout'; /** @@ -335,7 +335,7 @@ export async function transformerInit( ); // Check for initialization failure - if (configResult === false) return false; + if (returnedFalse(configResult)) return false; // Update config if returned, preserving env from definition transformer.config = { diff --git a/packages/config/eslint/index.mjs b/packages/config/eslint/index.mjs index 4016c9b86..e4272af7a 100644 --- a/packages/config/eslint/index.mjs +++ b/packages/config/eslint/index.mjs @@ -42,12 +42,12 @@ export default [ 'jest/no-disabled-tests': 'warn', 'jest/no-focused-tests': 'error', '@typescript-eslint/no-explicit-any': 'error', - 'walkeros/await-send': 'warn', + 'walkeros/await-send': 'error', 'no-restricted-syntax': [ 'error', { selector: - "ExportAllDeclaration[source.value=/(^|\\/)(examples|schemas)$/]", + 'ExportAllDeclaration[source.value=/(^|\\/)(examples|schemas)$/]', message: 'Examples and schemas must be exported only from src/dev.ts (the ./dev subpath); a production-entry export leaks them into bundled output.', }, @@ -65,7 +65,7 @@ export default [ { // A destination's push and pushBatch never end with a bare return. files: ['**/destinations/**/*.{js,ts,tsx}'], - rules: { 'walkeros/no-silent-return': 'warn' }, + rules: { 'walkeros/no-silent-return': 'error' }, }, { // Relaxed rules for test files diff --git a/packages/core/src/types/destination.ts b/packages/core/src/types/destination.ts index bfb70b8ce..7685279bc 100644 --- a/packages/core/src/types/destination.ts +++ b/packages/core/src/types/destination.ts @@ -331,7 +331,7 @@ export interface PushBatchContext< export type InitFn = ( context: Context, -) => WalkerOS.PromiseOrValue>; +) => WalkerOS.PromiseOrValue>; export type PushFn = ( event: WalkerOS.Event, diff --git a/packages/core/src/types/store.ts b/packages/core/src/types/store.ts index 256481f61..f59e3abd4 100644 --- a/packages/core/src/types/store.ts +++ b/packages/core/src/types/store.ts @@ -129,7 +129,7 @@ export type Init = ( export type InitFn = ( context: Context, -) => WalkerOS.PromiseOrValue>; +) => WalkerOS.PromiseOrValue>; export type InitStore = { code: Init; diff --git a/packages/core/src/types/transformer.ts b/packages/core/src/types/transformer.ts index 051b57647..6bcec3623 100644 --- a/packages/core/src/types/transformer.ts +++ b/packages/core/src/types/transformer.ts @@ -271,12 +271,12 @@ export type Fn< * * @param context - Transformer context * @returns void - initialization successful - * @returns false - initialization failed, skip this transformer + * @throws StepError - initialization failed; the push fails * @returns Config - return updated config */ export type InitFn = ( context: Context, -) => WalkerOS.PromiseOrValue>; +) => WalkerOS.PromiseOrValue>; /** * Transformer instance returned by Init function. From 2374f549c4a07edf40bdfb3bdd1360ebcdb5110b Mon Sep 17 00:00:00 2001 From: alexanderkirtzel Date: Wed, 21 Oct 2026 07:14:25 +0000 Subject: [PATCH 22/41] docs release --- .changeset/cli-server-flow-defaults.md | 9 + .changeset/cli-structure-errors.md | 9 + .changeset/collector-dev-conformance-kit.md | 9 + .changeset/collector-failure-reporting.md | 9 + .changeset/collector-ignored-event-skip.md | 8 + .changeset/collector-init-timeout.md | 8 + .changeset/collector-no-effect-config.md | 7 + .changeset/collector-server-consent-skip.md | 8 + .changeset/collector-walkerjs-size.md | 7 + .changeset/config-eslint-guards.md | 8 + .changeset/core-failure-contract.md | 9 + .changeset/core-logger-json-format.md | 9 + .changeset/core-step-error-api.md | 9 + .changeset/core-validate-structure-errors.md | 10 ++ .changeset/mcp-packages-resource-warnings.md | 7 + .changeset/runner-flow-logger.md | 8 + .changeset/server-core-send-throws.md | 7 + .../server-destination-amplitude-delivery.md | 8 + .../server-destination-api-url-required.md | 7 + .changeset/server-destination-aws-errors.md | 9 + .../server-destination-clickhouse-failures.md | 8 + .changeset/server-destination-file-writes.md | 8 + .changeset/server-destination-gcp-failures.md | 8 + .../server-destination-hubspot-batch.md | 9 + .../server-destination-kafka-first-use.md | 8 + .../server-destination-pinterest-body.md | 7 + .../server-destination-piwikpro-unmapped.md | 8 + .../server-destination-posthog-delivery.md | 8 + .../server-destination-redis-failures.md | 8 + ...server-destination-rudderstack-delivery.md | 8 + .../server-destination-segment-delivery.md | 8 + .../server-destination-sqlite-first-use.md | 8 + .changeset/server-destination-tiktok-body.md | 7 + .../server-destinations-report-failures.md | 24 +++ .../server-destinations-vendor-status.md | 9 + .changeset/server-source-express-body-log.md | 7 + .changeset/server-sources-acknowledge.md | 11 ++ .changeset/server-sources-pull-retry-lines.md | 9 + .changeset/server-stores-failed-reads.md | 12 ++ .../server-transformer-fingerprint-salt.md | 7 + .changeset/transformer-ga4-missing-request.md | 8 + .changeset/web-core-send-throws.md | 9 + .changeset/web-destination-amplitude-start.md | 8 + .changeset/web-destination-api-send.md | 8 + .changeset/web-destination-gtag-parameters.md | 8 + .changeset/web-destination-heap-sdk.md | 8 + .changeset/web-destination-matomo-unmapped.md | 8 + .changeset/web-destination-piano-load-wait.md | 7 + .../web-destination-piwikpro-unmapped.md | 7 + .changeset/web-destination-segment-cdn-url.md | 8 + .../web-destinations-consent-at-vendor.md | 12 ++ .../web-destinations-removed-settings.md | 9 + .../web-destinations-report-failures.md | 28 ++++ .changeset/web-destinations-vendor-scripts.md | 20 +++ .../web-source-browser-flush-on-hide.md | 7 + .changeset/web-source-datalayer-window.md | 7 + packages/mcps/SKILL.md | 2 +- .../server/destinations/amplitude/README.md | 2 +- packages/server/destinations/bing/README.md | 10 +- packages/server/destinations/criteo/README.md | 7 +- .../server/destinations/customerio/README.md | 4 +- .../server/destinations/datamanager/README.md | 14 +- packages/server/destinations/file/README.md | 5 +- packages/server/destinations/gcp/README.md | 5 +- .../server/destinations/hubspot/README.md | 7 +- packages/server/destinations/kafka/README.md | 9 +- .../server/destinations/klaviyo/README.md | 2 +- .../server/destinations/linkedin/README.md | 7 +- .../server/destinations/mixpanel/README.md | 2 +- .../server/destinations/mparticle/README.md | 7 +- .../server/destinations/pinterest/README.md | 7 +- .../server/destinations/posthog/README.md | 2 +- packages/server/destinations/reddit/README.md | 7 +- packages/server/destinations/redis/README.md | 9 +- .../server/destinations/rudderstack/README.md | 7 +- .../server/destinations/segment/README.md | 2 +- packages/server/destinations/slack/README.md | 6 +- .../server/destinations/snapchat/README.md | 7 +- packages/server/destinations/sqlite/README.md | 2 +- packages/server/destinations/tiktok/README.md | 7 +- .../server/destinations/twitter/README.md | 11 +- packages/server/stores/fs/README.md | 5 +- packages/server/stores/gcs/README.md | 5 +- packages/server/stores/s3/README.md | 12 +- packages/server/stores/sheets/README.md | 5 +- packages/web/destinations/gtag/README.md | 4 +- packages/web/destinations/linkedin/README.md | 2 +- packages/web/destinations/meta/README.md | 2 +- .../web/destinations/optimizely/README.md | 2 +- packages/web/destinations/pinterest/README.md | 2 +- packages/web/destinations/snowplow/README.md | 4 +- packages/web/destinations/tiktok/README.md | 2 +- packages/web/sources/demo/README.md | 5 +- skills/INDEX.json | 1 + skills/walkeros-create-cmp-source/SKILL.md | 4 + skills/walkeros-create-destination/SKILL.md | 117 ++++++++++++- .../templates/simple/conformance.test.ts | 37 +++++ .../templates/simple/index.test.ts | 17 +- .../templates/simple/index.ts | 143 ++++++++++------ skills/walkeros-create-source/SKILL.md | 31 +++- skills/walkeros-create-transformer/SKILL.md | 16 +- skills/walkeros-testing-strategy/SKILL.md | 27 +++ .../SKILL.md | 62 +++++-- skills/walkeros-using-cli/SKILL.md | 2 +- .../walkeros-using-cli/flow-configuration.md | 93 ++++++----- skills/walkeros-using-logger/SKILL.md | 134 +++++++++------ skills/walkeros-using-step-examples/SKILL.md | 41 +++++ website/docs/apps/cli.mdx | 36 ++-- website/docs/apps/mcp.mdx | 2 +- website/docs/apps/runtime.mdx | 7 + website/docs/collector/commands.mdx | 21 ++- website/docs/collector/index.mdx | 156 +++++++++++------- website/docs/collector/logger.mdx | 135 +++++++++++---- website/docs/destinations/api/server.mdx | 12 +- website/docs/destinations/api/web.mdx | 20 ++- website/docs/destinations/code.mdx | 8 +- website/docs/destinations/create-your-own.mdx | 101 ++++++++++-- website/docs/destinations/index.mdx | 17 ++ .../docs/destinations/server/amplitude.mdx | 17 ++ website/docs/destinations/server/gcp.mdx | 1 + website/docs/destinations/server/hubspot.mdx | 25 ++- website/docs/destinations/server/kafka.mdx | 2 +- .../docs/destinations/server/pinterest.mdx | 4 + website/docs/destinations/server/posthog.mdx | 17 ++ website/docs/destinations/server/redis.mdx | 2 + .../docs/destinations/server/rudderstack.mdx | 17 ++ website/docs/destinations/server/segment.mdx | 17 ++ website/docs/destinations/server/sqlite.mdx | 2 +- website/docs/destinations/server/tiktok.mdx | 4 + website/docs/destinations/web/amplitude.mdx | 21 ++- website/docs/destinations/web/clarity.mdx | 4 +- website/docs/destinations/web/fullstory.mdx | 5 + website/docs/destinations/web/gtag/index.mdx | 2 + website/docs/destinations/web/heap.mdx | 27 ++- website/docs/destinations/web/matomo.mdx | 6 + website/docs/destinations/web/meta-pixel.mdx | 5 + website/docs/destinations/web/piano.mdx | 17 +- website/docs/destinations/web/piwikpro.mdx | 5 + website/docs/destinations/web/segment.mdx | 6 +- website/docs/destinations/web/snowplow.mdx | 2 + website/docs/getting-started/deploy.mdx | 1 + website/docs/getting-started/flow/index.mdx | 5 +- website/docs/guides/deploy-gcp-cloudrun.mdx | 16 ++ website/docs/guides/reference-syntax.mdx | 4 +- website/docs/mapping/index.mdx | 2 +- website/docs/mapping/rule.mdx | 67 ++++---- website/docs/mapping/value.mdx | 13 ++ website/docs/sources/server/aws.mdx | 14 +- website/docs/sources/server/express.mdx | 15 +- website/docs/sources/server/fetch.mdx | 16 +- website/docs/sources/server/gcp.mdx | 14 +- website/docs/sources/server/index.mdx | 19 +++ website/docs/sources/server/pubsub.mdx | 10 +- website/docs/sources/server/sqs.mdx | 2 +- website/docs/sources/web/browser/index.mdx | 11 ++ website/docs/sources/web/dataLayer/index.mdx | 3 + website/docs/stores/index.mdx | 4 + website/docs/stores/server/s3.mdx | 2 +- website/docs/stores/server/sheets.mdx | 2 +- website/docs/transformers/fingerprint.mdx | 4 +- website/docs/transformers/ga4.mdx | 4 +- website/static/schema/flow/v4.json | 52 ++++++ 162 files changed, 1996 insertions(+), 441 deletions(-) create mode 100644 .changeset/cli-server-flow-defaults.md create mode 100644 .changeset/cli-structure-errors.md create mode 100644 .changeset/collector-dev-conformance-kit.md create mode 100644 .changeset/collector-failure-reporting.md create mode 100644 .changeset/collector-ignored-event-skip.md create mode 100644 .changeset/collector-init-timeout.md create mode 100644 .changeset/collector-no-effect-config.md create mode 100644 .changeset/collector-server-consent-skip.md create mode 100644 .changeset/collector-walkerjs-size.md create mode 100644 .changeset/config-eslint-guards.md create mode 100644 .changeset/core-failure-contract.md create mode 100644 .changeset/core-logger-json-format.md create mode 100644 .changeset/core-step-error-api.md create mode 100644 .changeset/core-validate-structure-errors.md create mode 100644 .changeset/mcp-packages-resource-warnings.md create mode 100644 .changeset/runner-flow-logger.md create mode 100644 .changeset/server-core-send-throws.md create mode 100644 .changeset/server-destination-amplitude-delivery.md create mode 100644 .changeset/server-destination-api-url-required.md create mode 100644 .changeset/server-destination-aws-errors.md create mode 100644 .changeset/server-destination-clickhouse-failures.md create mode 100644 .changeset/server-destination-file-writes.md create mode 100644 .changeset/server-destination-gcp-failures.md create mode 100644 .changeset/server-destination-hubspot-batch.md create mode 100644 .changeset/server-destination-kafka-first-use.md create mode 100644 .changeset/server-destination-pinterest-body.md create mode 100644 .changeset/server-destination-piwikpro-unmapped.md create mode 100644 .changeset/server-destination-posthog-delivery.md create mode 100644 .changeset/server-destination-redis-failures.md create mode 100644 .changeset/server-destination-rudderstack-delivery.md create mode 100644 .changeset/server-destination-segment-delivery.md create mode 100644 .changeset/server-destination-sqlite-first-use.md create mode 100644 .changeset/server-destination-tiktok-body.md create mode 100644 .changeset/server-destinations-report-failures.md create mode 100644 .changeset/server-destinations-vendor-status.md create mode 100644 .changeset/server-source-express-body-log.md create mode 100644 .changeset/server-sources-acknowledge.md create mode 100644 .changeset/server-sources-pull-retry-lines.md create mode 100644 .changeset/server-stores-failed-reads.md create mode 100644 .changeset/server-transformer-fingerprint-salt.md create mode 100644 .changeset/transformer-ga4-missing-request.md create mode 100644 .changeset/web-core-send-throws.md create mode 100644 .changeset/web-destination-amplitude-start.md create mode 100644 .changeset/web-destination-api-send.md create mode 100644 .changeset/web-destination-gtag-parameters.md create mode 100644 .changeset/web-destination-heap-sdk.md create mode 100644 .changeset/web-destination-matomo-unmapped.md create mode 100644 .changeset/web-destination-piano-load-wait.md create mode 100644 .changeset/web-destination-piwikpro-unmapped.md create mode 100644 .changeset/web-destination-segment-cdn-url.md create mode 100644 .changeset/web-destinations-consent-at-vendor.md create mode 100644 .changeset/web-destinations-removed-settings.md create mode 100644 .changeset/web-destinations-report-failures.md create mode 100644 .changeset/web-destinations-vendor-scripts.md create mode 100644 .changeset/web-source-browser-flush-on-hide.md create mode 100644 .changeset/web-source-datalayer-window.md create mode 100644 skills/walkeros-create-destination/templates/simple/conformance.test.ts diff --git a/.changeset/cli-server-flow-defaults.md b/.changeset/cli-server-flow-defaults.md new file mode 100644 index 000000000..cdadf1df0 --- /dev/null +++ b/.changeset/cli-server-flow-defaults.md @@ -0,0 +1,9 @@ +--- +'@walkeros/cli': patch +--- + +Server flows skip consent-denied events by default +(`collector.consentDenied: 'skip'`; an authored value wins). A flow's own +`logger` level, format and handler apply under `runneros`. `walkeros setup` +fails with the reason when a step's `init` throws or returns `false`. Inline +`on`, `pushBatch` and `destroy` reach the bundle. diff --git a/.changeset/cli-structure-errors.md b/.changeset/cli-structure-errors.md new file mode 100644 index 000000000..ab587d7a2 --- /dev/null +++ b/.changeset/cli-structure-errors.md @@ -0,0 +1,9 @@ +--- +'@walkeros/cli': patch +--- + +`walkeros validate` reports unknown keys, mapping keys with no effect, values +without an operator, malformed references and `batch` without `pushBatch` as +errors (`MAPPING_NO_OP` replaces the warning `TRANSFORMER_MAPPING_NO_OP`), and +`bundle`, `push`, `simulate`, `setup` stop on them before building. An unknown +settings key is an error where the package closes its schema. diff --git a/.changeset/collector-dev-conformance-kit.md b/.changeset/collector-dev-conformance-kit.md new file mode 100644 index 000000000..295ad666a --- /dev/null +++ b/.changeset/collector-dev-conformance-kit.md @@ -0,0 +1,9 @@ +--- +'@walkeros/collector': patch +--- + +`@walkeros/collector/dev` adds `runExamples`, which runs step examples on a real +collector, and `conformance`, a test kit with profiles (`http`, `sdk-env`, +`web-sdk`, `vendor-global`, `socket`, `transformer`, `source`) that checks the +failure contract of a destination, source, transformer or store and reports +every broken item at once. diff --git a/.changeset/collector-failure-reporting.md b/.changeset/collector-failure-reporting.md new file mode 100644 index 000000000..fa127e128 --- /dev/null +++ b/.changeset/collector-failure-reporting.md @@ -0,0 +1,9 @@ +--- +'@walkeros/collector': patch +--- + +Failures are explicit. A failing `init`, an open circuit breaker, a destination +held past one retry window, a full pre-run queue and a throwing transformer +answer `failed` or `ok: false` with a `reason`; consent waits stay `queued`. The +collector writes one ERROR line per cause and minute, with a `suppressed` count, +and never the event payload. diff --git a/.changeset/collector-ignored-event-skip.md b/.changeset/collector-ignored-event-skip.md new file mode 100644 index 000000000..5a3bee83a --- /dev/null +++ b/.changeset/collector-ignored-event-skip.md @@ -0,0 +1,8 @@ +--- +'@walkeros/collector': patch +--- + +An event that a rule with `ignore: true` drops is reported as skipped, not +delivered: it is no longer under `done`, does not move `count`, and counts in +`skipped`. A source answers `{ ok: true, dropped: true }`. New `walker flush` +sends every pending destination batch at once and leaves the collector running. diff --git a/.changeset/collector-init-timeout.md b/.changeset/collector-init-timeout.md new file mode 100644 index 000000000..a38bd37c1 --- /dev/null +++ b/.changeset/collector-init-timeout.md @@ -0,0 +1,8 @@ +--- +'@walkeros/collector': patch +--- + +A destination or transformer `init` that does not settle within the step's +timeout (destination `config.timeout`, default 10 seconds; transformers use the +default) now fails the waiting pushes with code `timeout` instead of leaving +them unanswered. The next push retries the `init`. diff --git a/.changeset/collector-no-effect-config.md b/.changeset/collector-no-effect-config.md new file mode 100644 index 000000000..c9a911a76 --- /dev/null +++ b/.changeset/collector-no-effect-config.md @@ -0,0 +1,7 @@ +--- +'@walkeros/collector': patch +--- + +A destination whose config sets `batch` without a `pushBatch`, or a mapping key +with no effect at its position, logs one ERROR when it registers and still runs. +The orphan connection error that lost no event logs at WARN instead of ERROR. diff --git a/.changeset/collector-server-consent-skip.md b/.changeset/collector-server-consent-skip.md new file mode 100644 index 000000000..628104e4e --- /dev/null +++ b/.changeset/collector-server-consent-skip.md @@ -0,0 +1,8 @@ +--- +'@walkeros/collector': patch +--- + +New `consentDenied: 'skip'` makes a server collector skip an event that a +destination's consent does not allow, instead of queueing it for a grant that +never comes. It counts in `skipped` and logs at DEBUG, never as an error. A +destination without `init` is initialized without one, so its `on` handlers run. diff --git a/.changeset/collector-walkerjs-size.md b/.changeset/collector-walkerjs-size.md new file mode 100644 index 000000000..ff7cc19e3 --- /dev/null +++ b/.changeset/collector-walkerjs-size.md @@ -0,0 +1,7 @@ +--- +'@walkeros/collector': patch +--- + +The web bundle `walker.js` carries the full failure contract, the circuit +breaker included, with no lean variant that leaves it out. It grows from 35,181 +bytes to 39,334 bytes, measured as gzip -9, and the bundle is not size-gated. diff --git a/.changeset/config-eslint-guards.md b/.changeset/config-eslint-guards.md new file mode 100644 index 000000000..21b6b48fa --- /dev/null +++ b/.changeset/config-eslint-guards.md @@ -0,0 +1,8 @@ +--- +'@walkeros/config': patch +--- + +The lint rules `walkeros/await-send` and `walkeros/no-silent-return` are errors: +a send that is not awaited, or a bare `return` in a destination's `push` or +`pushBatch`, fails lint. ESM builds of server-only packages define `require`, so +their CommonJS SDKs load, and tests are no longer published. diff --git a/.changeset/core-failure-contract.md b/.changeset/core-failure-contract.md new file mode 100644 index 000000000..8b8101766 --- /dev/null +++ b/.changeset/core-failure-contract.md @@ -0,0 +1,9 @@ +--- +'@walkeros/core': patch +--- + +`logger.throw` only throws and no longer writes a log line; the collector writes +the one failure line. A rule `condition` that throws fails the event instead of +falling through. A mapping value without an operator is dropped and reported. +The wildcard lookup tries exact, `entity *`, `* action`, then `* *`, and falls +through a tier whose conditions all fail. diff --git a/.changeset/core-logger-json-format.md b/.changeset/core-logger-json-format.md new file mode 100644 index 000000000..67950dcd8 --- /dev/null +++ b/.changeset/core-logger-json-format.md @@ -0,0 +1,9 @@ +--- +'@walkeros/core': patch +--- + +`logger.format: 'json'` prints one JSON object per line on server flows, with +`severity`, `message`, `time`, `scope` and `context`; errors and warnings go to +stderr. Web flows reject it. A log handler receives the cause code, status, +retryable flag and suppressed count as structured fields, so lines can be +forwarded without parsing. diff --git a/.changeset/core-step-error-api.md b/.changeset/core-step-error-api.md new file mode 100644 index 000000000..f876f4445 --- /dev/null +++ b/.changeset/core-step-error-api.md @@ -0,0 +1,9 @@ +--- +'@walkeros/core': patch +--- + +New `StepError` carries a code, an HTTP status and `retryable`; +`acknowledge(result)` tells a producer to ack, retry or reject a message, and +`reportCause` writes one failure line per cause. `SendResponse` is `{ data }`. +An `init` of a destination, transformer or store can no longer return `false`: +throw a `StepError`, and untyped `false` fails with `not-ready`. diff --git a/.changeset/core-validate-structure-errors.md b/.changeset/core-validate-structure-errors.md new file mode 100644 index 000000000..c4f499e08 --- /dev/null +++ b/.changeset/core-validate-structure-errors.md @@ -0,0 +1,10 @@ +--- +'@walkeros/core': patch +--- + +Flow validation now rejects unknown keys in step `config`, destination and store +entries, the `collector` block, inline `code` and contract rules, mapping keys +with no effect at their position, values without an operator or with two +producers, and malformed whole-value references. Inline code may define +`pushBatch`, `on` and `destroy`. Strict `$flow` treats an empty value as +unresolved. diff --git a/.changeset/mcp-packages-resource-warnings.md b/.changeset/mcp-packages-resource-warnings.md new file mode 100644 index 000000000..c95d77d55 --- /dev/null +++ b/.changeset/mcp-packages-resource-warnings.md @@ -0,0 +1,7 @@ +--- +'@walkeros/mcp': patch +--- + +The `walkeros://reference/packages` resource answers +`{ catalog, count, warnings? }` instead of a bare array, so a failed npm search +appears as a warning and a partial catalog is never read as an empty one. diff --git a/.changeset/runner-flow-logger.md b/.changeset/runner-flow-logger.md new file mode 100644 index 000000000..e219b4c7c --- /dev/null +++ b/.changeset/runner-flow-logger.md @@ -0,0 +1,8 @@ +--- +'@walkeros/runner': patch +--- + +A flow's `logger.level` and `logger.format` apply to the collector's lines, and +the format to the runner's own lines; `--verbose` still forces DEBUG. +`flow handler error` logs at ERROR, a rejected heartbeat logs a WARN, and Node +process warnings go through the logger at WARN. diff --git a/.changeset/server-core-send-throws.md b/.changeset/server-core-send-throws.md new file mode 100644 index 000000000..d85513939 --- /dev/null +++ b/.changeset/server-core-send-throws.md @@ -0,0 +1,7 @@ +--- +'@walkeros/server-core': patch +--- + +`sendServer` now throws a `StepError` instead of resolving `{ ok: false }`: +`transport` with the status and body, `timeout`, or `config` for an invalid URL. +It runs on the global `fetch`, and `GET` and `HEAD` requests send no body. diff --git a/.changeset/server-destination-amplitude-delivery.md b/.changeset/server-destination-amplitude-delivery.md new file mode 100644 index 000000000..0a13a35ae --- /dev/null +++ b/.changeset/server-destination-amplitude-delivery.md @@ -0,0 +1,8 @@ +--- +'@walkeros/server-destination-amplitude': patch +--- + +Batches by default (`config.batch` 200 events or 10 seconds) through +`pushBatch`; each event fails, with the status, when Amplitude does not answer +with 2xx, instead of counting as delivered. A single push waits for its answer +too. diff --git a/.changeset/server-destination-api-url-required.md b/.changeset/server-destination-api-url-required.md new file mode 100644 index 000000000..8229067f8 --- /dev/null +++ b/.changeset/server-destination-api-url-required.md @@ -0,0 +1,7 @@ +--- +'@walkeros/server-destination-api': patch +--- + +`settings.url` is required: without it the destination fails to start with a +`config` error. Requests run on `fetch`, and a request that fails, times out or +gets an error status is reported as a failed push. diff --git a/.changeset/server-destination-aws-errors.md b/.changeset/server-destination-aws-errors.md new file mode 100644 index 000000000..9ad24b5bb --- /dev/null +++ b/.changeset/server-destination-aws-errors.md @@ -0,0 +1,9 @@ +--- +'@walkeros/server-destination-aws': patch +--- + +Failures are reported with the AWS error name as code, the HTTP status and a +correct retryable flag, so a missing stream, denied access or missing +credentials is no longer retried. Messages name the stream or topic, region and +fix, not AWS's text. Settings schemas require a stream name (Firehose) or topic +ARN or name (SNS). diff --git a/.changeset/server-destination-clickhouse-failures.md b/.changeset/server-destination-clickhouse-failures.md new file mode 100644 index 000000000..ede43ea5d --- /dev/null +++ b/.changeset/server-destination-clickhouse-failures.md @@ -0,0 +1,8 @@ +--- +'@walkeros/server-destination-clickhouse': patch +--- + +A failed insert carries its ClickHouse code or network error and, for a missing +table or too many parts, the fix in its message. The destination no longer logs +a line of its own, and the server's text stays out of the message. Rules with +`silent: true` insert nothing. diff --git a/.changeset/server-destination-file-writes.md b/.changeset/server-destination-file-writes.md new file mode 100644 index 000000000..0d6096744 --- /dev/null +++ b/.changeset/server-destination-file-writes.md @@ -0,0 +1,8 @@ +--- +'@walkeros/server-destination-file': patch +--- + +Each push waits until its line is written; a failed open or write fails the push +instead of a warning, and a stream error no longer crashes the process. A +dynamic filename that resolves to nothing fails with `missing-input`. Rules with +`silent: true` write nothing. diff --git a/.changeset/server-destination-gcp-failures.md b/.changeset/server-destination-gcp-failures.md new file mode 100644 index 000000000..db0231942 --- /dev/null +++ b/.changeset/server-destination-gcp-failures.md @@ -0,0 +1,8 @@ +--- +'@walkeros/server-destination-gcp': patch +--- + +BigQuery opens its writer at the first push instead of at startup; a missing +dataset or table fails that push with the setup hint. BigQuery and Pub/Sub +failures are reported once by the collector instead of being logged twice, and +their messages name codes, not row data. diff --git a/.changeset/server-destination-hubspot-batch.md b/.changeset/server-destination-hubspot-batch.md new file mode 100644 index 000000000..1c1603cc8 --- /dev/null +++ b/.changeset/server-destination-hubspot-batch.md @@ -0,0 +1,9 @@ +--- +'@walkeros/server-destination-hubspot': patch +--- + +`settings.batch` and `settings.batchSize` are removed: batch with the +destination's `config.batch`, which sends through HubSpot's batch API in chunks +of 500 and reports each event's own outcome. An event without email or objectId +fails with `missing-input`, and a failed HubSpot call fails the push with its +status. diff --git a/.changeset/server-destination-kafka-first-use.md b/.changeset/server-destination-kafka-first-use.md new file mode 100644 index 000000000..d5472b4e1 --- /dev/null +++ b/.changeset/server-destination-kafka-first-use.md @@ -0,0 +1,8 @@ +--- +'@walkeros/server-destination-kafka': patch +--- + +The producer connects at the first push, so a flow starts without a reachable +broker. A failed send fails the push instead of being logged and counted +delivered; an unknown topic is a `config` error with the setup hint. Rules with +`silent: true` send nothing. diff --git a/.changeset/server-destination-pinterest-body.md b/.changeset/server-destination-pinterest-body.md new file mode 100644 index 000000000..5ea9e65ed --- /dev/null +++ b/.changeset/server-destination-pinterest-body.md @@ -0,0 +1,7 @@ +--- +'@walkeros/server-destination-pinterest': patch +--- + +An event that Pinterest reports as `failed` in its HTTP 200 answer now fails the +push instead of counting as delivered, with the dead-letter queue and one ERROR +line. Rules with `silent: true` send nothing. diff --git a/.changeset/server-destination-piwikpro-unmapped.md b/.changeset/server-destination-piwikpro-unmapped.md new file mode 100644 index 000000000..27e999e60 --- /dev/null +++ b/.changeset/server-destination-piwikpro-unmapped.md @@ -0,0 +1,8 @@ +--- +'@walkeros/server-destination-piwikpro': patch +--- + +Events that no rule maps, unknown methods, invalid arguments and a missing page +URL fail with `missing-input`; use an `ignore: true` rule to keep an event out +without a failure. Piwik PRO's own 429 warning is replaced by the collector's +line with the status. diff --git a/.changeset/server-destination-posthog-delivery.md b/.changeset/server-destination-posthog-delivery.md new file mode 100644 index 000000000..a92ff3194 --- /dev/null +++ b/.changeset/server-destination-posthog-delivery.md @@ -0,0 +1,8 @@ +--- +'@walkeros/server-destination-posthog': patch +--- + +`settings.include` is removed; it had no effect, use `config.include`. Events +batch by default (`config.batch` 20 events or 10 seconds) through `pushBatch`; a +failed send fails its events and lands them in the dead-letter queue instead of +counting as delivered. diff --git a/.changeset/server-destination-redis-failures.md b/.changeset/server-destination-redis-failures.md new file mode 100644 index 000000000..5c6cb8575 --- /dev/null +++ b/.changeset/server-destination-redis-failures.md @@ -0,0 +1,8 @@ +--- +'@walkeros/server-destination-redis': patch +--- + +A failed XADD fails the push and lands in the dead-letter queue instead of being +logged and counted delivered. Connection errors are reported through the +collector, and shutdown no longer waits on a disconnected client. Rules with +`silent: true` append nothing. diff --git a/.changeset/server-destination-rudderstack-delivery.md b/.changeset/server-destination-rudderstack-delivery.md new file mode 100644 index 000000000..a1ed0bb8f --- /dev/null +++ b/.changeset/server-destination-rudderstack-delivery.md @@ -0,0 +1,8 @@ +--- +'@walkeros/server-destination-rudderstack': patch +--- + +Batches by default (`config.batch` 20 events or 10 seconds) through `pushBatch`; +each event's own send decides its outcome, so a failed send fails that event +instead of counting as delivered, and no longer causes an unhandled promise +rejection. An event without userId or anonymousId fails with `missing-input`. diff --git a/.changeset/server-destination-segment-delivery.md b/.changeset/server-destination-segment-delivery.md new file mode 100644 index 000000000..32bf4b0fb --- /dev/null +++ b/.changeset/server-destination-segment-delivery.md @@ -0,0 +1,8 @@ +--- +'@walkeros/server-destination-segment': patch +--- + +Batches by default (`config.batch` 15 events or 10 seconds) through `pushBatch`; +each event's own delivery decides its outcome, so a failed delivery fails that +event and lands in the dead-letter queue instead of counting as delivered. An +event without userId or anonymousId fails with `missing-input`. diff --git a/.changeset/server-destination-sqlite-first-use.md b/.changeset/server-destination-sqlite-first-use.md new file mode 100644 index 000000000..b6f2c6ed6 --- /dev/null +++ b/.changeset/server-destination-sqlite-first-use.md @@ -0,0 +1,8 @@ +--- +'@walkeros/server-destination-sqlite': patch +--- + +The table check and the prepared insert run at the first push, so a flow starts +without querying the database. A missing table fails the push with the setup +hint, and a failed INSERT fails the push instead of being logged and counted +delivered. diff --git a/.changeset/server-destination-tiktok-body.md b/.changeset/server-destination-tiktok-body.md new file mode 100644 index 000000000..e197ff75b --- /dev/null +++ b/.changeset/server-destination-tiktok-body.md @@ -0,0 +1,7 @@ +--- +'@walkeros/server-destination-tiktok': patch +--- + +A request that TikTok answers with HTTP 200 and a non-zero `code` now fails the +push instead of counting as delivered; rate limits and server errors are +retryable, other rejections are not. Rules with `silent: true` send nothing. diff --git a/.changeset/server-destinations-report-failures.md b/.changeset/server-destinations-report-failures.md new file mode 100644 index 000000000..fe37f9e0e --- /dev/null +++ b/.changeset/server-destinations-report-failures.md @@ -0,0 +1,24 @@ +--- +'@walkeros/server-destination-api': patch +'@walkeros/server-destination-bing': patch +'@walkeros/server-destination-criteo': patch +'@walkeros/server-destination-linkedin': patch +'@walkeros/server-destination-meta': patch +'@walkeros/server-destination-mparticle': patch +'@walkeros/server-destination-pinterest': patch +'@walkeros/server-destination-piwikpro': patch +'@walkeros/server-destination-reddit': patch +'@walkeros/server-destination-slack': patch +'@walkeros/server-destination-snapchat': patch +'@walkeros/server-destination-tiktok': patch +'@walkeros/server-destination-twitter': patch +'@walkeros/server-destination-klaviyo': patch +'@walkeros/server-destination-customerio': patch +'@walkeros/server-destination-mixpanel': patch +'@walkeros/server-destination-datamanager': patch +--- + +A failed request is reported as a failed push (dead-letter queue, one ERROR line +with the status) instead of counting as delivered or being logged at debug. +Events a destination cannot send, such as a missing user identifier, fail with +`missing-input`. Rules with `silent: true` send nothing. diff --git a/.changeset/server-destinations-vendor-status.md b/.changeset/server-destinations-vendor-status.md new file mode 100644 index 000000000..1e52cb322 --- /dev/null +++ b/.changeset/server-destinations-vendor-status.md @@ -0,0 +1,9 @@ +--- +'@walkeros/server-destination-klaviyo': patch +'@walkeros/server-destination-customerio': patch +'@walkeros/server-destination-datamanager': patch +--- + +Failures carry the vendor's HTTP status, so a 4xx other than 429 is not retried +and a 429 or 5xx is. Data Manager's `settings.logLevel` is removed; it was never +read. diff --git a/.changeset/server-source-express-body-log.md b/.changeset/server-source-express-body-log.md new file mode 100644 index 000000000..782c7f707 --- /dev/null +++ b/.changeset/server-source-express-body-log.md @@ -0,0 +1,7 @@ +--- +'@walkeros/server-source-express': patch +--- + +The `Request body rejected` debug line carries only the status and the content +type, no longer a body sample or the parser message, so a rejected request body +can never leak its content into the logs. diff --git a/.changeset/server-sources-acknowledge.md b/.changeset/server-sources-acknowledge.md new file mode 100644 index 000000000..da1c6a352 --- /dev/null +++ b/.changeset/server-sources-acknowledge.md @@ -0,0 +1,11 @@ +--- +'@walkeros/server-source-aws': patch +'@walkeros/server-source-gcp': patch +'@walkeros/server-source-express': patch +'@walkeros/server-source-fetch': patch +--- + +A push that failed for a reason a resend cannot fix (a missing setting, a 4xx, a +destination that is not ready) is acknowledged: SQS deletes the message, Pub/Sub +acks it, HTTP sources answer 200. Only retryable failures nack or answer 500, or +207 for a batch, with the fixed text `Event was not processed`. diff --git a/.changeset/server-sources-pull-retry-lines.md b/.changeset/server-sources-pull-retry-lines.md new file mode 100644 index 000000000..0c43c1d5e --- /dev/null +++ b/.changeset/server-sources-pull-retry-lines.md @@ -0,0 +1,9 @@ +--- +'@walkeros/server-source-aws': patch +'@walkeros/server-source-gcp': patch +--- + +A nack for a retryable failure logs at DEBUG, because the collector's ERROR line +reports the cause once per minute instead of one line per message. Transient +Pub/Sub stream errors and SQS receive errors that the SDK or loop retries log at +WARN instead of ERROR. diff --git a/.changeset/server-stores-failed-reads.md b/.changeset/server-stores-failed-reads.md new file mode 100644 index 000000000..bc6d23d72 --- /dev/null +++ b/.changeset/server-stores-failed-reads.md @@ -0,0 +1,12 @@ +--- +'@walkeros/server-store-fs': patch +'@walkeros/server-store-gcs': patch +'@walkeros/server-store-s3': patch +'@walkeros/server-store-sheets': patch +--- + +A read, write or delete that fails (403, 5xx, network) now throws instead of +looking like a missing key, and a stored value that cannot be decoded throws +too. The collector reports a failed read as a miss with one error line. S3 and +Sheets check their bucket or spreadsheet at the first operation instead of at +startup. diff --git a/.changeset/server-transformer-fingerprint-salt.md b/.changeset/server-transformer-fingerprint-salt.md new file mode 100644 index 000000000..85c925a21 --- /dev/null +++ b/.changeset/server-transformer-fingerprint-salt.md @@ -0,0 +1,7 @@ +--- +'@walkeros/server-transformer-fingerprint': patch +--- + +`settings.salt` is required; without it the transformer does not start. An event +whose IP, user agent and fields all resolve empty fails with `missing-input` +instead of getting the hash every other visitor gets. diff --git a/.changeset/transformer-ga4-missing-request.md b/.changeset/transformer-ga4-missing-request.md new file mode 100644 index 000000000..f2737928a --- /dev/null +++ b/.changeset/transformer-ga4-missing-request.md @@ -0,0 +1,8 @@ +--- +'@walkeros/transformer-ga4': patch +--- + +An event without a GA4 request in its ingest now fails with `missing-input` +instead of being dropped at debug level, so a source missing `config.ingest` +shows as an error. A request that cannot be decoded fails the event with code +`transformer`. diff --git a/.changeset/web-core-send-throws.md b/.changeset/web-core-send-throws.md new file mode 100644 index 000000000..933b0204c --- /dev/null +++ b/.changeset/web-core-send-throws.md @@ -0,0 +1,9 @@ +--- +'@walkeros/web-core': patch +--- + +`sendWeb` now throws a `StepError` instead of resolving `{ ok: false }`: +`transport` with the status for an error answer or a network error, `timeout`, +or `beacon-rejected`. A fetch skips `keepalive` past the browser's 64 KiB limit. +New `loadVendor` loads a vendor script once per page and reports +`vendor-not-loaded`, `vendor-not-ready` or `vendor-timeout`. diff --git a/.changeset/web-destination-amplitude-start.md b/.changeset/web-destination-amplitude-start.md new file mode 100644 index 000000000..9133a0555 --- /dev/null +++ b/.changeset/web-destination-amplitude-start.md @@ -0,0 +1,8 @@ +--- +'@walkeros/web-destination-amplitude': patch +--- + +Without Session Replay, Experiment or Guides & Surveys settings only Amplitude +analytics starts; enabling one still starts all three. Options now reach the SDK +(`autocapture: false` was lost before). A missing `apiKey` is a `config` error +and a failed init is retried by the next push. diff --git a/.changeset/web-destination-api-send.md b/.changeset/web-destination-api-send.md new file mode 100644 index 000000000..a6ff567e7 --- /dev/null +++ b/.changeset/web-destination-api-send.md @@ -0,0 +1,8 @@ +--- +'@walkeros/web-destination-api': patch +--- + +A failed request (an error status, a network error, a timeout, a beacon the +browser refuses) is reported as a failed push instead of counting as delivered. +Without `settings.url` the destination fails with a `config` error. Batches over +64 KiB are no longer sent with `keepalive`. diff --git a/.changeset/web-destination-gtag-parameters.md b/.changeset/web-destination-gtag-parameters.md new file mode 100644 index 000000000..47e68cce4 --- /dev/null +++ b/.changeset/web-destination-gtag-parameters.md @@ -0,0 +1,8 @@ +--- +'@walkeros/web-destination-gtag': patch +--- + +A GA4 parameter whose name is empty, or collides with another after +normalization, is dropped with one ERROR line per cause and the event is still +sent; before, it logged a warning per event and the later value overwrote the +earlier. gtag.js and GTM containers load through walkerOS. diff --git a/.changeset/web-destination-heap-sdk.md b/.changeset/web-destination-heap-sdk.md new file mode 100644 index 000000000..94520ab76 --- /dev/null +++ b/.changeset/web-destination-heap-sdk.md @@ -0,0 +1,8 @@ +--- +'@walkeros/web-destination-heap': patch +--- + +Validation rejects `disablePageviewAutocapture`, `disableSessionReplay` and +`ingestServer`: the classic Heap SDK never read them (use `heapConfig`). Events +pushed before Heap loads now reach it, consent commands no longer stall the +destination, and a missing `appId` is a `config` error. diff --git a/.changeset/web-destination-matomo-unmapped.md b/.changeset/web-destination-matomo-unmapped.md new file mode 100644 index 000000000..17950fc37 --- /dev/null +++ b/.changeset/web-destination-matomo-unmapped.md @@ -0,0 +1,8 @@ +--- +'@walkeros/web-destination-matomo': patch +--- + +An event that no rule maps fails with `missing-input` instead of a warning. An +invalid `goalId` sends the hit without the goal and logs one ERROR per cause. A +consent revocation sends `forgetConsentGiven`, stopping link tracking too, and +silent rules send only their goal. diff --git a/.changeset/web-destination-piano-load-wait.md b/.changeset/web-destination-piano-load-wait.md new file mode 100644 index 000000000..9358661c9 --- /dev/null +++ b/.changeset/web-destination-piano-load-wait.md @@ -0,0 +1,7 @@ +--- +'@walkeros/web-destination-piano': patch +--- + +Events no longer vanish while the Piano script loads: pushes wait for it, and +fail visibly when it is blocked or missing. A missing `site` or `collectDomain` +is a `config` error, and rules with `silent: true` send no event. diff --git a/.changeset/web-destination-piwikpro-unmapped.md b/.changeset/web-destination-piwikpro-unmapped.md new file mode 100644 index 000000000..04bf3c420 --- /dev/null +++ b/.changeset/web-destination-piwikpro-unmapped.md @@ -0,0 +1,7 @@ +--- +'@walkeros/web-destination-piwikpro': patch +--- + +Events that no rule maps fail with `missing-input` instead of a warning; use an +`ignore: true` rule to keep an event out without a failure. A blocked `ppms.js` +fails later events visibly. diff --git a/.changeset/web-destination-segment-cdn-url.md b/.changeset/web-destination-segment-cdn-url.md new file mode 100644 index 000000000..28655d3e5 --- /dev/null +++ b/.changeset/web-destination-segment-cdn-url.md @@ -0,0 +1,8 @@ +--- +'@walkeros/web-destination-segment': patch +--- + +`cdnURL` now reaches Analytics.js; before it was ignored, so a proxy was +bypassed. A missing `apiKey` is a `config` error, a failed Analytics.js load +fails later events, and a consent-gated destination loads Analytics.js on its +first event after the grant. diff --git a/.changeset/web-destinations-consent-at-vendor.md b/.changeset/web-destinations-consent-at-vendor.md new file mode 100644 index 000000000..2973bd875 --- /dev/null +++ b/.changeset/web-destinations-consent-at-vendor.md @@ -0,0 +1,12 @@ +--- +'@walkeros/web-destination-meta': patch +'@walkeros/web-destination-matomo': patch +'@walkeros/web-destination-clarity': patch +'@walkeros/web-destination-fullstory': patch +'@walkeros/web-destination-piwikpro': patch +--- + +A destination with `config.consent` now tells the vendor about consent changes: +Meta calls `fbq('consent', 'revoke' | 'grant')`, Matomo `forgetConsentGiven` and +`setConsentGiven`, Clarity sends ConsentV2, FullStory stops and starts capture, +and Piwik PRO turns off the link tracking it enabled. diff --git a/.changeset/web-destinations-removed-settings.md b/.changeset/web-destinations-removed-settings.md new file mode 100644 index 000000000..6a64d9137 --- /dev/null +++ b/.changeset/web-destinations-removed-settings.md @@ -0,0 +1,9 @@ +--- +'@walkeros/web-destination-optimizely': patch +'@walkeros/web-destination-snowplow': patch +--- + +Validation rejects `skipOdp` (Optimizely) and `pageViewTracking` (Snowplow): +neither was read, so they had no effect. Snowplow's settings schema now lists +every setting the code reads, with `collectorUrl` required, so a typo in a +setting name now fails validation. diff --git a/.changeset/web-destinations-report-failures.md b/.changeset/web-destinations-report-failures.md new file mode 100644 index 000000000..6051ec75c --- /dev/null +++ b/.changeset/web-destinations-report-failures.md @@ -0,0 +1,28 @@ +--- +'@walkeros/web-destination-amplitude': patch +'@walkeros/web-destination-api': patch +'@walkeros/web-destination-clarity': patch +'@walkeros/web-destination-d8a': patch +'@walkeros/web-destination-fullstory': patch +'@walkeros/web-destination-gtag': patch +'@walkeros/web-destination-heap': patch +'@walkeros/web-destination-hotjar': patch +'@walkeros/web-destination-linkedin': patch +'@walkeros/web-destination-matomo': patch +'@walkeros/web-destination-meta': patch +'@walkeros/web-destination-mixpanel': patch +'@walkeros/web-destination-optimizely': patch +'@walkeros/web-destination-piano': patch +'@walkeros/web-destination-pinterest': patch +'@walkeros/web-destination-piwikpro': patch +'@walkeros/web-destination-plausible': patch +'@walkeros/web-destination-posthog': patch +'@walkeros/web-destination-segment': patch +'@walkeros/web-destination-snowplow': patch +'@walkeros/web-destination-tiktok': patch +--- + +A failed vendor call, a missing required setting (`config`) and an event that +cannot be sent (`missing-input`) are reported as failed pushes, with the +dead-letter queue and one ERROR line, instead of counting as delivered. Rules +with `silent: true` send no event. Settings schemas reject unknown keys. diff --git a/.changeset/web-destinations-vendor-scripts.md b/.changeset/web-destinations-vendor-scripts.md new file mode 100644 index 000000000..8795a4ab2 --- /dev/null +++ b/.changeset/web-destinations-vendor-scripts.md @@ -0,0 +1,20 @@ +--- +'@walkeros/web-destination-clarity': patch +'@walkeros/web-destination-gtag': patch +'@walkeros/web-destination-heap': patch +'@walkeros/web-destination-matomo': patch +'@walkeros/web-destination-meta': patch +'@walkeros/web-destination-piano': patch +'@walkeros/web-destination-piwikpro': patch +'@walkeros/web-destination-plausible': patch +'@walkeros/web-destination-pinterest': patch +'@walkeros/web-destination-tiktok': patch +'@walkeros/web-destination-linkedin': patch +'@walkeros/web-destination-snowplow': patch +'@walkeros/web-destination-segment': patch +--- + +Vendor scripts load through walkerOS: a blocked or slow script fails later +events visibly (`vendor-not-loaded`, `vendor-timeout`) instead of losing them. +Pushes made while the script loads wait for it, and each script is injected once +per page. diff --git a/.changeset/web-source-browser-flush-on-hide.md b/.changeset/web-source-browser-flush-on-hide.md new file mode 100644 index 000000000..2136466bd --- /dev/null +++ b/.changeset/web-source-browser-flush-on-hide.md @@ -0,0 +1,7 @@ +--- +'@walkeros/web-source-browser': patch +--- + +New: the source sends `walker flush` when the page goes away (`pagehide`, or the +document turning hidden), so batched events are delivered. A failing `elbLayer` +entry now logs one ERROR naming its event, never its data, and counts as failed. diff --git a/.changeset/web-source-datalayer-window.md b/.changeset/web-source-datalayer-window.md new file mode 100644 index 000000000..1021e4b0f --- /dev/null +++ b/.changeset/web-source-datalayer-window.md @@ -0,0 +1,7 @@ +--- +'@walkeros/web-source-datalayer': patch +--- + +Works without an injected `window`: the source reads the page's own, so the +documented integration captures `dataLayer.push` calls in integrated and bundled +mode alike, with no `env` needed. In tests, `env.window` replaces it. diff --git a/packages/mcps/SKILL.md b/packages/mcps/SKILL.md index 04bc19639..80b3da60b 100644 --- a/packages/mcps/SKILL.md +++ b/packages/mcps/SKILL.md @@ -53,7 +53,7 @@ register the same code on its own server. | `walkeros://reference/variables` | Variable patterns ($var/$env/$code/$store) | | `walkeros://reference/contract` | Event schemas, wildcards, inheritance | | `walkeros://reference/examples` | A complete, tested flow config (flow-complete.json) | -| `walkeros://reference/packages` | Full package catalog | +| `walkeros://reference/packages` | Full package catalog, `{ catalog, count, warnings? }` | | `walkeros://schema/{packageName}` | Per-package JSON schemas | ## Prompts diff --git a/packages/server/destinations/amplitude/README.md b/packages/server/destinations/amplitude/README.md index f872b4f7c..168d8d60a 100644 --- a/packages/server/destinations/amplitude/README.md +++ b/packages/server/destinations/amplitude/README.md @@ -32,7 +32,7 @@ npm install @walkeros/server-destination-amplitude "destinations": { "amplitude": { "package": "@walkeros/server-destination-amplitude", - "config": {} + "config": { "settings": { "apiKey": "YOUR_AMPLITUDE_API_KEY" } } } } } diff --git a/packages/server/destinations/bing/README.md b/packages/server/destinations/bing/README.md index 1b18af105..2b3bcb5ac 100644 --- a/packages/server/destinations/bing/README.md +++ b/packages/server/destinations/bing/README.md @@ -29,7 +29,15 @@ npm install @walkeros/server-destination-bing "default": { "config": { "platform": "server" }, "destinations": { - "bing": { "package": "@walkeros/server-destination-bing", "config": {} } + "bing": { + "package": "@walkeros/server-destination-bing", + "config": { + "settings": { + "accessToken": "YOUR_ACCESS_TOKEN", + "tagId": "YOUR_UET_TAG_ID" + } + } + } } } } diff --git a/packages/server/destinations/criteo/README.md b/packages/server/destinations/criteo/README.md index b2b204c6e..075e13971 100644 --- a/packages/server/destinations/criteo/README.md +++ b/packages/server/destinations/criteo/README.md @@ -30,7 +30,12 @@ npm install @walkeros/server-destination-criteo "destinations": { "criteo": { "package": "@walkeros/server-destination-criteo", - "config": {} + "config": { + "settings": { + "partnerId": "YOUR_PARTNER_ID", + "callerId": "YOUR_CALLER_ID" + } + } } } } diff --git a/packages/server/destinations/customerio/README.md b/packages/server/destinations/customerio/README.md index 75dd591f9..afbde4ffb 100644 --- a/packages/server/destinations/customerio/README.md +++ b/packages/server/destinations/customerio/README.md @@ -32,7 +32,9 @@ npm install @walkeros/server-destination-customerio "destinations": { "customerio": { "package": "@walkeros/server-destination-customerio", - "config": {} + "config": { + "settings": { "siteId": "YOUR_SITE_ID", "apiKey": "YOUR_API_KEY" } + } } } } diff --git a/packages/server/destinations/datamanager/README.md b/packages/server/destinations/datamanager/README.md index 375c71222..5bf3f09c9 100644 --- a/packages/server/destinations/datamanager/README.md +++ b/packages/server/destinations/datamanager/README.md @@ -32,7 +32,19 @@ npm install @walkeros/server-destination-datamanager "destinations": { "datamanager": { "package": "@walkeros/server-destination-datamanager", - "config": {} + "config": { + "settings": { + "destinations": [ + { + "operatingAccount": { + "accountId": "123-456-7890", + "accountType": "GOOGLE_ADS" + }, + "productDestinationId": "YOUR_CONVERSION_ACTION_ID" + } + ] + } + } } } } diff --git a/packages/server/destinations/file/README.md b/packages/server/destinations/file/README.md index d21ad3e41..3ae961d4a 100644 --- a/packages/server/destinations/file/README.md +++ b/packages/server/destinations/file/README.md @@ -29,7 +29,10 @@ npm install @walkeros/server-destination-file "default": { "config": { "platform": "server" }, "destinations": { - "file": { "package": "@walkeros/server-destination-file", "config": {} } + "file": { + "package": "@walkeros/server-destination-file", + "config": { "settings": { "filename": "events.jsonl" } } + } } } } diff --git a/packages/server/destinations/gcp/README.md b/packages/server/destinations/gcp/README.md index 2b00a0b79..f6af78c3a 100644 --- a/packages/server/destinations/gcp/README.md +++ b/packages/server/destinations/gcp/README.md @@ -29,7 +29,10 @@ npm install @walkeros/server-destination-gcp "default": { "config": { "platform": "server" }, "destinations": { - "gcp": { "package": "@walkeros/server-destination-gcp", "config": {} } + "gcp": { + "package": "@walkeros/server-destination-gcp", + "config": { "settings": { "projectId": "YOUR_PROJECT_ID" } } + } } } } diff --git a/packages/server/destinations/hubspot/README.md b/packages/server/destinations/hubspot/README.md index 49fadfc09..8391a8074 100644 --- a/packages/server/destinations/hubspot/README.md +++ b/packages/server/destinations/hubspot/README.md @@ -30,7 +30,12 @@ npm install @walkeros/server-destination-hubspot "destinations": { "hubspot": { "package": "@walkeros/server-destination-hubspot", - "config": {} + "config": { + "settings": { + "accessToken": "YOUR_HUBSPOT_ACCESS_TOKEN", + "eventNamePrefix": "pe12345678_" + } + } } } } diff --git a/packages/server/destinations/kafka/README.md b/packages/server/destinations/kafka/README.md index a35e33c78..b14384ee4 100644 --- a/packages/server/destinations/kafka/README.md +++ b/packages/server/destinations/kafka/README.md @@ -31,7 +31,14 @@ npm install @walkeros/server-destination-kafka "destinations": { "kafka": { "package": "@walkeros/server-destination-kafka", - "config": {} + "config": { + "settings": { + "kafka": { + "brokers": ["localhost:9092"], + "topic": "walkeros-events" + } + } + } } } } diff --git a/packages/server/destinations/klaviyo/README.md b/packages/server/destinations/klaviyo/README.md index fe75773aa..1f5231a8b 100644 --- a/packages/server/destinations/klaviyo/README.md +++ b/packages/server/destinations/klaviyo/README.md @@ -30,7 +30,7 @@ npm install @walkeros/server-destination-klaviyo "destinations": { "klaviyo": { "package": "@walkeros/server-destination-klaviyo", - "config": {} + "config": { "settings": { "apiKey": "YOUR_API_KEY" } } } } } diff --git a/packages/server/destinations/linkedin/README.md b/packages/server/destinations/linkedin/README.md index 8bf49453d..8fd2e5a90 100644 --- a/packages/server/destinations/linkedin/README.md +++ b/packages/server/destinations/linkedin/README.md @@ -32,7 +32,12 @@ npm install @walkeros/server-destination-linkedin "destinations": { "linkedin": { "package": "@walkeros/server-destination-linkedin", - "config": {} + "config": { + "settings": { + "accessToken": "YOUR_ACCESS_TOKEN", + "conversionRuleId": "YOUR_CONVERSION_RULE_ID" + } + } } } } diff --git a/packages/server/destinations/mixpanel/README.md b/packages/server/destinations/mixpanel/README.md index 3c908e387..3bcda0956 100644 --- a/packages/server/destinations/mixpanel/README.md +++ b/packages/server/destinations/mixpanel/README.md @@ -32,7 +32,7 @@ npm install @walkeros/server-destination-mixpanel "destinations": { "mixpanel": { "package": "@walkeros/server-destination-mixpanel", - "config": {} + "config": { "settings": { "apiKey": "YOUR_PROJECT_TOKEN" } } } } } diff --git a/packages/server/destinations/mparticle/README.md b/packages/server/destinations/mparticle/README.md index 1c513507d..00e10c954 100644 --- a/packages/server/destinations/mparticle/README.md +++ b/packages/server/destinations/mparticle/README.md @@ -32,7 +32,12 @@ npm install @walkeros/server-destination-mparticle "destinations": { "mparticle": { "package": "@walkeros/server-destination-mparticle", - "config": {} + "config": { + "settings": { + "apiKey": "YOUR_API_KEY", + "apiSecret": "YOUR_API_SECRET" + } + } } } } diff --git a/packages/server/destinations/pinterest/README.md b/packages/server/destinations/pinterest/README.md index 82ff869bd..ef288f63a 100644 --- a/packages/server/destinations/pinterest/README.md +++ b/packages/server/destinations/pinterest/README.md @@ -33,7 +33,12 @@ npm install @walkeros/server-destination-pinterest "destinations": { "pinterest": { "package": "@walkeros/server-destination-pinterest", - "config": {} + "config": { + "settings": { + "accessToken": "YOUR_ACCESS_TOKEN", + "adAccountId": "YOUR_AD_ACCOUNT_ID" + } + } } } } diff --git a/packages/server/destinations/posthog/README.md b/packages/server/destinations/posthog/README.md index ab7ca470d..b5e82988b 100644 --- a/packages/server/destinations/posthog/README.md +++ b/packages/server/destinations/posthog/README.md @@ -32,7 +32,7 @@ npm install @walkeros/server-destination-posthog "destinations": { "posthog": { "package": "@walkeros/server-destination-posthog", - "config": {} + "config": { "settings": { "apiKey": "YOUR_API_KEY" } } } } } diff --git a/packages/server/destinations/reddit/README.md b/packages/server/destinations/reddit/README.md index 707b8f38f..bc2a99ac9 100644 --- a/packages/server/destinations/reddit/README.md +++ b/packages/server/destinations/reddit/README.md @@ -32,7 +32,12 @@ npm install @walkeros/server-destination-reddit "destinations": { "reddit": { "package": "@walkeros/server-destination-reddit", - "config": {} + "config": { + "settings": { + "accessToken": "YOUR_ACCESS_TOKEN", + "pixelId": "YOUR_PIXEL_ID" + } + } } } } diff --git a/packages/server/destinations/redis/README.md b/packages/server/destinations/redis/README.md index d6d2cf6ac..e508fcdbe 100644 --- a/packages/server/destinations/redis/README.md +++ b/packages/server/destinations/redis/README.md @@ -32,7 +32,14 @@ npm install @walkeros/server-destination-redis "destinations": { "redis": { "package": "@walkeros/server-destination-redis", - "config": {} + "config": { + "settings": { + "redis": { + "streamKey": "walkeros:events", + "url": "redis://localhost:6379" + } + } + } } } } diff --git a/packages/server/destinations/rudderstack/README.md b/packages/server/destinations/rudderstack/README.md index f0ca38526..c08602d67 100644 --- a/packages/server/destinations/rudderstack/README.md +++ b/packages/server/destinations/rudderstack/README.md @@ -33,7 +33,12 @@ npm install @walkeros/server-destination-rudderstack "destinations": { "rudderstack": { "package": "@walkeros/server-destination-rudderstack", - "config": {} + "config": { + "settings": { + "writeKey": "YOUR_WRITE_KEY", + "dataPlaneUrl": "https://your-dataplane.example.com" + } + } } } } diff --git a/packages/server/destinations/segment/README.md b/packages/server/destinations/segment/README.md index fecd2713b..b17f07af0 100644 --- a/packages/server/destinations/segment/README.md +++ b/packages/server/destinations/segment/README.md @@ -32,7 +32,7 @@ npm install @walkeros/server-destination-segment "destinations": { "segment": { "package": "@walkeros/server-destination-segment", - "config": {} + "config": { "settings": { "writeKey": "YOUR_WRITE_KEY" } } } } } diff --git a/packages/server/destinations/slack/README.md b/packages/server/destinations/slack/README.md index 2b27dc03e..ba88bac9a 100644 --- a/packages/server/destinations/slack/README.md +++ b/packages/server/destinations/slack/README.md @@ -31,7 +31,11 @@ npm install @walkeros/server-destination-slack "destinations": { "slack": { "package": "@walkeros/server-destination-slack", - "config": {} + "config": { + "settings": { + "webhookUrl": "https://hooks.slack.com/services/YOUR/WEBHOOK/URL" + } + } } } } diff --git a/packages/server/destinations/snapchat/README.md b/packages/server/destinations/snapchat/README.md index 0946cf76a..ceb1206d7 100644 --- a/packages/server/destinations/snapchat/README.md +++ b/packages/server/destinations/snapchat/README.md @@ -33,7 +33,12 @@ npm install @walkeros/server-destination-snapchat "destinations": { "snapchat": { "package": "@walkeros/server-destination-snapchat", - "config": {} + "config": { + "settings": { + "accessToken": "YOUR_ACCESS_TOKEN", + "pixelId": "YOUR_PIXEL_ID" + } + } } } } diff --git a/packages/server/destinations/sqlite/README.md b/packages/server/destinations/sqlite/README.md index 9698a1647..4d4aa29bb 100644 --- a/packages/server/destinations/sqlite/README.md +++ b/packages/server/destinations/sqlite/README.md @@ -31,7 +31,7 @@ npm install @walkeros/server-destination-sqlite "destinations": { "sqlite": { "package": "@walkeros/server-destination-sqlite", - "config": {} + "config": { "settings": { "sqlite": { "url": "./events.db" } } } } } } diff --git a/packages/server/destinations/tiktok/README.md b/packages/server/destinations/tiktok/README.md index 10e3c9389..445a6af58 100644 --- a/packages/server/destinations/tiktok/README.md +++ b/packages/server/destinations/tiktok/README.md @@ -32,7 +32,12 @@ npm install @walkeros/server-destination-tiktok "destinations": { "tiktok": { "package": "@walkeros/server-destination-tiktok", - "config": {} + "config": { + "settings": { + "pixelCode": "YOUR_PIXEL_CODE", + "accessToken": "YOUR_ACCESS_TOKEN" + } + } } } } diff --git a/packages/server/destinations/twitter/README.md b/packages/server/destinations/twitter/README.md index 7128a655a..4b1266b24 100644 --- a/packages/server/destinations/twitter/README.md +++ b/packages/server/destinations/twitter/README.md @@ -31,7 +31,16 @@ npm install @walkeros/server-destination-twitter "destinations": { "twitter": { "package": "@walkeros/server-destination-twitter", - "config": {} + "config": { + "settings": { + "pixelId": "YOUR_PIXEL_ID", + "eventId": "tw-xxxxx-xxxxx", + "consumerKey": "YOUR_CONSUMER_KEY", + "consumerSecret": "YOUR_CONSUMER_SECRET", + "accessToken": "YOUR_ACCESS_TOKEN", + "accessTokenSecret": "YOUR_ACCESS_TOKEN_SECRET" + } + } } } } diff --git a/packages/server/stores/fs/README.md b/packages/server/stores/fs/README.md index d73495717..e66596d1c 100644 --- a/packages/server/stores/fs/README.md +++ b/packages/server/stores/fs/README.md @@ -28,7 +28,10 @@ npm install @walkeros/server-store-fs "default": { "config": { "platform": "server" }, "stores": { - "assets": { "package": "@walkeros/server-store-fs", "config": {} } + "assets": { + "package": "@walkeros/server-store-fs", + "config": { "settings": { "basePath": "./assets" } } + } } } } diff --git a/packages/server/stores/gcs/README.md b/packages/server/stores/gcs/README.md index ea424018c..49f5836b6 100644 --- a/packages/server/stores/gcs/README.md +++ b/packages/server/stores/gcs/README.md @@ -29,7 +29,10 @@ npm install @walkeros/server-store-gcs "default": { "config": { "platform": "server" }, "stores": { - "assets": { "package": "@walkeros/server-store-gcs", "config": {} } + "assets": { + "package": "@walkeros/server-store-gcs", + "config": { "settings": { "bucket": "my-assets" } } + } } } } diff --git a/packages/server/stores/s3/README.md b/packages/server/stores/s3/README.md index 951fec1e2..14545ae93 100644 --- a/packages/server/stores/s3/README.md +++ b/packages/server/stores/s3/README.md @@ -29,7 +29,17 @@ npm install @walkeros/server-store-s3 "default": { "config": { "platform": "server" }, "stores": { - "assets": { "package": "@walkeros/server-store-s3", "config": {} } + "assets": { + "package": "@walkeros/server-store-s3", + "config": { + "settings": { + "bucket": "my-assets", + "endpoint": "https://s3.eu-west-1.amazonaws.com", + "accessKeyId": "YOUR_ACCESS_KEY", + "secretAccessKey": "YOUR_SECRET_KEY" + } + } + } } } } diff --git a/packages/server/stores/sheets/README.md b/packages/server/stores/sheets/README.md index 8daa5dab2..f9cab20c3 100644 --- a/packages/server/stores/sheets/README.md +++ b/packages/server/stores/sheets/README.md @@ -30,7 +30,10 @@ npm install @walkeros/server-store-sheets "default": { "config": { "platform": "server" }, "stores": { - "assets": { "package": "@walkeros/server-store-sheets", "config": {} } + "assets": { + "package": "@walkeros/server-store-sheets", + "config": { "settings": { "id": "YOUR_SPREADSHEET_ID" } } + } } } } diff --git a/packages/web/destinations/gtag/README.md b/packages/web/destinations/gtag/README.md index fff62a5bb..309f0c65a 100644 --- a/packages/web/destinations/gtag/README.md +++ b/packages/web/destinations/gtag/README.md @@ -33,7 +33,9 @@ npm install @walkeros/web-destination-gtag "destinations": { "gtag": { "package": "@walkeros/web-destination-gtag", - "config": {} + "config": { + "settings": { "ga4": { "measurementId": "G-XXXXXXXXXX" } } + } } } } diff --git a/packages/web/destinations/linkedin/README.md b/packages/web/destinations/linkedin/README.md index 1d411092c..bb3ff61b1 100644 --- a/packages/web/destinations/linkedin/README.md +++ b/packages/web/destinations/linkedin/README.md @@ -32,7 +32,7 @@ npm install @walkeros/web-destination-linkedin "destinations": { "linkedin": { "package": "@walkeros/web-destination-linkedin", - "config": {} + "config": { "settings": { "apiKey": "YOUR_PARTNER_ID" } } } } } diff --git a/packages/web/destinations/meta/README.md b/packages/web/destinations/meta/README.md index e2dec6d9b..3bf3bbc6d 100644 --- a/packages/web/destinations/meta/README.md +++ b/packages/web/destinations/meta/README.md @@ -32,7 +32,7 @@ npm install @walkeros/web-destination-meta "destinations": { "meta": { "package": "@walkeros/web-destination-meta", - "config": {} + "config": { "settings": { "pixelId": "YOUR_PIXEL_ID" } } } } } diff --git a/packages/web/destinations/optimizely/README.md b/packages/web/destinations/optimizely/README.md index e59419dd7..adbc04941 100644 --- a/packages/web/destinations/optimizely/README.md +++ b/packages/web/destinations/optimizely/README.md @@ -32,7 +32,7 @@ npm install @walkeros/web-destination-optimizely "destinations": { "optimizely": { "package": "@walkeros/web-destination-optimizely", - "config": {} + "config": { "settings": { "sdkKey": "YOUR_SDK_KEY" } } } } } diff --git a/packages/web/destinations/pinterest/README.md b/packages/web/destinations/pinterest/README.md index 9f35f353d..4c6d4264c 100644 --- a/packages/web/destinations/pinterest/README.md +++ b/packages/web/destinations/pinterest/README.md @@ -33,7 +33,7 @@ npm install @walkeros/web-destination-pinterest "destinations": { "pinterest": { "package": "@walkeros/web-destination-pinterest", - "config": {} + "config": { "settings": { "apiKey": "YOUR_TAG_ID" } } } } } diff --git a/packages/web/destinations/snowplow/README.md b/packages/web/destinations/snowplow/README.md index 9110408c9..dfef24d37 100644 --- a/packages/web/destinations/snowplow/README.md +++ b/packages/web/destinations/snowplow/README.md @@ -32,7 +32,9 @@ npm install @walkeros/web-destination-snowplow "destinations": { "snowplow": { "package": "@walkeros/web-destination-snowplow", - "config": {} + "config": { + "settings": { "collectorUrl": "https://collector.example.com" } + } } } } diff --git a/packages/web/destinations/tiktok/README.md b/packages/web/destinations/tiktok/README.md index b6f158aef..bc0eb0a30 100644 --- a/packages/web/destinations/tiktok/README.md +++ b/packages/web/destinations/tiktok/README.md @@ -32,7 +32,7 @@ npm install @walkeros/web-destination-tiktok "destinations": { "tiktok": { "package": "@walkeros/web-destination-tiktok", - "config": {} + "config": { "settings": { "apiKey": "YOUR_PIXEL_ID" } } } } } diff --git a/packages/web/sources/demo/README.md b/packages/web/sources/demo/README.md index f54f5f74d..d11a89d4f 100644 --- a/packages/web/sources/demo/README.md +++ b/packages/web/sources/demo/README.md @@ -28,7 +28,10 @@ npm install @walkeros/source-demo "default": { "config": { "platform": "web" }, "sources": { - "demo": { "package": "@walkeros/source-demo", "config": {} } + "demo": { + "package": "@walkeros/source-demo", + "config": { "settings": { "events": [{ "name": "page view" }] } } + } } } } diff --git a/skills/INDEX.json b/skills/INDEX.json index fdcc28b3d..27ebe9201 100644 --- a/skills/INDEX.json +++ b/skills/INDEX.json @@ -41,6 +41,7 @@ "examples/index.ts", "examples/mapping.ts", "examples/outputs.ts", + "templates/simple/conformance.test.ts", "templates/simple/index.test.ts", "templates/simple/index.ts", "templates/simple/types.ts" diff --git a/skills/walkeros-create-cmp-source/SKILL.md b/skills/walkeros-create-cmp-source/SKILL.md index ba9c84904..623446196 100644 --- a/skills/walkeros-create-cmp-source/SKILL.md +++ b/skills/walkeros-create-cmp-source/SKILL.md @@ -776,6 +776,10 @@ requirements (build, test, lint, no `any`): - [ ] Category mapping uses OR logic for many-to-one - [ ] Consent state always includes explicit `false` for denied groups - [ ] `destroy()` cleans up ALL listeners and restores wrapped functions +- [ ] Detection never fails silently: a setting the source needs that is missing + throws a `StepError` (`code: 'config'`) from `init`, and `init` never + returns early or returns `false`; a CMP that is simply absent is not an + error (consent stays unknown) - [ ] README follows required structure - [ ] Consent guide page updated diff --git a/skills/walkeros-create-destination/SKILL.md b/skills/walkeros-create-destination/SKILL.md index 4699134eb..80a6d2b2e 100644 --- a/skills/walkeros-create-destination/SKILL.md +++ b/skills/walkeros-create-destination/SKILL.md @@ -325,6 +325,55 @@ ship **per-tool init examples** — `examples.step.ga4Init`, `adsInit`, `gtmInit ``, and tests pick the right init per sub-tool. +#### Failure, silent and consent examples + +Besides the happy path, `step.ts` carries `public: false` examples that the +conformance kit (Phase 8) runs on a real collector: + +```typescript +// A silent rule sends nothing. +export const silentRule: Flow.StepExample = { + public: false, + description: 'A rule with silent: true sends no event.', + in: getEvent('page view', { timestamp: 1700000006 }), + mapping: { silent: true }, + out: [], +}; + +// An event the destination cannot send is reported, not counted delivered. +export const missingInput: Flow.StepExample = { + public: false, + description: 'Without a user id the push fails with missing-input.', + in: getEvent('user login', { timestamp: 1700000007 }), + mapping: { settings: { identify: { map: { id: 'user.id' } } } }, + out: [], + outcome: { status: 'failed', code: 'missing-input', retryable: false }, +}; + +// A vendor that answers 200 with an error in the body. +export const rejectedByVendor: Flow.StepExample = { + public: false, + description: 'The vendor answers 200 and reports the event as failed.', + in: getEvent('page view', { timestamp: 1700000008 }), + respond: { status: 200, body: { events: [{ status: 'failed' }] } }, + outcome: { status: 'failed', code: 'transport', retryable: false }, +}; + +// A consent revocation reaches the vendor. +export const consentRevoke: Flow.StepExample = { + public: false, + command: 'consent', + in: { marketing: false }, + out: [['window.vendorSdk', 'consent', 'revoke']], +}; +``` + +`outcome` is `{ status: 'failed' | 'queued', code, retryable? }`; `respond` is +what the vendor's boundary answers (`status`, `body`, or +`error: 'network' | 'timeout'`). An example that expects no vendor call has +`out: []`. A "one of these settings" rule (`ga4`, `ads` or `gtm`) is stated as +`anyOf` in the settings schema, not only in `init`. + For destinations, the Three Type Zones collapse to: - `in` = walkerOS event (`WalkerOS.Event`) @@ -551,10 +600,11 @@ Guidelines: Use these templates as your starting point: -| File | Purpose | Template | -| ---------------- | ---------------- | --------------------------------------- | -| `types/index.ts` | Type definitions | [types.ts](./templates/simple/types.ts) | -| `index.ts` | Main destination | [index.ts](./templates/simple/index.ts) | +| File | Purpose | Template | +| ------------------------------- | --------------------- | ------------------------------------------------------------- | +| `types/index.ts` | Type definitions | [types.ts](./templates/simple/types.ts) | +| `index.ts` | Main destination | [index.ts](./templates/simple/index.ts) | +| `__tests__/conformance.test.ts` | Failure-contract test | [conformance.test.ts](./templates/simple/conformance.test.ts) | ### Key Patterns @@ -597,6 +647,39 @@ Use these templates as your starting point: - For step-example tests, use `command: 'consent'` on `Flow.StepExample` to invoke the `on('consent')` handler. Do not push consent data as an event. +8. **A failure is a throw** (the failure contract). A `push` or `init` that + returns normally counts as delivered, so: + - Throw a `StepError` from `@walkeros/core` with a `code` (`config` for a + missing or invalid setting, `missing-input` for an event the destination + cannot send, `transport` for a failed request; add `status` for an HTTP + status, `retryable` to override the default). Never `return` early, never + `return false` from `init`, never log the failure yourself: the collector + writes one ERROR line per cause, a failed counter and a DLQ entry. + - Await every call the vendor makes. A call that is not awaited fails after + the push was already counted as delivered. The lint rules + `walkeros/await-send` and `walkeros/no-silent-return` flag both. + - A response body goes on the error's `body`, never into its message; + messages and log lines never carry event data. + - A value-level problem (one mapped field cannot be built) drops that field, + sends the event and reports once with `reportCause`. Only a wholly + unsendable event throws `missing-input`. +9. **Scripts through `loadVendor`** (`@walkeros/web-core`). It injects the + script once per page and reports `vendor-not-loaded`, `vendor-not-ready` and + `vendor-timeout`. For a vendor that replays its own stub, call `check()` in + `push`; for any other, `await ready()`. Pass attributes the tag needs + (`data-domain`) through `options.attributes`. +10. **Honour `rule.silent`.** A rule with `silent: true` sends no event of its + own and runs only the side effects it maps (identify, group). With nothing + to send it returns normally. Every destination ships a `silent` step example + (see Phase 3). +11. **Consent reaches the vendor.** `config.consent` decides: in + `on('consent')`, tell the vendor about a revocation and a grant when it has + a switch (`getGrantedConsent(config.consent, collector.consent)`), and throw + a `consent-not-applied` StepError when the vendor does not take it. +12. **Batching.** A server SDK that batches implements `pushBatch` and gets a + default `config.batch`; each entry has its own outcome, and + `config.batch: { size: 1 }` is the per-event setting. + ### Reserved fields `config.setup` is reserved for the setup lifecycle (see @@ -769,6 +852,25 @@ Use the test template: [index.test.ts](./templates/simple/index.test.ts). Reference canonical implementation: `packages/web/destinations/gtag/src/__tests__/stepExamples.test.ts`. +### Conformance test + +Every destination has a conformance test that runs it on a real collector with +`conformance('destination', destination, options)` from +`@walkeros/collector/dev` (template: +[conformance.test.ts](./templates/simple/conformance.test.ts)). It checks the +failure contract: a failing send is reported with a cause, a failure never leaks +event data into a log line, a `silent` example sends nothing, a consent +revocation reaches the vendor, `init` throws on a missing setting, and every +event the destination cannot send has a declared outcome. It fails with every +broken item at once. + +The `profile` option says what the destination talks to: `http` (calls `fetch`), +`sdk-env` (a server SDK injected through `env`), `web-sdk` (a web SDK started in +`init`: `sdk-env` without the init check), `vendor-global` (a vendor script that +defines a global), `socket` (a long-lived connection). Never prove delivery or +loss with a mocked collector: `startFlow` and mock only the network or the +vendor global. + ### Key Test Patterns 1. **`it.each(Object.entries(examples.step))` is mandatory** — one iteration per @@ -788,6 +890,10 @@ Reference canonical implementation: - [ ] Tests iterate via `it.each(Object.entries(examples.step))` - [ ] Tests contain no hardcoded payloads, vendor configs, or expected outputs - [ ] Every assertion reads from `examples.step[...].out` +- [ ] The conformance test passes with the right profile +- [ ] `examples.step` has a `silent` example, a failure example (`outcome`) for + each event the destination cannot send, and a consent revocation example + when the vendor has a consent switch --- @@ -824,6 +930,9 @@ requirements (build, test, lint, no `any`): included, and every `exportSchemas` entry has `settings`; the build fails otherwise (see [using-step-examples](../walkeros-using-step-examples/SKILL.md)) +- [ ] No silent path: a failure is a thrown StepError, every vendor call is + awaited, `init` never returns `false`, scripts load through `loadVendor`, + `rule.silent` is honoured - [ ] Examples match type signatures - [ ] Tests use examples for assertions (not hardcoded values) - [ ] `walkerOS.json` generated at build time diff --git a/skills/walkeros-create-destination/templates/simple/conformance.test.ts b/skills/walkeros-create-destination/templates/simple/conformance.test.ts new file mode 100644 index 000000000..38c5e326f --- /dev/null +++ b/skills/walkeros-create-destination/templates/simple/conformance.test.ts @@ -0,0 +1,37 @@ +import path from 'path'; +import { conformance } from '@walkeros/collector/dev'; +import { destinationVendor } from '..'; +import { examples, schemas } from '../dev'; + +/** + * Conformance test template. `conformance` runs the destination on a real + * collector and checks its failure contract. Pick the profile that matches + * the destination: + * + * - `http` calls fetch directly + * - `sdk-env` a server SDK injected through env + * - `web-sdk` a web SDK started in init (sdk-env without the init check) + * - `vendor-global` a vendor script that defines a global (this template) + * - `socket` a long-lived connection + * + * `examples.step` must carry, besides the happy-path examples: a `silent` + * example (`public: false`, `out: []`), a consent revocation example when the + * vendor has a consent switch, and a failure example (`outcome`) for each + * event the destination cannot send. + */ +describe('vendor conformance', () => { + beforeEach(() => { + jest.useRealTimers(); + }); + + it('conforms', async () => { + await conformance('destination', destinationVendor, { + profile: 'vendor-global', + examples: examples.step, + schema: schemas.settings, + settings: { apiKey: 'test-key' }, + env: { push: examples.env.push, simulation: examples.env.simulation }, + root: path.join(__dirname, '..', '..'), + }); + }); +}); diff --git a/skills/walkeros-create-destination/templates/simple/index.test.ts b/skills/walkeros-create-destination/templates/simple/index.test.ts index 7ae3a656a..5e86e56ae 100644 --- a/skills/walkeros-create-destination/templates/simple/index.test.ts +++ b/skills/walkeros-create-destination/templates/simple/index.test.ts @@ -12,6 +12,11 @@ import { events, outputs, env as mockEnv } from '../../examples'; * 2. Include id field - required in context * 3. Use rule instead of mapping - property renamed in PushContext * 4. Use examples for test data - don't hardcode test values + * 5. Await push: it returns a promise and a failure is a rejection + * + * The failure contract (a failing send is reported, a silent rule sends + * nothing, consent reaches the vendor) is proven by conformance.test.ts on a + * real collector. Do not prove delivery or loss with a mocked collector. */ // Helper to create push context for testing @@ -32,7 +37,7 @@ describe('destinationVendor', () => { jest.clearAllMocks(); }); - test('page view produces correct output', () => { + test('page view produces correct output', async () => { const mockSdk = jest.fn(); const context = createPushContext({ env: { @@ -42,7 +47,7 @@ describe('destinationVendor', () => { data: { url: '/home', title: 'Home Page' }, }); - destinationVendor.push(events.pageView, context); + await destinationVendor.push(events.pageView, context); // Verify against expected output from examples expect(mockSdk).toHaveBeenCalledWith( @@ -51,7 +56,7 @@ describe('destinationVendor', () => { ); }); - test('purchase produces correct output', () => { + test('purchase produces correct output', async () => { const mockSdk = jest.fn(); const context = createPushContext({ env: { @@ -66,7 +71,7 @@ describe('destinationVendor', () => { }, }); - destinationVendor.push(events.purchase, context); + await destinationVendor.push(events.purchase, context); expect(mockSdk).toHaveBeenCalledWith( outputs.purchaseCall.method, @@ -74,7 +79,7 @@ describe('destinationVendor', () => { ); }); - test('custom event produces correct output', () => { + test('custom event produces correct output', async () => { const mockSdk = jest.fn(); const context = createPushContext({ env: { @@ -84,7 +89,7 @@ describe('destinationVendor', () => { data: { button_id: 'cta', button_text: 'Sign Up' }, }); - destinationVendor.push(events.buttonClick, context); + await destinationVendor.push(events.buttonClick, context); expect(mockSdk).toHaveBeenCalledWith( outputs.customEventCall.method, diff --git a/skills/walkeros-create-destination/templates/simple/index.ts b/skills/walkeros-create-destination/templates/simple/index.ts index 0fe1f96f7..da20cb29b 100644 --- a/skills/walkeros-create-destination/templates/simple/index.ts +++ b/skills/walkeros-create-destination/templates/simple/index.ts @@ -1,80 +1,123 @@ -import type { Destination, Env, Settings } from './types'; -import { isObject } from '@walkeros/core'; -import { getEnv } from '@walkeros/web-core'; +import type { Config, Destination, Env } from './types'; +import { getGrantedConsent, isObject, StepError } from '@walkeros/core'; +import { getEnv, loadVendor } from '@walkeros/web-core'; export * as DestinationVendor from './types'; +const SCRIPT_SRC = 'https://vendor.com/sdk.js'; + /** - * Simple destination template. + * Vendor destination template (web, vendor script that defines a global). * * Key patterns: - * 1. Init receives context - destructure config, env, logger, id - * 2. Push receives context - includes data, rule (renamed from mapping), ingest - * 3. Use getEnv(env) - never access window/document directly, and never - * cast them: the generic returns your narrowed Env merged with the DOM - * globals, so window.vendorSdk / document.createElement are already typed - * 4. Return config from init - allows updating config during initialization + * 1. A failure is a throw. Never `return` early when nothing was sent, and + * never log a failure yourself: the collector turns a thrown StepError + * into a failed push, a DLQ entry and one ERROR line. + * 2. Await every call the vendor makes; a call that is not awaited fails after + * the push was already counted as delivered. + * 3. Load scripts with `loadVendor`: a blocked or slow script becomes + * `vendor-not-loaded` / `vendor-timeout` instead of lost events. + * 4. Honour `rule.silent`: send no event of its own, keep the side effects. + * 5. Pass consent to the vendor in `on('consent')` when the vendor has a + * switch for it; `config.consent` decides. + * 6. Use getEnv(env) - never access window/document directly, and never + * cast them. */ + +/** The script `loadVendor` injects with `loadScript`; `check()` reports a failed one. */ +function vendor(config: Config, env?: Env) { + return loadVendor( + config.loadScript ? SCRIPT_SRC : undefined, + () => getEnv(env).window.vendorSdk, + { timeout: config.timeout }, + ); +} + +/** The SDK global, or a StepError when it is not there. */ +function sdk(env?: Env) { + const { vendorSdk } = getEnv(env).window; + if (!vendorSdk) + throw new StepError('Vendor SDK not ready: window.vendorSdk is missing', { + code: 'vendor-not-ready', + }); + return vendorSdk; +} + +/** Runs one vendor call; a throw or a rejection becomes a StepError of `code`. */ +async function call( + fn: () => unknown, + code: string, + message: string, +): Promise { + try { + await fn(); + } catch (error) { + throw new StepError(message, { code, cause: error }); + } +} + export const destinationVendor: Destination = { type: 'vendor', config: {}, /** - * Initialize destination - receives context object. - * - * @param context - Init context containing: - * - config: Destination configuration (settings, mapping, etc.) - * - env: Environment with window, document, etc. - * - logger: Logger instance - * - id: Unique destination identifier - * - collector: Collector instance reference - * - data: Pre-computed data from mapping + * Initialize destination. Throw a StepError when it cannot start; never + * return `false`. A destination with a consent requirement is initialized + * once consent is granted. */ - init(context) { - const { config, env } = context; - const { window } = getEnv(env); - const settings = config.settings || {}; + async init({ config, env }) { + const { apiKey } = config.settings || {}; + if (!apiKey) + throw new StepError('Vendor needs settings.apiKey', { code: 'config' }); - if (config.loadScript) addScript(settings, env); - - // Initialize vendor SDK queue + // The SDK stub first, so calls queue until the vendor script replays them + const { window } = getEnv(env); window.vendorSdk = window.vendorSdk || function () { (window.vendorSdk!.q = window.vendorSdk!.q || []).push(arguments); }; + vendor(config, env); - return config; + await call( + () => sdk(env)('init', apiKey), + 'vendor-not-ready', + 'Vendor init call failed', + ); }, /** - * Push event to destination - receives event and push context. - * - * @param event - The walkerOS event to send - * @param context - Push context containing: - * - config: Destination configuration - * - env: Environment - * - logger: Logger instance - * - id: Destination identifier - * - data: Pre-computed data from mapping - * - rule: The matching mapping rule (renamed from 'mapping') - * - ingest: Optional request metadata from source + * Push event to destination. `data` is the mapped data, `rule` the matching + * mapping rule. */ - push(event, context) { - const { data, env } = context; + async push(event, { config, rule, data, env }) { + // A script that fired `error` fails every later push + vendor(config, env).check(); + + // A silent rule sends no event of its own + if (rule?.silent) return; + const params = isObject(data) ? data : {}; - const { window } = getEnv(env); + await call( + () => sdk(env)('track', event.name, params), + 'transport', + 'Vendor call failed', + ); + }, + + /** Tell the vendor about a consent change when it has a switch for it. */ + async on(type, { config, data, env, collector }) { + const required = config.consent; + if (type !== 'consent' || !required || !isObject(data)) return; + if (!Object.keys(required).some((key) => key in data)) return; - // Call vendor API - must match outputs.ts examples - window.vendorSdk!('track', event.name, params); + const granted = getGrantedConsent(required, collector.consent) !== false; + await call( + () => sdk(env)('consent', granted ? 'grant' : 'revoke'), + 'consent-not-applied', + 'Vendor did not take the consent update', + ); }, }; -function addScript(settings: Settings, env?: Env) { - const { document } = getEnv(env); - const script = document.createElement('script'); - script.src = `https://vendor.com/sdk.js?key=${settings.apiKey}`; - document.head.appendChild(script); -} - export default destinationVendor; diff --git a/skills/walkeros-create-source/SKILL.md b/skills/walkeros-create-source/SKILL.md index 16b11f8d9..e4a3bd3fa 100644 --- a/skills/walkeros-create-source/SKILL.md +++ b/skills/walkeros-create-source/SKILL.md @@ -666,10 +666,23 @@ for the complete contract. 4. **Platform deps via env**: All platform dependencies (window, document, express, cors) must go through `env` with fallback to globals/imports: `env.express ?? express`. This enables testing without mocking globals. -5. **Error logging**: Use `logger?.error()` for errors only, not routine - operations -6. **Return Source.Instance**: Return `{ type, config, push }` object -7. **Optional `destroy` method**: Implement if the source holds resources (HTTP +5. **Failures are reported, not logged by the source**: a source that cannot + start throws a `StepError` (`code: 'config'`) from its factory or `init`; it + never returns early and never returns `false`. Input the collector rejects + comes back as `invalid` in the push result and is counted as `rejected` by + the collector. A failure on the source's own side that the push result does + not carry (for example an `elbLayer` entry that fails) goes through + `reportCause` from `@walkeros/core` (step `source.`, named by its event, + never its payload). Use `logger` for `debug` traces of routine operations. +6. **Tell the producer the truth**: an HTTP source builds its answer from the + push result with `pushResultToOutcome` (or `acknowledge(result)`): `ack` + answers 200 with today's body, `retry` (a retryable failure) answers 500, or + 207 for the failed events of a batch, `reject` (invalid input) answers 400. A + failure a resend cannot fix is acknowledged, because redelivery would loop. A + queue consumer acks on `ack` and `reject` and nacks only on `retry`. The body + never contains a destination id or a step's error text. +7. **Return Source.Instance**: Return `{ type, config, push }` object +8. **Optional `destroy` method**: Implement if the source holds resources (HTTP servers, timers, connections) that need cleanup on shutdown ### Gate: Implementation Compiles @@ -833,8 +846,14 @@ Canonical references: 4. **Zero hardcoded payloads** - every trigger type, request body, DOM HTML, and expected event comes from `examples.step` or `examples.env`. If you need something new, add it to examples first. -5. **Test error paths** - verify graceful error handling and logging for invalid - input (add an error example to `examples.step` if needed). +5. **Test error paths** - verify graceful error handling for invalid input (add + an error example to `examples.step` if needed), and that a push that fails + for a retryable reason is answered with a retry, a deterministic one with an + ack. +6. **Conformance** - run the source with + `conformance('source', source, options)` from `@walkeros/collector/dev` + (profile `source`, `createTrigger` for how to feed it); it runs on a real + collector. ### Gate: Tests Pass diff --git a/skills/walkeros-create-transformer/SKILL.md b/skills/walkeros-create-transformer/SKILL.md index d8147290a..4aae8c5cb 100644 --- a/skills/walkeros-create-transformer/SKILL.md +++ b/skills/walkeros-create-transformer/SKILL.md @@ -296,7 +296,14 @@ complete implementation example. [templates/validation/schemas.ts](templates/validation/schemas.ts)) 3. **Push receives pushContext**: The `push` function gets event + push context 4. **Return values**: `{ event }` (continue), `void` (passthrough), `false` - (cancel) + (cancel on purpose: a deliberate drop, not a failure) +5. **A failure is a throw**: throw a `StepError` from `@walkeros/core` with a + `code`. `missing-input` is the code for an event or an ingest that lacks the + input the transformer needs (do not drop it silently); `config` for a missing + setting. `init` throws instead of returning `false` or returning early, and a + transformer never logs a failure itself: the collector turns the throw into + `ok: false` with the transformer named and writes the one ERROR line. A + transformer that throws a plain error fails with the code `transformer`. ### 5.3 Export @@ -326,6 +333,11 @@ a complete test suite showing: 2. **`createPushContext()` helper** - Standardizes push context creation 3. **Examples for test data** - Don't hardcode test values 4. **Return value testing** - Verify `event`, `void`, or `false` returns +5. **Conformance** - `conformance('transformer', transformer, options)` from + `@walkeros/collector/dev` (profile `transformer`) runs it on a real + collector. A transformer whose input is the ingest and not an event declares + its failure with a `missing-input` example (an event as `in`, `out: []`, a + description). ### Gate: Tests Pass @@ -367,6 +379,8 @@ requirements (build, test, lint, no `any`): [using-step-examples](../walkeros-using-step-examples/SKILL.md)) - [ ] Examples include before/after event pairs - [ ] Return values handle all cases (event, void, false) +- [ ] A failure is a thrown `StepError` (`missing-input` for absent input), + `init` throws instead of returning `false`, conformance test passes - [ ] Tests use examples for assertions (not hardcoded values) - [ ] `walkerOS.json` generated at build time - [ ] `walkerOS` field in package.json diff --git a/skills/walkeros-testing-strategy/SKILL.md b/skills/walkeros-testing-strategy/SKILL.md index e7044dd6a..ef799e2fa 100644 --- a/skills/walkeros-testing-strategy/SKILL.md +++ b/skills/walkeros-testing-strategy/SKILL.md @@ -110,6 +110,30 @@ the code. **Fix:** Test what the code actually does. If external APIs must be mocked, verify the real API would receive correct data. +### Rule 3b: Prove Delivery, Loss and Failures on a Real Collector + +A test or repro that claims an event was delivered, lost or reported as failed +starts a real collector with `startFlow` and mocks only the network or the +vendor global (through `env`), never `collector.push`. A suite that mocks the +collector cannot see loss: a destination that returns early, swallows an error +or forgets an `await` passes it. Assert on what a user sees at default settings: +the push result (`ok: false`, the destination under `failed` with its `reason`), +the status counters, or the ERROR line the collector wrote. + +Failure paths are examples too. A step package's failure modes (a missing +required setting, a failing transport, a vendor script that is not loaded yet) +are `public: false` step examples with an `outcome` (and `respond` for a vendor +answer), not only the happy path. A `silent` rule and a consent revocation get +examples as well. + +**Use the conformance kit.** `conformance(kind, step, options)` from +`@walkeros/collector/dev` runs a destination, source, transformer or store on a +real collector against its step examples and checks the failure contract in one +call. The `profile` states what the step talks to: for destinations `http`, +`sdk-env`, `web-sdk` (a web SDK started in `init`), `vendor-global` or `socket`; +`transformer` and `source` for those kinds. `runExamples` runs the examples +alone. + ### Rule 4: Test First, Watch It Fail If you didn't see the test fail, you don't know it tests the right thing. @@ -382,6 +406,9 @@ approach destinations use — see - Using `jest.mock()` for internal modules when `env` pattern is available - Tests that don't import from `../dev` - Assertions only checking mock call counts +- Proving delivery or loss with a mocked `collector.push` instead of `startFlow` +- A destination without a conformance test, a `silent` example or a failure + example - Tests with extensive mock setup (>50% of test is setup) - Test-only methods added to production classes - Claiming tests pass without running them diff --git a/skills/walkeros-understanding-destinations/SKILL.md b/skills/walkeros-understanding-destinations/SKILL.md index 8865b76ad..134e76cad 100644 --- a/skills/walkeros-understanding-destinations/SKILL.md +++ b/skills/walkeros-understanding-destinations/SKILL.md @@ -161,6 +161,41 @@ The same bound applies at shutdown, where the collector races each pending batch flush against a fixed 5000 ms step timeout before destroying the destination. A destination whose retries need longer than that loses them on shutdown. +## Outcomes and failure reasons + +A push ends in one place per destination in the `PushResult`: `done`, `queued` +(reason `consent`, `batch` or `held`) or `failed`. A failed entry carries a +`reason` `{ code, message, retryable, status? }` that the collector derives from +the thrown value: a `StepError` keeps its `code`, any other throw is +`transport`. Codes the collector assigns itself: `timeout`, `not-ready` (a +destination whose `init` could not start or returned `false`), `breaker-open`, +`blocked` (held past one retry window), `queue-full`, `mapping` (a rule +`condition` threw), `transformer`. Codes a destination throws: `config`, +`missing-input`, `vendor-not-loaded`, `vendor-not-ready`, `vendor-timeout`, +`transport`. `retryable` decides whether a server source asks the producer to +resend (`acknowledge(result)` from `@walkeros/core`: `ack`, `retry` or +`reject`); it never reads message text. + +**Unmet consent is never a failure.** A destination waiting for consent answers +`queued` with the reason `consent`; a server collector skips a consent-denied +event instead (`consentDenied: 'skip'`, counted in +`status.destinations[id].skipped`, a `debug` line). A rule with `ignore: true` +is a skip as well, not a delivery. + +**Failed means thrown.** A destination reports a failure only by throwing a +`StepError`; it never returns early, never returns `false` from `init`, never +logs the failure itself, and awaits every call it makes. The collector writes +one ERROR line per (step, code, status) and minute. Every destination honours +`rule.silent` (no event of its own, side effects still run) and ships a `silent` +example. + +**The conformance kit.** `conformance('destination', destination, options)` from +`@walkeros/collector/dev` runs a destination on a real collector against its +step examples and fails with every broken item of the contract at once. Pick the +`profile` by what the destination talks to: `http`, `sdk-env`, `web-sdk` (a web +SDK started in `init`), `vendor-global`, `socket`. See +[create-destination](../walkeros-create-destination/SKILL.md). + ## Batch scheduling Set `config.batch` on a destination (with `pushBatch` implemented) to batch @@ -168,12 +203,16 @@ Set `config.batch` on a destination (with `pushBatch` implemented) to batch is needed. The configuration shape is `batch?: number | { wait?, size?, age? }` at both the destination-config layer and the mapping-rule layer. -**`pushBatch` is the gate, and a missing one fails silently.** The collector -engages batching only when `destination.pushBatch` exists. Set `config.batch` on -a destination that lacks it and every event is sent one at a time, with no -warning and no error: the config is accepted and inert. When adding batching to -a destination, implementing `pushBatch` is the change; `config.batch` alone buys -nothing. +**`pushBatch` is the gate.** The collector engages batching only when +`destination.pushBatch` exists. Set `config.batch` on a destination that lacks +it and every event is sent one at a time; the collector logs one ERROR at +registration naming the setting, and `walkeros validate` and the bundle fail +(`BATCH_WITHOUT_PUSH_BATCH`) for inline `code`. When adding batching to a +destination, implementing `pushBatch` is the change; `config.batch` alone buys +nothing. A batched event answers `queued` with the reason `batch`, so a server +source acknowledges the producer at queue time; a failed flush is visible (ERROR +line, failed counter, DLQ) but is not redelivered. `walker flush` flushes every +pending batch. - `wait` (ms): debounce window. The timer resets on every push. Legacy form `batch: 1000` is shorthand for `{ wait: 1000 }`. @@ -232,11 +271,12 @@ after a successful flush. **Three rules every destination follows.** First, no retry loops, backoff or DLQ code of its own: each request is sent once through the vendor SDK, whose built-in retry stays at its defaults, because a standard retry belongs in the -collector, not in each package. Second, `init` is offline: it validates the -config and builds clients, but makes no network call, loads no credentials and -creates no remote resources; authentication and lookups happen on the first -delivery, and provisioning belongs to `setup`. Third, every failure, thrown or -reported in a `BatchOutcome`, is an `Error` that names the cause (and, for a +collector, not in each package. Second, a server destination's `init` is +offline: it validates the config and builds clients, but makes no network call, +loads no credentials and creates no remote resources; connections (Kafka, +BigQuery, S3, Sheets, SQLite) open at the first use, memoized and retried after +a failure, and provisioning belongs to `setup`. Third, every failure, thrown or +reported in a `BatchOutcome`, is a `StepError` that names the cause (and, for a configuration error, the fix) and carries `code` (the vendor error name or a package code), `status` (the HTTP status when the vendor answered) and `retryable`. Take `retryable` from the vendor SDK's own public classifier where diff --git a/skills/walkeros-using-cli/SKILL.md b/skills/walkeros-using-cli/SKILL.md index 08cbec4d5..10625e8f7 100644 --- a/skills/walkeros-using-cli/SKILL.md +++ b/skills/walkeros-using-cli/SKILL.md @@ -118,7 +118,7 @@ not used there. "destinations": { "gtag": { "package": "@walkeros/web-destination-gtag", - "config": { "measurementId": "G-XXXXXX" } + "config": { "settings": { "ga4": { "measurementId": "G-XXXXXX" } } } } } } diff --git a/skills/walkeros-using-cli/flow-configuration.md b/skills/walkeros-using-cli/flow-configuration.md index 532673cab..df1f32825 100644 --- a/skills/walkeros-using-cli/flow-configuration.md +++ b/skills/walkeros-using-cli/flow-configuration.md @@ -178,11 +178,11 @@ globs both work, resolved against the install root (where pacote put files): "gtag": { "package": "@walkeros/web-destination-gtag", "config": { - "measurementId": "G-XXXXXX" - }, - "mapping": { - "product": { - "view": { "name": "view_item" } + "settings": { "ga4": { "measurementId": "G-XXXXXX" } }, + "mapping": { + "product": { + "view": { "name": "view_item" } + } } } } @@ -192,13 +192,11 @@ globs both work, resolved against the install root (where pacote put files): ### Destination Properties -| Property | Type | Description | -| --------- | ---------------------------------- | -------------------------------------------------------------------------- | -| `package` | `string` | NPM package or local package name | -| `config` | `object` | Destination-specific configuration | -| `mapping` | `object` | Event transformation rules | -| `consent` | `object` | Required consent levels | -| `before` | `string \| Route[] \| RouteConfig` | Destination chain: runs for this destination only (a `stop` skips only it) | +| Property | Type | Description | +| --------- | ---------------------------------- | ---------------------------------------------------------------------------------------------------------- | +| `package` | `string` | NPM package or local package name | +| `config` | `object` | `settings` (package-specific), `mapping` (event transformation rules), `consent` (required consent levels) | +| `before` | `string \| Route[] \| RouteConfig` | Destination chain: runs for this destination only (a `stop` skips only it) | **Route shape** (used wherever the type column shows `Route[]` or `RouteConfig`, and by `collector.next`). A `RouteConfig` is a **disjoint union**: set at most @@ -260,11 +258,10 @@ For mapping syntax, see ```json { "sources": { - "web": { - "package": "@walkeros/web-source", + "browser": { + "package": "@walkeros/web-source-browser", "config": { - "dataLayer": true, - "globalsStatic": true + "settings": { "pageview": true } } } } @@ -469,51 +466,53 @@ walkeros bundle config.json --all -o dist/all "platform": "web", "bundle": { "packages": { - "@walkeros/web-source": {}, + "@walkeros/web-source-browser": {}, "@walkeros/web-destination-gtag": {} } } }, "sources": { - "web": { - "package": "@walkeros/web-source", - "config": { "dataLayer": true } + "browser": { + "package": "@walkeros/web-source-browser", + "config": { "settings": { "pageview": true } } } }, "destinations": { "ga4": { "package": "@walkeros/web-destination-gtag", - "config": { "measurementId": "G-XXXXXX" }, - "mapping": { - "product": { - "view": { - "name": "view_item", - "data": { - "map": { - "currency": { "value": "USD" }, - "value": "data.price", - "items": { - "loop": [ - "this", - { - "map": { - "item_id": "data.id", - "item_name": "data.name" + "config": { + "settings": { "ga4": { "measurementId": "G-XXXXXX" } }, + "mapping": { + "product": { + "view": { + "name": "view_item", + "data": { + "map": { + "currency": { "value": "USD" }, + "value": "data.price", + "items": { + "loop": [ + "this", + { + "map": { + "item_id": "data.id", + "item_name": "data.name" + } } - } - ] + ] + } } } } - } - }, - "order": { - "complete": { - "name": "purchase", - "data": { - "map": { - "transaction_id": "data.orderId", - "value": "data.value" + }, + "order": { + "complete": { + "name": "purchase", + "data": { + "map": { + "transaction_id": "data.orderId", + "value": "data.value" + } } } } diff --git a/skills/walkeros-using-logger/SKILL.md b/skills/walkeros-using-logger/SKILL.md index 4b62a780e..6e3fca714 100644 --- a/skills/walkeros-using-logger/SKILL.md +++ b/skills/walkeros-using-logger/SKILL.md @@ -14,9 +14,10 @@ The logger is walkerOS's standard logging system, available in all sources and destinations via `env.logger` or `logger` parameter. It provides scoped, level-aware logging that replaces console.log. -**Core principle:** Don't log what the collector already logs. Only log -meaningful operations like external API calls, transformations, and validation -errors. +**Core principle:** Don't log what the collector already logs, and never log a +failure yourself. A step reports a failure by throwing a `StepError`; the +collector writes the one ERROR line per cause. Log only meaningful operations +like external API calls and transformations, at `debug`. ## Logger Access @@ -104,86 +105,104 @@ scoped logger. - **External API calls**: Before/after with request/response details - **Auth operations**: Token refresh, client creation/failures - **Transformations**: Complex mappings or data processing -- **Validation errors**: Always use `logger.throw` for fatal errors +- **Notes that are not failures**: a deliberate decision, an advisory about + configuration (`warn`: walkerOS acted, the intent is still met) +- **Never failures**: a missing setting, a failed request or an unsendable event + is a thrown `StepError`, not a log line ## Usage Patterns -### Pattern 1: Validation Errors (Always Use logger.throw) +### Pattern 1: Validation Errors (Throw a StepError) ```typescript -// ✅ GOOD - Fatal configuration error -async init({ config, logger }) { +import { StepError } from '@walkeros/core'; + +// ✅ GOOD - A missing setting is a failure: throw, never return or log +async init({ config }) { const { apiKey, projectId } = config.settings || {}; - if (!apiKey) { - logger.throw('Config settings apiKey missing'); - } + if (!apiKey) + throw new StepError('Config settings apiKey missing', { code: 'config' }); - if (!projectId) { - logger.throw('Config settings projectId missing'); - } + if (!projectId) + throw new StepError('Config settings projectId missing', { + code: 'config', + }); } ``` -**Why logger.throw:** +**Why a thrown StepError:** + +- The collector turns the throw into a failed push (or a failed start), counts + it, and writes **one** ERROR line per cause and minute with the code +- A step that also logged the failure would write a second line for the same + cause +- `code` and `retryable` tell a server source whether to ask the producer to + resend (`config`, `missing-input` and a 4xx are not retryable) +- `logger.throw` only throws now. It writes no log line, so it is not a way to + report a failure -- Logs the error at ERROR level (always visible) -- Throws Error automatically (no separate throw needed) -- Collector catches and handles gracefully -- Never returns (TypeScript type: `never`) +Never `return` early or `return false` from `init` instead of throwing: a step +that returns normally counts as delivered. ### Pattern 2: External API Calls ```typescript -// ✅ GOOD - Log external calls with context +import { StepError } from '@walkeros/core'; + +// ✅ GOOD - Trace the call at debug, throw on a failure async push(event, { config, logger }) { const endpoint = 'https://api.vendor.com/events'; - // Log before call - logger.debug('Calling API', { - endpoint, - method: 'POST', - eventId: event.id, - }); + logger.debug('Calling API', { endpoint, method: 'POST', eventId: event.id }); const response = await fetch(endpoint, { method: 'POST', body: JSON.stringify(event), }); - // Log after call - logger.debug('API response', { - status: response.status, - ok: response.ok, - }); - - if (!response.ok) { - const errorText = await response.text(); - logger.throw(`API error (${response.status}): ${errorText}`); - } + logger.debug('API response', { status: response.status, ok: response.ok }); + + if (!response.ok) + // The status sets retryable (429 and 5xx retry, other 4xx do not). The + // response body goes on `body`, never into the message, and never into a + // log line. + throw new StepError(`HTTP ${response.status}`, { + code: 'transport', + status: response.status, + body: await response.text(), + }); } ``` +Prefer `sendWeb` / `sendServer`: they already throw a `StepError` with the +status and body. + ### Pattern 3: Auth Operations ```typescript -// ✅ GOOD - Log auth client creation +import { StepError } from '@walkeros/core'; + +// ✅ GOOD - Trace the client creation, throw on failure async init({ config, logger }) { try { const authClient = await createAuthClient(config.settings); logger.debug('Auth client created'); - return { env: { authClient }, }; } catch (error) { - logger.throw( - `Authentication failed: ${error instanceof Error ? error.message : 'Unknown error'}`, - ); + throw new StepError('Authentication failed', { + code: 'config', + cause: error, + }); } } ``` +The SDK's own message stays on `cause`, so a vendor error that carries personal +data never reaches a log line. + ### Pattern 4: Server Startup (Sources) ```typescript @@ -254,7 +273,8 @@ When updating a source/destination to use the logger: - [ ] Remove all `console.log`, `console.warn`, `console.error` statements - [ ] Remove verbose init/push status logging (let collector handle) -- [ ] Add `logger.throw` for all validation errors (apiKey missing, etc.) +- [ ] Throw a `StepError` with a `code` for every failure (apiKey missing, + failed request, unsendable event); never log a failure and return - [ ] Add `logger.debug` before external API calls (with endpoint, method) - [ ] Add `logger.debug` after external API calls (with response status) - [ ] Add `logger.debug` for auth operations (client creation, token refresh) @@ -263,19 +283,23 @@ When updating a source/destination to use the logger: ## Testing with Logger -Use `createMockLogger` from `@walkeros/core` in tests: +Use `createMockLogger` from `@walkeros/core` to check `debug` traces. To prove +that a failure is reported, start a real collector (`startFlow`), mock only the +network or the vendor global, and read the push result; a mocked collector +cannot see loss: ```typescript -import { createMockLogger } from '@walkeros/core'; +import { startFlow } from '@walkeros/collector'; -test('throws on missing apiKey', () => { - const logger = createMockLogger(); +test('a missing apiKey fails the push', async () => { + const { elb } = await startFlow({ + destinations: { vendor: { code: destination, config: { settings: {} } } }, + }); - expect(() => { - destination.init({ config: {}, logger }); - }).toThrow('Config settings apiKey missing'); + const result = await elb({ name: 'page view' }); - expect(logger.throw).toHaveBeenCalledWith('Config settings apiKey missing'); + expect(result.ok).toBe(false); + expect(result.failed?.vendor.reason?.code).toBe('config'); }); ``` @@ -303,6 +327,18 @@ const { elb } = await startFlow({ - `'INFO'`: Errors + warnings + info - `'DEBUG'`: Everything +**Level contract:** `ERROR` means the user's intent is not met (an event was not +delivered, a setting has no effect). `WARN` means walkerOS acted and the intent +is still met. `INFO` is lifecycle and self-healing. `DEBUG` is the per-event +trace and deliberate decisions: unmet consent, a skipped event and a consent +wait are `debug`, never a failure. + +**Format:** on a server flow, `"collector": { "logger": { "format": "json" } }` +prints one JSON object per line (`severity`, `message`, `time`, `scope`, +`context`), which platforms such as Cloud Run read as structured logs. A web +flow rejects `format`. The flow's own handler and the runtime's handler both see +every line, so forwarding to a monitoring service needs no parsing. + ## Related Skills - [walkeros-understanding-destinations](../walkeros-understanding-destinations/SKILL.md) - diff --git a/skills/walkeros-using-step-examples/SKILL.md b/skills/walkeros-using-step-examples/SKILL.md index 9ebfb6a60..16b2e47d3 100644 --- a/skills/walkeros-using-step-examples/SKILL.md +++ b/skills/walkeros-using-step-examples/SKILL.md @@ -258,6 +258,44 @@ Supported commands: `config`, `consent`, `user`, `run`. --simulate destination.ga4 --command consent ``` +### Failure Step Example (`outcome`, `respond`) + +A step's failure modes are examples too. A failure fixture is `public: false` +and declares what the push result reports through `outcome`; `respond` states +what the vendor's boundary answers when the failure comes from a response: + +```typescript +export const rejectedByVendor: Flow.StepExample = { + public: false, + description: 'The vendor answers 200 and reports the event as failed.', + in: getEvent('page view'), + respond: { status: 200, body: { events: [{ status: 'failed' }] } }, + outcome: { status: 'failed', code: 'transport', retryable: false }, +}; + +export const missingInput: Flow.StepExample = { + public: false, + description: + 'An event with nothing to send fails instead of counting as delivered.', + in: getEvent('page view'), + out: [], + outcome: { status: 'failed', code: 'missing-input' }, +}; +``` + +- `outcome` is `{ status: 'failed' | 'queued', code, retryable? }`. Absent means + the step must not fail. The codes are the collector's failure codes (`config`, + `missing-input`, `transport`, `timeout`, `vendor-not-loaded`, ...). +- `respond` is `{ status?, body?, error?: 'network' | 'timeout' }`. +- A step with no vendor call for an example has `out: []`. A `silent` rule is an + example with `mapping: { silent: true }` and `out: []`, and a deliberate skip + (`ignore: true`) is `out: []` with no `outcome`. +- `runExamples(kind, step, examples, options?)` and + `conformance(kind, step, options)` from `@walkeros/collector/dev` run the + examples on a real collector and compare the push result with `outcome`. + Conformance profiles: destinations `http`, `sdk-env`, `web-sdk`, + `vendor-global`, `socket`; `transformer` and `source` for the other kinds. + ## Writing Examples ### Best Practices @@ -713,6 +751,9 @@ When adding step examples to a package or flow: - [ ] Include at least one happy-path example - [ ] For transformers, include a `false` (filtered) case - [ ] Add `it.each` test using step examples +- [ ] Failure examples (`public: false`, `outcome`, `respond`) for every event + the step cannot send, a `silent` example for a destination, and a + conformance test (`conformance` from `@walkeros/collector/dev`) - [ ] Verify examples compile: `npm run build` - [ ] Run tests: `npm run verify:touched -- ` (L1) - [ ] Add `title` and `description` to every public example. diff --git a/website/docs/apps/cli.mdx b/website/docs/apps/cli.mdx index 71b16ccaf..13b70832d 100644 --- a/website/docs/apps/cli.mdx +++ b/website/docs/apps/cli.mdx @@ -1101,7 +1101,7 @@ If the package returns nothing, no JSON is emitted; you only see the framework n | Code | Meaning | |------|---------| | 0 | Success or skip (any of the three skip reasons above) | -| non-zero | Failure (resolution error, missing default export, package threw) | +| non-zero | Failure (resolution error, missing default export, package threw, or the step's `init` threw or returned `false`; the message carries the reason) | ### IAM note @@ -1291,10 +1291,10 @@ At runtime an unknown id is logged and skipped, and the chain continues. A `stop ### Reference checks -`$flow.`, `$store.`, `$secret.` and `$contract.` resolve only as the whole string value; `$var.` and `$env.` also resolve inline. Flow validation warns when a whole-value reference would not resolve: +`$flow.`, `$store.`, `$secret.` and `$contract.` resolve only as the whole string value; `$var.` and `$env.` also resolve inline. Flow validation fails with `MALFORMED_REFERENCE` when a whole-value reference would not resolve, and so does the bundle: -| Value | Warning | -|-------|---------| +| Value | Error message | +|-------|---------------| | `"$flow.server.url/collect"` | `does not match the $flow.NAME grammar (...); it would not resolve. To build a longer string, compose with $var or $env.` | | `"$secret.api_key"` | `does not match the $secret.NAME grammar (uppercase letters, digits and _); it would not resolve. ...` | | `"https://x/$store.cache"` | `uses $store. inline. $store. references resolve only as the whole value; it would ship as literal text. ...` | @@ -1311,13 +1311,26 @@ When the file has a root `contract`, the default run validates it exactly as `wa Every step with a `package` has its resolved `config.settings` checked against the package's settings schema, at the version the bundle would install: the `config.bundle.packages` pin, else the inline `@version`, else latest. A local `path` package is read from disk. -- Findings are warnings (`ENTRY_SCHEMA`, with `keyword`) for one minor release; they become errors after that. `--strict` fails on them now. +- An unknown settings key is an error (`ENTRY_SCHEMA`, keyword `additionalProperties`) where the package closes its schema. Other findings are warnings (`ENTRY_SCHEMA`, with `keyword`) for one minor release; they become errors after that. `--strict` fails on them now. - Placeholders: `$var` is resolved first and the value checked. `$env.NAME:default` is checked using the default, and an error names the default, never the placeholder. `$flow` resolves statically; an unresolvable `$flow` is an error. Values known only at runtime (`$env.NAME` without default, `$secret`, `$store`, `$code`) are not checked and are listed in `details.deferred`. - A schema that cannot be fetched is a skip (`SCHEMA_UNAVAILABLE`); a package without a settings schema is a skip (`NO_SETTINGS_SCHEMA`). - **Steps that import a named export** are checked against that export's schema. The export is picked as the bundler picks it: the step's `import`, else the first `imports` entry of its `config.bundle.packages` pin, else the default export. A package with several exports publishes a settings schema per export (`exportSchemas` in its `walkerOS.json`); a package with one export uses its `settings` schema for a named import too. An export missing from `exportSchemas`, a multi-export package version published before per-export schemas, or a package version that does not list its exports, is a skip (`NO_SETTINGS_SCHEMA`), so `--strict` fails on it. - `--offline` skips this check without fetching anything; the `Scope:` line says `offline: package settings not checked`. -Unknown top-level keys on a destination or store are warnings (`UNKNOWN_KEY`), and a removed package such as `@walkeros/store-memory` is a warning (`DEPRECATED_PACKAGE`). +Unknown keys are errors (`UNKNOWN_KEY`) on a source, transformer, destination or store entry, inside any step `config`, in the `collector` block and in inline `code`, because the runtime ignores them. Contract rules accept only `extend`, `tagging`, `description`, `events` and `schema`; the sections `globals`, `context`, `custom`, `user` and `consent` are rejected, use `schema`. A removed package such as `@walkeros/store-memory` is a warning (`DEPRECATED_PACKAGE`). + +### Mapping and logger checks + +Validate and the bundle (so also `push`, `simulate`, `setup` and the MCP `flow_bundle`) stop on these structural errors before building anything: + +| Code | Meaning | +|------|---------| +| `MAPPING_NO_OP` | A mapping key that has no effect at its position: a source rule with `data`, `include`, `remove`, `batch`, `settings`, `extend` or `silent`, a destination rule with `extend`, a transformer field that does nothing there. One error per step, naming the paths, what applies there and the fix | +| `MAPPING_VALUE_NO_OPERATOR` | A mapping value object with none of `key`, `value`, `fn`, `map`, `loop`, `set` | +| `MAPPING_VALUE_PRODUCERS` | A mapping value with two producers | +| `BATCH_WITHOUT_PUSH_BATCH` | `batch` on inline `code` that has no `pushBatch` | +| `MALFORMED_REFERENCE` | A whole-value `$flow.`, `$store.`, `$secret.` or `$contract.` reference that would ship as literal text | +| `WEB_LOGGER_FORMAT` | `logger.format` on a web flow; the browser console has no output format | ### Example checks @@ -1369,17 +1382,16 @@ walkeros validate flow.json --strict --json || exit 1`} JSON output format (`details.context`, the IntelliSense context, omitted): ```json { - "valid": true, + "valid": false, "type": "flow", - "errors": [], - "warnings": [ + "errors": [ { "path": "flows.server.destinations.pubsub.setings", - "message": "Unknown key \"setings\" on Destination is ignored by the runtime", - "suggestion": "Remove the key, or move it to where it belongs.", + "message": "Unknown key \"setings\" on destination is ignored by the runtime. Remove the key, or move it to where it belongs.", "code": "UNKNOWN_KEY" } ], + "warnings": [], "details": { "flowNames": ["server"], "flowCount": 1, @@ -1413,7 +1425,7 @@ Codes group as follows (the full list is exported as `VALIDATE_CODES` from `@wal | Group | Codes | |-------|-------| | Scope and options | `FLOW_NOT_FOUND`, `ENTRY_NOT_FOUND`, `AMBIGUOUS_ENTRY`, `OPTION_NOT_APPLICABLE` | -| Structure and resolution | `SCHEMA_VALIDATION`, `INVALID_COMPONENT_NAME`, `INVALID_REFERENCE`, `STORE_REFERENCE_NOT_FOUND`, `STATE_STORE_UNKNOWN`, `STATE_STORE_FILE`, `UNRESOLVED_REFERENCE`, `FLOW_CYCLE`, `VAR_CYCLE`, `UNKNOWN_KEY`, `DEPRECATED_PACKAGE`, `TRANSFORMER_MAPPING_NO_OP` (a warning: a transformer mapping field that does nothing there) | +| Structure and resolution | `SCHEMA_VALIDATION`, `INVALID_COMPONENT_NAME`, `INVALID_REFERENCE`, `STORE_REFERENCE_NOT_FOUND`, `STATE_STORE_UNKNOWN`, `STATE_STORE_FILE`, `UNRESOLVED_REFERENCE`, `FLOW_CYCLE`, `VAR_CYCLE`, `UNKNOWN_KEY`, `DEPRECATED_PACKAGE`, `MAPPING_NO_OP`, `MAPPING_VALUE_NO_OPERATOR`, `MAPPING_VALUE_PRODUCERS`, `BATCH_WITHOUT_PUSH_BATCH`, `MALFORMED_REFERENCE`, `WEB_LOGGER_FORMAT` | | Settings | `ENTRY_SCHEMA` | | Contracts | `NO_CONTRACT_SECTION`, `INVALID_EXTENDS`, `CIRCULAR_EXTENDS`, `UNRESOLVED_CONTRACT` | | Skips | `GATED_BY_ERRORS`, `CONTRACT_NOT_STATIC`, `NO_PACKAGE`, `NO_SETTINGS_SCHEMA`, `SCHEMA_UNAVAILABLE` | diff --git a/website/docs/apps/mcp.mdx b/website/docs/apps/mcp.mdx index 868700d36..9fb790352 100644 --- a/website/docs/apps/mcp.mdx +++ b/website/docs/apps/mcp.mdx @@ -354,7 +354,7 @@ package. The hosted server describes them itself when a client connects; see | `walkeros://reference/variables` | Variable patterns ($var/$env/$code/$store) | | `walkeros://reference/contract` | Event schemas, wildcards, inheritance | | `walkeros://reference/examples` | A complete, tested flow config with web, server and warehouse flows | -| `walkeros://reference/packages` | Full package catalog | +| `walkeros://reference/packages` | Full package catalog as `{ catalog, count, warnings? }`; `warnings` says when the npm search failed, so a partial catalog is never read as an empty one | | `walkeros://schema/{packageName}` | Per-package JSON schemas | ### Prompts diff --git a/website/docs/apps/runtime.mdx b/website/docs/apps/runtime.mdx index 259ad6463..595a6e5b1 100644 --- a/website/docs/apps/runtime.mdx +++ b/website/docs/apps/runtime.mdx @@ -237,6 +237,13 @@ credential-named fields, long high-entropy strings), but they are not masked by exact value. Prefer app-managed secrets for anything that must never reach a log. +The runtime prints text lines by default. A flow that sets +`collector.logger.format` to `json` prints one JSON object per line instead, +errors and warnings on stderr, which log collectors such as Cloud Run read as +structured logs (see [Logger](/docs/collector/logger#log-format)). The flow's +`logger.level` applies to the collector's lines, and `--verbose` still forces +`DEBUG`. + Log lines print a path under the system temp directory as `$TMPDIR/...`. On Linux `TMPDIR` is often unset, so read such a path as the system temp directory (usually `/tmp`). diff --git a/website/docs/collector/commands.mdx b/website/docs/collector/commands.mdx index 3d28c853c..b4c8bbab6 100644 --- a/website/docs/collector/commands.mdx +++ b/website/docs/collector/commands.mdx @@ -310,9 +310,9 @@ This gives five practical guarantees: `require` only delays when its first `on` delivery lands. A dependent source reacts to state correctly whether or not it declares `require`. * **Before the first push.** A destination's `on` handler receives the current - state before any event is delivered to it. This needs an `init` method on the - destination: without one, its `on` handler is never called, while the state - command still reports the destination under `done`. The collector waits for + state before any event is delivered to it. A destination without an `init` + method is initialized without one, once its consent requirement is met, and + its waiting `on` deliveries reach it then. The collector waits for an async handler to finish, up to the destination's `config.timeout`, before it delivers events. If the handler does not finish in that time, or fails, the destination's events stay in its queue instead of going out without the @@ -502,6 +502,21 @@ elb({ name: 'page view' }); language="javascript" /> +## flush {#walker-flush} + +Send every pending destination batch now (see +[Batch scheduling](/docs/mapping/rule#batch-scheduling)) and keep the collector +running. It answers a `PushResult` that merges the outcome of each flushed +batch, so a failed flush shows under `failed` with its reason. Shutdown uses +the same flush, and the browser source sends `walker flush` when the page goes +away (`pagehide`, or the page turning hidden), so batched web events are not +lost when a visitor leaves. + + + ## shutdown Stop the collector: sources are destroyed first, then pending batches are diff --git a/website/docs/collector/index.mdx b/website/docs/collector/index.mdx index 2b13bfeee..ed85f515f 100644 --- a/website/docs/collector/index.mdx +++ b/website/docs/collector/index.mdx @@ -235,8 +235,8 @@ console.log(collector.status); // browser: { count: 100, lastAt: 1707580900000, duration: 1200 } // }, // destinations: { -// ga4: { count: 95, failed: 0, lastAt: 1707580900000, duration: 800, -// queuePushSize: 0, dlqSize: 0 }, +// ga4: { count: 95, failed: 0, skipped: 0, lastAt: 1707580900000, +// duration: 800, queuePushSize: 0, dlqSize: 0 }, // meta: { count: 0, failed: 5, lastAt: 1707580850000, duration: 400, // queuePushSize: 0, dlqSize: 5 } // }, @@ -255,21 +255,51 @@ const avgMs = ga4.duration / (ga4.count + ga4.failed);`} language="typescript" /> +### What a push answers + +`elb` and `collector.push` answer a `PushResult`. For each destination the +event ends in one of three places: + +* `done`: the destination delivered the event, +* `queued`: the event is waiting, with a `reason`: `consent` (the destination + waits for a grant), `batch` (it sits in a batch until the flush) or `held` + (the destination had a failure and retries after a window), +* `failed`: the destination did not deliver it, with a `reason` that carries + a `code`, a `message` and `retryable` (see the table in + [Logger](./logger#failure-codes)). + +An event a destination skips on purpose, because a rule has `ignore: true` +or because consent is denied on a server collector, appears under none of the +three and is counted in `status.destinations..skipped`. + +`ok` is `false` when any destination failed, and also when a step before the +destinations failed: `reason` then names the cause, for example `queue-full` +(the collector has not run yet and its queue is full), `transformer` (a +transformer threw or is not ready) or `mapping` (a rule `condition` threw). +Unmet consent is never a failure, so a consent wait answers `ok: true` with +`queued`. + +A source that answers a producer reads this result: see +[Producer acknowledgement](/docs/sources/server#producer-acknowledgement). + ### What `failed` counts `collector.status.failed` is the single counter for any walkerOS-internal -pipeline failure. Contributing sites: - -* a destination's `push` threw, or the destination reported an error through - `context.reportError`, +pipeline failure. Each destination also counts its own failures in +`status.destinations..failed`. Contributing sites: + +* a destination's `push` or `pushBatch` threw, or the destination reported an + error through `context.reportError`, +* a destination could not start or did not become ready: `init` threw or did + not settle within `config.timeout`, an open circuit breaker, a destination + held past one retry window, a full queue, * an exception escaped the inner pipeline of `collector.push` or `collector.command` (any uncaught error inside the boundary), -* mapping outer-wrap failures (an internal throw inside - `processMappingValue`), +* a transformer threw or is not ready, +* a rule `condition` threw (the event fails with `mapping`), * source startup failures (`code()` threw, `init()` threw, queued-on flush threw), -* transformer init failures, -* destination init failures. +* transformer init failures. Rejected client input does not increment `status.failed`. An event the pipeline declares invalid (a missing or malformed `name`) resolves as @@ -278,59 +308,54 @@ counts on `collector.status.sources..rejected`. That keeps `failed` a measure of walkerOS faults rather than of whatever the public internet posts at a collector endpoint. -User-supplied callback throws are visibility-only: they log at error -level but do not increment `status.failed`. This keeps the counter a -clean pipeline-health signal. Sites in this group: - -* `on` subscriptions (`destination.on`, `source.on`, consent rules, - `ready`, `run`, `session`, generic), -* mapping `condition`, `fn`, and `validate` callbacks. +User-supplied callback throws in `on` subscriptions (`destination.on`, +`source.on`, consent rules, `ready`, `run`, `session`, generic) log at error +level but do not increment `status.failed`. A mapping `fn`, `validate` or +value-level callback that throws drops only that value; the event still goes +out and the collector logs one ERROR line per cause. A destination whose +`on('consent')` handler fails is reported as `consent-not-applied`. -In both cases the collector logs at error level: - -').error('Push failed', { error, event }); - -// collector-side boundary failure (event: the event name) -logger.error('push failed', { error, event }); -logger.error('command failed', { error, command }); - -// pipeline-internal failures (counted) -logger.error('mapping processing failed', { event, error }); -logger.scope('source').error('source factory failed', { sourceId, error }); -logger.scope('source').error('source init failed', { sourceId, error }); -logger - .scope('source') - .error('source on flush failed', { sourceId, type, error }); -logger - .scope('transformer:') - .error('transformer init failed', { transformer, error }); -logger.scope('').error('destination init failed', { error }); - -// user-callback failures (logged, NOT counted; event: the whole event) -logger.error('mapping rule condition failed', { event, mappingKey, error }); -logger.error('mapping condition failed', { event, error }); -logger.error('mapping fn failed', { event, error }); -logger.error('mapping validate failed', { event, error }); -logger.scope('on').error('on callback failed', { kind, error });`} - language="typescript" -/> +Every failure is written to the log by the collector, once per cause. See +[Logger](./logger#failure-lines) for the line format, the rate limit and the +codes. A source whose `init()` throws stays with `config.init === false` instead of being marked initialized, so `source.config.init` shows that the source did not start. -Alarm on the ratio of `status.failed` to `status.in`. The log message text -plus structured fields are enough to filter destination vs boundary -failures downstream. +Alarm on the ratio of `status.failed` to `status.in`. The structured fields of +each line (`step`, `code`, `status`, `retryable`) are enough to filter +destination vs boundary failures downstream. + +**Personal data in logs.** Failure lines carry the step id, the cause code, +the HTTP status, the event name, the mapping key and the error message +(capped at 300 characters), never the event's `data`, `user`, `consent`, +`globals`, `context` or `custom`, an ingest value, a vendor response body or a +setting value. A vendor response body stays on the error object. + +### Consent-denied events on a server + +A collector holds an event that a destination's `consent` does not allow, and +sends it when consent is granted later (`consentDenied: 'queue'`, the +default). A server has no later grant for an event's own consent, so server +flows built by the CLI default to `consentDenied: 'skip'`: the event is not +queued, the destination counts it in `skipped`, and the collector writes a +DEBUG line. Set the value in the flow's `collector` block to choose the other +mode; an authored value always wins. Unmet consent is never an error or a +warning. + + + +### Flushing batches -**Personal data in logs.** The push, command and destination failure logs -carry the event name and the error message, never the event's data. The -mapping logs (`mapping processing failed` and the callback failures) carry the -whole event, including `user` and `data`. If your events carry personal data, -redact it in a custom logger `handler`; do not parse and rewrite at every call -site. +`walker flush` sends every pending destination batch at once and leaves the +collector running (see [Commands](./commands#walker-flush)). The browser source +sends it when the page goes away. ### Fatal errors @@ -383,10 +408,21 @@ metaDrops?.dlq ?? 0; // monotonic count of evicted DLQ entries`} ## Buffer bounds The collector keeps four internal buffers per process. Each is size-bounded -with a configurable cap; on overflow the oldest entries are evicted (FIFO), -the corresponding `dropped` counter is incremented, and a warning is logged. -The collector buffers and the DLQ warn once per overflow window; the consent -queue warns on every eviction. +with a configurable cap, and the `dropped` or `failed` counter moves on +overflow. What happens to the entry and how loud it is depends on the buffer: + +* `collector.queue` evicts the oldest entry and logs once per overflow window + at INFO, because it only feeds destinations added later. +* `collector.preRunQueue` refuses the newest event. That push answers + `ok: false` with the reason `queue-full`, counts as failed and logs an ERROR + line. +* `destination.queuePush` evicts the oldest consent-waiting event first and + counts it in `dropped`; it logs once per overflow window at DEBUG, because + unmet consent is never an error. When the destination is held and the queue + holds only held events, the newest event is refused and fails with + `queue-full`. +* `destination.dlq` evicts the oldest failed entry (it was reported failed + already, only its replay material is lost) and logs an ERROR line. | Buffer | Purpose | Default cap | Config | | ----------------------- | ---------------------------------------------------- | ----------- | ----------------------------------- | diff --git a/website/docs/collector/logger.mdx b/website/docs/collector/logger.mdx index 51652a769..e78d0c198 100644 --- a/website/docs/collector/logger.mdx +++ b/website/docs/collector/logger.mdx @@ -39,7 +39,15 @@ const { collector, elb } = await startFlow({ ### Choosing a level on public endpoints A collector endpoint is exposed to the open internet, so log level decides -whether ambient noise becomes an alert. +whether ambient noise becomes an alert. The levels follow one contract: + +* `ERROR`: the intent is not met. An event was not delivered, a setting has no + effect, a producer was told something false. +* `WARN`: walkerOS acted and the intent is still met, for example an invalid + event it rejected. +* `INFO`: lifecycle and self-healing. +* `DEBUG`: the trace of each event and deliberate decisions. Unmet consent, a + skipped event and a consent wait are `DEBUG`, never a failure. | Situation | Level | |-----------|-------| @@ -66,29 +74,32 @@ The logger is automatically available in destination and source contexts: ### In destinations +A destination does not log its own failures. It throws, and the collector +writes one line per cause (see [Failure lines](#failure-lines)). Use the logger +for `debug` traces and for notes that are not failures. + ### In sources -### Boundary error logs - -When an exception escapes the inner pipeline of `collector.push` or -`collector.command`, the collector emits a structured error log and -increments `collector.status.failed`. The messages are `'push failed'` -and `'command failed'` respectively, with the full event/ingest or -command/data context as structured fields. See -[Status: What `failed` counts](./#what-failed-counts) for details and -PII implications. - -Internal pipeline failures from mapping, source startup, transformer -init, and destination init are scoped under `'source'`, -`'transformer:'`, or the destination type. User-callback throws -from `on` subscriptions are scoped under `'on'`. The full list of error -verbs and scopes is documented in -[Status: What `failed` counts](./#what-failed-counts). - ### Throwing errors -Use `logger.throw()` to log an error and throw in a single call: +`logger.throw()` throws an exception in a single call and returns `never`. It +does not log: the collector reports the failed step once, so a throw never +produces a second line. +In a destination, a transformer or a store, prefer throwing a `StepError` +from `@walkeros/core` with a `code`, so the failure carries its cause (see +[Create your own destination](/docs/destinations/create-your-own#reporting-failures)). + +## Failure lines + +The collector writes every failure of a step, once per cause. A destination, +transformer, store or source never writes a failure line of its own. A line is +keyed by the step, the cause code and the HTTP status, and the collector writes +one line per key and minute. The first line names the counter that the repeats +are counted in, for example `status.destinations.ga4.failed`. Later +occurrences are counted, and the next line of that key, or the one written at +server shutdown, carries `suppressed: `. If more than 100 different causes +are active, the extra ones are logged on every occurrence rather than dropped. + +A line carries the step id, the cause `code`, the HTTP `status`, `retryable`, +the event name, the mapping key, a count and the error message capped at 300 +characters. It never carries event data (`data`, `user`, `consent`, `globals`, +`context`, `custom`), ingest values, vendor response bodies, setting values or +an error's `cause`. + +The `handler` of the logger configuration receives every line, including these, +with the structured fields (`scope`, `code`, `status`, `retryable`, +`suppressed`) in its context, so forwarding failures to a monitoring service +needs no parsing. Handlers add up: a flow's own handler and the runtime's both +see each line. + +### Failure codes + +| Code | Cause | Retryable | Fix | +| ---- | ----- | --------- | --- | +| `transport` | A destination's request failed or threw an unknown error | yes, except a `4xx` other than `429` | Check the endpoint and credentials; the status is on the line | +| `timeout` | A push or an `init` did not settle within `config.timeout` (10 s) | yes | Raise `config.timeout` or check the vendor | +| `not-ready` | The destination or transformer could not start | no | Fix the cause its message names | +| `breaker-open` | Repeated retryable failures opened the circuit breaker; the destination is paused | yes | Wait, or fix the outage; the destination retries after a pause | +| `blocked` | The destination was held for longer than one retry window | yes | Fix the failing call; later events retry | +| `queue-full` | A queue refused the event: the collector has not run yet, or a held destination's queue is full | yes | Start the collector sooner or raise `queueMax` | +| `mapping` | A mapping rule `condition` threw, or a mapped value could not be built | no | Fix the condition or the value | +| `transformer` | A transformer threw | no | Fix the transformer | +| `missing-input` | The event has nothing this step can send (a required field is empty) | no | Map the missing value or skip the event with `ignore: true` | +| `config` | A required setting is missing or invalid | no | Set it; `walkeros validate` finds most of these | +| `vendor-not-loaded` | The vendor script did not load or was blocked | no | Allow the script or load it yourself | +| `vendor-not-ready` | The vendor SDK is not ready yet | yes | Usually resolves on the next event | +| `vendor-timeout` | The vendor script did not become ready in time | yes | Check the network and the script URL | +| `beacon-rejected` | The browser refused a `sendBeacon` | only up to 64 KiB | Send smaller batches | +| `consent-not-applied` | A destination's `on('consent')` handler failed | n/a | Fix the handler; the destination's events wait | +| `invalid` | An event was rejected as invalid input (a `WARN`) | no | Fix the producer | + +A step's own `StepError.code` wins over these. A code outside this table with +neither a status nor a `retryable` option is not retryable. Unmet consent is +not in this table because it is never a failure. + +## Log format + +By default lines print as text. On a server flow, set `format` to `json` to +print one JSON object per line, with `severity`, `message`, `time`, `scope` and +`context`. Errors and warnings go to stderr, info and debug to stdout, which +platforms such as Cloud Run read as structured logs. + + + +A flow's own `logger` level and format apply under `runneros` as well. `--verbose` +still forces `DEBUG`. A web flow has no output format, so `walkeros validate` +rejects `logger.format` there (`WEB_LOGGER_FORMAT`). Secrets are masked in both +formats, and a JSON line is capped at 32 KiB. + ## Scoped loggers Scoped loggers prefix log messages with context information, making it easier to diff --git a/website/docs/destinations/api/server.mdx b/website/docs/destinations/api/server.mdx index 8c4b90a82..8437a2e42 100644 --- a/website/docs/destinations/api/server.mdx +++ b/website/docs/destinations/api/server.mdx @@ -20,14 +20,14 @@ import WhereThisFits from '@site/src/components/snippets/_where-this-fits.mdx'; /> The server API destination sends events from Node.js to any HTTP(S) endpoint -using native HTTP/HTTPS modules. It auto-selects HTTP or HTTPS from the URL, supports configurable timeouts (default 5000ms), custom transforms, and dependency injection for testing. +using `fetch`. It supports configurable timeouts (default 5000ms), custom transforms, and dependency injection for testing. Server API is a server destination in the walkerOS flow:} flowMap={{ destinations: { default: { label: 'Destination', text: 'API', highlight: true } }, }} - post={<>Sends events server-side to any HTTP endpoint using Node.js native modules with configurable request formatting and batching.} + post={<>Sends events server-side to any HTTP endpoint using `fetch` with configurable request formatting and batching.} /> ## Installation @@ -89,9 +89,11 @@ await startFlow({ language="typescript" /> -A request that fails, times out or gets an error status is only logged at -debug level: the event counts as delivered and does not reach the dead-letter -queue. +A request that fails, times out or gets an error status is reported as a failed +push: the event reaches the dead-letter queue and the collector logs one ERROR +line per cause and minute with the HTTP status. A `429` and a `5xx` answer are +retryable, other `4xx` answers are not. Without `settings.url` the destination +fails to start with a `config` error. ### With custom transform diff --git a/website/docs/destinations/api/web.mdx b/website/docs/destinations/api/web.mdx index 6f977f930..55d706826 100644 --- a/website/docs/destinations/api/web.mdx +++ b/website/docs/destinations/api/web.mdx @@ -153,11 +153,10 @@ per element, not once for the whole array. Batched requests carry no `traceparent` header: a batch can aggregate events from distinct traces, so there is no single identity to stamp. -Set `size` explicitly and keep it small: the `fetch` and `beacon` transports -send with `keepalive`, and browsers reject a `keepalive` request once the bodies -in flight exceed 64 KiB, while the default `size` is 1000 events. Such a flush -fails without being reported (see [Transport methods](#transport-methods)). A -full event is around 0.5 to 1 KiB. +Keep `size` small: the `beacon` transport is rejected by the browser once the +bodies in flight exceed 64 KiB, and the default `size` is 1000 events. The +`fetch` transport sends a request larger than that without `keepalive`, so it +may be lost if the page unloads first. A full event is around 0.5 to 1 KiB. = { type: 'webhook', config: {}, - async push(event, { config }) { + async push(event, { config, rule }) { const { settings } = config; - // 3. Access your settings - if (!settings?.url) return; + // 3. A missing setting is a failure, never a silent return + if (!settings?.url) + throw new StepError('Webhook needs settings.url', { code: 'config' }); + + // A rule with silent: true sends nothing + if (rule?.silent) return; - // 4. Send the event; a rejection marks the push as failed - await fetch(settings.url, { + // 4. Send the event and await it; fetch resolves on an error status + const response = await fetch(settings.url, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(event), }); + if (!response.ok) + throw new StepError(\`HTTP \${response.status}\`, { + code: 'transport', + status: response.status, + }); }, };`} language="typescript" /> +## Reporting failures + +A destination reports a failure by throwing. A `push` that returns normally +counts as delivered, so it must not return early when it could not send, and it +must await every call it makes: a call that is not awaited can fail after the +push has already been counted. + +Throw a `StepError` from `@walkeros/core` with a `code`, and optionally the +HTTP `status` and `retryable`. The collector turns the throw into a failed push +with that reason, counts it in the destination's `failed` counter, adds the +event to the dead-letter queue and writes one ERROR line per cause and minute. + +* **`config`** - a required setting is missing or invalid (not retryable) +* **`missing-input`** - the event has nothing this destination can send (not + retryable) +* **`vendor-not-loaded`**, **`vendor-not-ready`**, **`vendor-timeout`** - the + vendor script is missing, not ready yet or too slow +* **`transport`** - the request failed; a `429` and a `5xx` status are + retryable, other `4xx` statuses are not + +Any other throw is a `transport` failure. `retryable` tells a server source +whether to ask the producer to resend the message. Keep payload data out of the +error message; put a vendor response on the error's `body`, which is never +logged. + +A value that cannot be mapped is not a failure of the event: drop that field, +send the event and report the problem once with `reportCause` from +`@walkeros/core`. + +`init` follows the same rule. It throws a `StepError` when the destination +cannot start, such as a missing setting or a blocked vendor script, and it does +not return `false`. A destination with a consent requirement is initialized only +once consent is granted, and an `init` that does not settle within the +destination's `config.timeout` (10 seconds by default) fails the waiting pushes +with the `timeout` code. + +### Rule options every destination honours + +A destination honours `rule.silent`: a rule with `silent: true` sends no event +of its own and only runs the side effects the rule maps, such as an identify. A +silent rule that has nothing to send returns normally. + +### Vendor scripts + +A web destination that loads a vendor script uses `loadVendor` from +`@walkeros/web-core`. It injects the script once per page and throws +`vendor-not-loaded`, `vendor-not-ready` or `vendor-timeout`, so a blocked +script shows as failed pushes instead of lost events. + +### Conformance test + +`@walkeros/collector/dev` exports `conformance`, a test kit that runs a +destination against a real collector and checks, among others, that a failing +send is reported, that a failure never leaks event data into a log line, that a +`silent` example sends nothing and that a revoked consent reaches the vendor. +Call it from a Jest test with the profile that matches the destination: +`http` for a destination that calls `fetch`, `sdk-env` for a server SDK, +`web-sdk` for a web SDK started in `init`, `vendor-global` for a vendor script +that defines a global, or `socket`. A setting that only works in pairs, such as +"one of these two", is stated as `anyOf` in the settings schema. + ## The `pushBatch` function (optional) When the destination's config sets `batch` (see @@ -106,8 +176,12 @@ events and calls `pushBatch` once per flush instead of `push` per event. If `pushBatch` throws or rejects, every entry goes to the destination's dead-letter queue and counts as failed. To report single rows, resolve `{ failed: [{ index, error }] }` with indices into `batch.entries`: only those -entries fail. A batch flushes after the `elb` calls that filled it have -resolved, so its outcome shows only in the destination's status and logs. +entries fail. A batched event answers `queued` with the reason `batch`, so a +server source acknowledges it when it is queued. A batch flushes after the +`elb` calls that filled it have resolved, so a failed flush shows in the +destination's `failed` counter, the dead-letter queue and one ERROR line, but +the producer's message is not redelivered. A `walker flush` command flushes +all pending batches at once. Pending batches are flushed at shutdown: the collector waits up to 5 seconds per flush and logs one that takes longer as failed. @@ -127,8 +201,10 @@ The optional `on` method lets your destination respond to collector lifecycle events, for example to handle consent changes or session management. Cleanup belongs in `destroy`. -The collector calls `on` only after `init` has run. A destination with `on` -needs an `init` method, even an empty `init() {}`, or `on` is never called. +The collector calls `on` once the destination is initialized. A destination +without an `init` method is initialized without one, when its first event +clears its consent requirement (or consent is granted), and the `on` deliveries +that waited meanwhile reach it then, in order. ### Available Events @@ -155,9 +231,6 @@ are in [State delivery](/docs/collector/commands#state-delivery). type: 'webhook-consent', config: {}, - // Required for on() to be called - init() {}, - on(type, context) { if (type === 'consent') { // context.data holds the consent values the command set @@ -184,7 +257,7 @@ The same lifecycle applies to sources, destinations, and stores. Transformers ar - Setup is opt-in via `config.setup` in the flow config. The CLI invocation requires it to be `true` or an object; `false` or omitted means the operator's setup invocation is a narrated skip. - Idempotency is the package's responsibility. Re-running setup against a fully provisioned environment is a safe no-op. -- `walkeros setup` runs `init` first and gives `setup` the config `init` returns, if any; when `init` returns `false`, setup is skipped. `destroy` runs after `setup`. +- `walkeros setup` runs `init` first and gives `setup` the config `init` returns, if any; when `init` throws or returns `false`, the command fails with the reason and setup does not run. `destroy` runs after `setup`. - Structured JSON output is opt-in via `--json` (e.g. `walkeros setup . --json`). In normal mode the CLI narrates the action and never splices a JSON envelope into the output, even if `setup()` returns a value. The opt-in setup behavior and idempotency are independent of `--json`. ### Example: a destination with setup diff --git a/website/docs/destinations/index.mdx b/website/docs/destinations/index.mdx index 7ff4188c7..b078baf94 100644 --- a/website/docs/destinations/index.mdx +++ b/website/docs/destinations/index.mdx @@ -46,6 +46,23 @@ analytics platforms, marketing tools, data warehouses, and custom APIs. Each destination operates independently, so one failed destination won't affect others. +## When delivery fails + +An event that a destination did not deliver is never counted as delivered. The +push result lists the destination under `failed` with a `reason` (a `code`, a +`message` and whether a retry could help), the destination's `failed` counter +moves, the event goes to its dead-letter queue and the collector writes one +ERROR line per cause and minute. This covers a failed request (including an +error status or a rejected answer inside a `200`), a missing required setting, +an event the destination cannot send, a vendor script that did not load, and a +destination whose `init` failed. See [Logger](/docs/collector/logger#failure-codes) +for the codes. + +Two things are not failures. Unmet consent only waits (or, on a server, skips) +the event and is a `DEBUG` line. A rule with `ignore: true` or `silent: true` +is a decision, not an error, and every destination honours both. Events a +destination batches answer `queued` until the flush. + ## Analytics tools | Destination | Web | Server | diff --git a/website/docs/destinations/server/amplitude.mdx b/website/docs/destinations/server/amplitude.mdx index 4adac0504..9ee17cac3 100644 --- a/website/docs/destinations/server/amplitude.mdx +++ b/website/docs/destinations/server/amplitude.mdx @@ -79,3 +79,20 @@ Add to your `flow.json` destinations: + +## Batching and delivery + +This destination batches by default: events are collected and sent together once +`config.batch` reaches 200 events or 10 seconds, whichever comes +first. Each event keeps its own outcome, so a failed delivery fails that event +and lands in the dead-letter queue instead of counting as delivered. Set +`config.batch: { size: 1 }` to send each event at once; there is no JSON switch +that turns the default batch off. + +A batched event answers `queued` with the reason `batch`. A server source +acknowledges the producer when the event is queued, and a failed flush is +reported (one ERROR line with the status, the failed counter, the dead-letter +queue) but not redelivered. Options are in +[Batch scheduling](/docs/mapping/rule#batch-scheduling). + +A delivery that Amplitude does not answer with `2xx` fails with its status. diff --git a/website/docs/destinations/server/gcp.mdx b/website/docs/destinations/server/gcp.mdx index e85daf388..af16c07b8 100644 --- a/website/docs/destinations/server/gcp.mdx +++ b/website/docs/destinations/server/gcp.mdx @@ -244,6 +244,7 @@ ingestion. This replaces the legacy `tabledata.insertAll` path. - **Cost**: $25/TB after the 2 TiB/month free tier (vs ~$50/TB for the legacy path). Most low-volume deployments fit entirely in the free tier. +- **First use**: the writer opens at the first push, not at startup; a missing dataset or table fails that push with the setup hint. BigQuery and Pub/Sub failures are reported once by the collector, and the messages name codes, not row data. - **Batching**: `pushBatch` is implemented. Set `config.batch` on the destination (a bare number is the debounce `wait` in ms) to flush all events in a window as a single `appendRows` call. No `'* *'` wildcard mapping rule is diff --git a/website/docs/destinations/server/hubspot.mdx b/website/docs/destinations/server/hubspot.mdx index 2453f26cb..4c88377c3 100644 --- a/website/docs/destinations/server/hubspot.mdx +++ b/website/docs/destinations/server/hubspot.mdx @@ -14,7 +14,7 @@ import WhereThisFits from '@site/src/components/snippets/_where-this-fits.mdx'; -Server-side event delivery to [HubSpot](https://www.hubspot.com/) via the official [`@hubspot/api-client`](https://www.npmjs.com/package/@hubspot/api-client) SDK. Sends custom behavioral events, upserts contacts via the CRM API, and supports optional batch mode with graceful shutdown. +Server-side event delivery to [HubSpot](https://www.hubspot.com/) via the official [`@hubspot/api-client`](https://www.npmjs.com/package/@hubspot/api-client) SDK. Sends custom behavioral events, upserts contacts via the CRM API, and batches events through the collector's `config.batch`. HubSpot is a server destination in the walkerOS flow:} @@ -81,7 +81,28 @@ Add to your `flow.json` destinations: -Every event must be associated with a contact via `email` or `objectId`. The `email` path defaults to `user.email`. Events where neither value can be resolved are skipped with a warning. +Every event must be associated with a contact via `email` or `objectId`. The `email` path defaults to `user.email`. An event where neither value can be resolved fails with the `missing-input` code, and a failed HubSpot call fails the push with HubSpot's HTTP status. A `429` and a `5xx` answer are retryable. + +## Batching + +Set `config.batch` to send events through HubSpot's batch API instead of one +request per event. The options are in +[Batch scheduling](/docs/mapping/rule#batch-scheduling). A flush sends chunks of +500 events, and each event keeps its own outcome. `settings.batch` and +`settings.batchSize` no longer exist; a flow that sets them fails `walkeros +validate`. + + + +A batched event answers `queued` with the reason `batch`. A failed flush is +reported (one ERROR line, the failed counter, the dead-letter queue), but the +producer's message is not redelivered. ## Contact upsert diff --git a/website/docs/destinations/server/kafka.mdx b/website/docs/destinations/server/kafka.mdx index a84a87a7a..3890c51ca 100644 --- a/website/docs/destinations/server/kafka.mdx +++ b/website/docs/destinations/server/kafka.mdx @@ -86,7 +86,7 @@ Add to your `flow.json` destinations: -The destination creates a single long-lived `kafkajs` producer during `init()` and calls `producer.connect()` before accepting events. On flow hot-swap or server shutdown, `destroy()` calls `producer.disconnect()` to flush in-flight messages and close TCP connections. +The destination creates a single long-lived `kafkajs` producer and connects it at the first push, so a flow starts without a reachable broker. A failed send fails the push (dead-letter queue, one ERROR line) instead of being logged and counted delivered, and an unknown topic is a `config` error with the setup hint. On flow hot-swap or server shutdown, `destroy()` calls `producer.disconnect()` to flush in-flight messages and close TCP connections. ## Message format diff --git a/website/docs/destinations/server/pinterest.mdx b/website/docs/destinations/server/pinterest.mdx index 2af060465..b70f667a4 100644 --- a/website/docs/destinations/server/pinterest.mdx +++ b/website/docs/destinations/server/pinterest.mdx @@ -82,6 +82,10 @@ Add to your `flow.json` destinations: +## Failures + +Pinterest can answer HTTP 200 and report an event as `failed` in the body. That counts as a failed push, not as delivered: the event reaches the dead-letter queue and the collector logs one ERROR line. A rule with `silent: true` sends nothing. + ## Client IP and user agent The destination sends the client IP as `user_data.client_ip_address` and the user agent as `user_data.client_user_agent` by default. The `ip` and `userAgent` settings resolve against `{ ingest, event }` and default to `["ingest.ip", "event.user.ip"]` and `["ingest.userAgent", "event.user.userAgent"]`, so they fill in whenever the source provides them (see the `config.ingest` recipe on the [Express source](/docs/sources/server/express#ingest-metadata)). A value your mapping sets for `user_data.client_ip_address` or `user_data.client_user_agent` wins. A value that resolves to nothing is not sent. Set either setting to `false` to turn it off, or to another mapping value to read it elsewhere. diff --git a/website/docs/destinations/server/posthog.mdx b/website/docs/destinations/server/posthog.mdx index 94acac5ca..dec0a5332 100644 --- a/website/docs/destinations/server/posthog.mdx +++ b/website/docs/destinations/server/posthog.mdx @@ -81,3 +81,20 @@ Add to your `flow.json` destinations: + +## Batching and delivery + +This destination batches by default: events are collected and sent together once +`config.batch` reaches 20 events or 10 seconds, whichever comes +first. Each event keeps its own outcome, so a failed delivery fails that event +and lands in the dead-letter queue instead of counting as delivered. Set +`config.batch: { size: 1 }` to send each event at once; there is no JSON switch +that turns the default batch off. + +A batched event answers `queued` with the reason `batch`. A server source +acknowledges the producer when the event is queued, and a failed flush is +reported (one ERROR line with the status, the failed counter, the dead-letter +queue) but not redelivered. Options are in +[Batch scheduling](/docs/mapping/rule#batch-scheduling). + +`settings.include` no longer exists; use `config.include`. diff --git a/website/docs/destinations/server/redis.mdx b/website/docs/destinations/server/redis.mdx index 55298c5b5..20f7ebfb5 100644 --- a/website/docs/destinations/server/redis.mdx +++ b/website/docs/destinations/server/redis.mdx @@ -88,6 +88,8 @@ Add to your `flow.json` destinations: The destination creates a single long-lived `ioredis` client during `init()`. On flow hot-swap or server shutdown, `destroy()` calls `client.quit()` to flush in-flight commands and close the TCP connection gracefully. User-provided clients (wired in via `_client`) are left untouched. +A failed `XADD` fails the push: the event reaches the dead-letter queue and the collector logs one ERROR line per cause and minute. Connection errors are reported through the collector too, and shutdown does not wait on a disconnected client. A rule with `silent: true` appends nothing. + ## Stream entry format Each event is appended via `XADD [MAXLEN [~] ] * ...`: diff --git a/website/docs/destinations/server/rudderstack.mdx b/website/docs/destinations/server/rudderstack.mdx index 267b0e20a..a04ddec28 100644 --- a/website/docs/destinations/server/rudderstack.mdx +++ b/website/docs/destinations/server/rudderstack.mdx @@ -83,6 +83,23 @@ Add to your `flow.json` destinations: Identity is resolved automatically from each event: `userId` defaults to `user.id` and `anonymousId` defaults to `user.session`. At least one must be present for a call to be sent. Override these paths in settings to match your event structure. +## Batching and delivery + +This destination batches by default: events are collected and sent together once +`config.batch` reaches 20 events or 10 seconds, whichever comes +first. Each event keeps its own outcome, so a failed delivery fails that event +and lands in the dead-letter queue instead of counting as delivered. Set +`config.batch: { size: 1 }` to send each event at once; there is no JSON switch +that turns the default batch off. + +A batched event answers `queued` with the reason `batch`. A server source +acknowledges the producer when the event is queued, and a failed flush is +reported (one ERROR line with the status, the failed counter, the dead-letter +queue) but not redelivered. Options are in +[Batch scheduling](/docs/mapping/rule#batch-scheduling). + +An event without `userId` or `anonymousId` fails with `missing-input`. + ## Alias support Unlike Segment, RudderStack also supports per-event `alias()` calls, useful for linking an anonymous identity to an identified user after signup or login. Configure via `mapping.settings.alias` with a `previousId` field. diff --git a/website/docs/destinations/server/segment.mdx b/website/docs/destinations/server/segment.mdx index a65d30d31..139395d33 100644 --- a/website/docs/destinations/server/segment.mdx +++ b/website/docs/destinations/server/segment.mdx @@ -80,3 +80,20 @@ Add to your `flow.json` destinations: Identity is resolved automatically from each event: `userId` defaults to `user.id` and `anonymousId` defaults to `user.session`. At least one must be present for a call to be sent. Override these paths in settings to match your event structure. + +## Batching and delivery + +This destination batches by default: events are collected and sent together once +`config.batch` reaches 15 events or 10 seconds, whichever comes +first. Each event keeps its own outcome, so a failed delivery fails that event +and lands in the dead-letter queue instead of counting as delivered. Set +`config.batch: { size: 1 }` to send each event at once; there is no JSON switch +that turns the default batch off. + +A batched event answers `queued` with the reason `batch`. A server source +acknowledges the producer when the event is queued, and a failed flush is +reported (one ERROR line with the status, the failed counter, the dead-letter +queue) but not redelivered. Options are in +[Batch scheduling](/docs/mapping/rule#batch-scheduling). + +An event without `userId` or `anonymousId` fails with `missing-input`. diff --git a/website/docs/destinations/server/sqlite.mdx b/website/docs/destinations/server/sqlite.mdx index b6da98844..682dcf866 100644 --- a/website/docs/destinations/server/sqlite.mdx +++ b/website/docs/destinations/server/sqlite.mdx @@ -90,7 +90,7 @@ Add to your `flow.json` destinations: -The destination opens the connection during `init()` and prepares the INSERT statement once. Schema creation lives in the `setup` lifecycle (see below), not in `init()`. On `destroy()` the connection is closed cleanly. User-provided clients (wired via `env.client` or `settings.sqlite._client`) are left untouched. +The destination opens the connection during `init()`. The table check and the prepared INSERT run at the first push, so a flow starts without querying the database, and a missing table fails that push with the setup hint. A failed INSERT fails the push and lands in the dead-letter queue. Schema creation lives in the `setup` lifecycle (see below), not in `init()`. On `destroy()` the connection is closed cleanly. User-provided clients (wired via `env.client` or `settings.sqlite._client`) are left untouched. ## Setup diff --git a/website/docs/destinations/server/tiktok.mdx b/website/docs/destinations/server/tiktok.mdx index 7c46c6458..029b76f82 100644 --- a/website/docs/destinations/server/tiktok.mdx +++ b/website/docs/destinations/server/tiktok.mdx @@ -82,6 +82,10 @@ Add to your `flow.json` destinations: +## Failures + +TikTok can answer HTTP 200 with a non-zero `code` in the body. That counts as a failed push, not as delivered: the event reaches the dead-letter queue and the collector logs one ERROR line. Rate limits and server errors are retryable; other rejections are not. A rule with `silent: true` sends nothing. + ## Client IP and user agent The destination sends the client IP as `context.ip` and the user agent as `context.user_agent` by default. The `ip` and `userAgent` settings resolve against `{ ingest, event }` and default to `["ingest.ip", "event.user.ip"]` and `["ingest.userAgent", "event.user.userAgent"]`, so they fill in whenever the source provides them (see the `config.ingest` recipe on the [Express source](/docs/sources/server/express#ingest-metadata)). A value that resolves to nothing is not sent. Set either setting to `false` to turn it off, or to another mapping value to read it elsewhere. diff --git a/website/docs/destinations/web/amplitude.mdx b/website/docs/destinations/web/amplitude.mdx index 535544233..d42969c3b 100644 --- a/website/docs/destinations/web/amplitude.mdx +++ b/website/docs/destinations/web/amplitude.mdx @@ -101,14 +101,23 @@ device ID and session on every page load. To keep them across page loads, map `device` and `session` in `settings.identify` (see the Destination identify example above), or set `identityStorage` yourself. +## Consent and failures + +The destination's `config.consent` decides when Amplitude is allowed to +collect: a revocation calls `setOptOut(true)` and a grant `setOptOut(false)`. +A missing `apiKey` fails the destination with a `config` error, and an Amplitude +call that fails fails the event, which shows as a failed push. + ## Plugins -`@amplitude/unified` sets up Session Replay, Experiment, and Guides & Surveys -on every init, whether or not the settings below are present; the settings -only pass options to them. With just `apiKey` set, the SDK fetches the Session -Replay remote configuration from Amplitude and loads the Guides & Surveys SDK -from `cdn.amplitude.com`. Session Replay and Experiment ship in the -destination's bundle. +With just `apiKey` set, only Amplitude analytics starts. Session Replay, +Experiment, and Guides & Surveys are set up together as soon as one of them is +enabled: setting `sessionReplay`, `experiment.deploymentKey` or `engagement` +starts all three, and the others then run with their defaults. Session Replay +then fetches its remote configuration from Amplitude and Guides & Surveys +loads its SDK from `cdn.amplitude.com`. Session Replay and Experiment ship in +the destination's bundle. To use one of them without the other two, a plugin +subset is not possible today. Session Replay records a sampled share of sessions. The share is `sessionReplay.sampleRate` (default 0) unless the project's Session Replay diff --git a/website/docs/destinations/web/clarity.mdx b/website/docs/destinations/web/clarity.mdx index f163790c2..3119145aa 100644 --- a/website/docs/destinations/web/clarity.mdx +++ b/website/docs/destinations/web/clarity.mdx @@ -96,6 +96,8 @@ Add to your `flow.json` destinations: +The Clarity script loads through walkerOS, so a blocked script fails later events visibly. A missing `apiKey` is a `config` error. + ## Consent translation Clarity expects its own category names (`analytics_Storage`, `ad_Storage`). @@ -103,7 +105,7 @@ walkerOS uses arbitrary consent keys, so the translation is explicit: configure `settings.consent` once and the destination maps every `walker consent` event into `Clarity.consentV2({ ... })`. A granted key becomes `granted`, any other value `denied`, and a mapped key missing from the consent event is left out. -Without `settings.consent`, the destination takes no action on consent changes. +Without `settings.consent`, a destination with `config.consent` sends ConsentV2 for both categories (`analytics_Storage` and `ad_Storage`): `denied` when the required consent is revoked and `granted` when it is granted. `settings.consent` only tells Clarity the consent state. The destination loads the Clarity script when the flow starts. To load Clarity only after consent is diff --git a/website/docs/destinations/web/fullstory.mdx b/website/docs/destinations/web/fullstory.mdx index f001bc4e2..67ac61ead 100644 --- a/website/docs/destinations/web/fullstory.mdx +++ b/website/docs/destinations/web/fullstory.mdx @@ -89,6 +89,11 @@ Add to your `flow.json` destinations: +Without `settings.consent`, a destination with `config.consent` stops capture +when the required consent is revoked (`shutdown`) and starts it on a grant +(`start`). A missing `orgId` is a `config` error, and a FullStory call that +fails fails the event. + ## Consent FullStory supports runtime consent via `start`/`shutdown` (recording on/off) or diff --git a/website/docs/destinations/web/gtag/index.mdx b/website/docs/destinations/web/gtag/index.mdx index f66473464..6a4ad9f41 100644 --- a/website/docs/destinations/web/gtag/index.mdx +++ b/website/docs/destinations/web/gtag/index.mdx @@ -92,6 +92,8 @@ Add to your `flow.json` destinations: +gtag.js and a GTM container load through walkerOS, so a blocked script fails later events visibly. An empty GA4 parameter name, or two names that normalize to one, drops that parameter with one ERROR line per cause and still sends the event. A rule with `silent: true` sends nothing. Settings with a key gtag does not read fail `walkeros validate`. + ## Next steps - [**Mapping**](/docs/mapping): transform events for Google's expected format diff --git a/website/docs/destinations/web/heap.mdx b/website/docs/destinations/web/heap.mdx index a31af40f4..94802e418 100644 --- a/website/docs/destinations/web/heap.mdx +++ b/website/docs/destinations/web/heap.mdx @@ -90,12 +90,12 @@ Add to your `flow.json` destinations: :::caution `loadScript: true` loads the classic Heap SDK -(`https://cdn.heapanalytics.com/js/heap-.js`). That SDK does not replay -the calls the destination queues before it finishes loading, so events pushed -earlier (such as the first page view) do not reach Heap. It also ignores -`disablePageviewAutocapture`, `disableSessionReplay`, and `ingestServer`, and -has no `startTracking()` or `stopTracking()`, so any consent command after it -has loaded stops all further Heap events (see Consent). +(`https://cdn.heapanalytics.com/js/heap-.js`). Events pushed while the +script loads wait for it and reach Heap once it is ready; if the script is +blocked or fails to load, later pushes fail and show up as failed events. Pass +SDK options in `settings.heapConfig`. The classic SDK has no switch to stop +tracking, so a consent revocation stops the events walkerOS sends but not +Heap's own autocapture (see Consent). ::: @@ -108,15 +108,12 @@ required consent keys are granted (see `heap.startTracking()` when all required keys are granted and `heap.stopTracking()` otherwise. -The classic SDK that `loadScript: true` loads has neither method. The grant -that loads Heap works, but once the script has loaded, every later -`walker consent` command, even one that repeats the current state, makes the -destination's `on('consent')` handler fail. The collector logs -`on callback failed` and holds the destination's events in its queue, so Heap -receives nothing for the rest of the page view, while `elb` still returns -`ok`. A revocation also does not stop Heap's own capture. This setup keeps -sending events only when your consent source sends no further consent command -after the grant that loads Heap. +The classic SDK that `loadScript: true` loads has neither method. Consent +commands after it has loaded do not stall the destination, but a revocation +only stops the events walkerOS sends: Heap's own autocapture keeps running +because the classic SDK cannot be switched off. To keep Heap from capturing +before consent, load Heap's script only after consent is granted, which is what +`config.consent` does with `loadScript: true`. +A destination with `config.consent` tells Matomo about a revocation: +`forgetConsentGiven` stops link tracking too, and a grant sends +`setConsentGiven`. An event no rule maps fails with `missing-input`, and an +invalid `goalId` sends the hit without the goal and logs one ERROR line per +cause. A rule with `silent: true` sends only its goal. + ## Unmapped events An event without `rule.name` that is not a page view and sets no tracking flag diff --git a/website/docs/destinations/web/meta-pixel.mdx b/website/docs/destinations/web/meta-pixel.mdx index b1622c5df..93d64378a 100644 --- a/website/docs/destinations/web/meta-pixel.mdx +++ b/website/docs/destinations/web/meta-pixel.mdx @@ -90,3 +90,8 @@ Add to your `flow.json` destinations: + +A destination with `config.consent` calls `fbq('consent', 'revoke')` when the +required consent is revoked and `fbq('consent', 'grant')` when it is granted. A +blocked `fbevents.js` fails later events visibly, and a missing `pixelId` is a +`config` error. A rule with `silent: true` sends no event. diff --git a/website/docs/destinations/web/piano.mdx b/website/docs/destinations/web/piano.mdx index 48b19574f..0c3678ea7 100644 --- a/website/docs/destinations/web/piano.mdx +++ b/website/docs/destinations/web/piano.mdx @@ -94,14 +94,15 @@ Add to your `flow.json` destinations: With `loadScript: true`, the destination adds the SDK script `https://tag.aticdn.net/piano-analytics.js` and configures `pa` with `site`, -`collectDomain` and `options` once the script has loaded. If the script fails -to load, it logs a warning. Until the SDK has loaded, the destination drops -every event it receives and counts it as delivered. That includes the page -view a browser source sends when the flow starts. - -To keep the first events, load the Piano SDK on your page before walkerOS -starts and leave `loadScript` unset. The destination then configures `pa` at -init. +`collectDomain` and `options` once the script has loaded. Events pushed while +the script loads wait for it, including the page view a browser source sends +when the flow starts. If the script is blocked or fails to load, later pushes +fail and show up as failed events. A missing `site` or `collectDomain` fails +the destination with a `config` error, and a rule with `silent: true` sends no +event. + +If Piano's SDK is already on your page before walkerOS starts, leave +`loadScript` unset. The destination then configures `pa` at init. ## Event names diff --git a/website/docs/destinations/web/piwikpro.mdx b/website/docs/destinations/web/piwikpro.mdx index 252c6038d..70ff864fa 100644 --- a/website/docs/destinations/web/piwikpro.mdx +++ b/website/docs/destinations/web/piwikpro.mdx @@ -93,6 +93,11 @@ Add to your `flow.json` destinations: +An event no rule maps fails with `missing-input`; use a rule with +`ignore: true` to keep an event out without a failure. A consent revocation for +a destination with `config.consent` turns off the link tracking walkerOS +enabled, and a blocked `ppms.js` fails later events visibly. + ## Loading the tracker With `loadScript: true`, the destination loads `ppms.js` from your account URL diff --git a/website/docs/destinations/web/segment.mdx b/website/docs/destinations/web/segment.mdx index d49f792e2..94f0c895b 100644 --- a/website/docs/destinations/web/segment.mdx +++ b/website/docs/destinations/web/segment.mdx @@ -114,7 +114,11 @@ to Segment category names: that grants all required keys. Analytics.js has no `optOut()` method, so the destination enforces consent by not loading the SDK before it is granted. A later revocation stops further events at the walkerOS consent gate; the -loaded SDK stays on the page. +loaded SDK stays on the page. Analytics.js loads on the first event after the +grant. `settings.cdnURL` reaches Analytics.js, so a proxy set there is used. + +A missing `apiKey` fails the destination with a `config` error. If Analytics.js +fails to load, later events fail visibly instead of disappearing. +`settings.collectorUrl` is required and the settings schema is closed, so `walkeros validate` rejects a key Snowplow does not read. The Snowplow script is injected once per script URL; if it fails to load, later events fail visibly. + ## User identity and privacy ### Cross-session user stitching diff --git a/website/docs/getting-started/deploy.mdx b/website/docs/getting-started/deploy.mdx index fcac4711a..c2b92f307 100644 --- a/website/docs/getting-started/deploy.mdx +++ b/website/docs/getting-started/deploy.mdx @@ -73,6 +73,7 @@ flow, bundle again and redeploy the new artifact. - Gate CI: `walkeros validate flow.json --strict` turns contract violations into errors, see [validate](/docs/getting-started/flow/validate). - Wire consent before real traffic, see the [consent guide](/docs/guides/consent). +- On a server, set `"logger": { "format": "json" }` in the `collector` block so your platform reads each log line as structured data, see [Logger](/docs/collector/logger#log-format). - Keep `flow.json` in git: deploys stay reviewable and reversible. ## Next steps diff --git a/website/docs/getting-started/flow/index.mdx b/website/docs/getting-started/flow/index.mdx index 3b3819c40..2471870c1 100644 --- a/website/docs/getting-started/flow/index.mdx +++ b/website/docs/getting-started/flow/index.mdx @@ -286,7 +286,7 @@ Transformers process events between sources and destinations. They validate, enr **Configuration options:** - **`package`** - The npm package (required for package-based steps, optional when using `code:`) - **`import`** - Named export of `package` to use (optional; the package's default export otherwise) -- **`code`** - Inline code object `{ push, type?, init? }`, used instead of `package` (see [Inline Code](#inline-code-without-packages)) +- **`code`** - Inline code object `{ push, type?, init?, pushBatch?, on?, destroy? }`, used instead of `package` (see [Inline Code](#inline-code-without-packages)) - **`config`** - Transformer-specific configuration - **`env`** - Environment-specific settings - **`before`** - Pre-transformer chain (runs before this transformer's push) @@ -447,8 +447,11 @@ Use `code` object instead of `package`: - **`push`** - The push function with `$code:` prefix (required) - **`type`** - Optional instance type identifier - **`init`** - Optional init function with `$code:` prefix +- **`pushBatch`**, **`on`**, **`destroy`** - Optional functions of a destination with `$code:` prefix (`batch` in the destination's `config` needs `pushBatch`) **Rules:** +- Any other key in `code` fails `walkeros validate` and the bundle +- A `push` that cannot send throws; a `push` that returns normally counts as delivered - Use `package` OR `code`, never both - `config` stays separate (same as package-based) - `$code:` prefix outputs raw JavaScript at bundle time diff --git a/website/docs/guides/deploy-gcp-cloudrun.mdx b/website/docs/guides/deploy-gcp-cloudrun.mdx index ade3cdbfb..cc11f6bc0 100644 --- a/website/docs/guides/deploy-gcp-cloudrun.mdx +++ b/website/docs/guides/deploy-gcp-cloudrun.mdx @@ -182,6 +182,22 @@ bq query --use_legacy_sql=false \ "SELECT name, data, timestamp FROM walkerOS.events WHERE DATE(timestamp) = CURRENT_DATE() ORDER BY timestamp DESC LIMIT 5" ``` +### Read the Logs + +Cloud Run reads JSON lines from stdout and stderr as structured logs, with +`severity` and `message`. Ask the flow for them with `format: "json"` in its +collector block, then bundle and deploy again: + +```json +"collector": { + "logger": { "level": "INFO", "format": "json" } +} +``` + +A destination that fails to deliver shows up as one `ERROR` entry per cause and +minute, with the destination id, the cause `code` and the HTTP `status` in the +`context`. See [Logger](/docs/collector/logger#failure-codes) for the codes. + ## Next Steps - Configure event mapping to transform events before sending to BigQuery diff --git a/website/docs/guides/reference-syntax.mdx b/website/docs/guides/reference-syntax.mdx index 30e68d8a8..45e8ac60d 100644 --- a/website/docs/guides/reference-syntax.mdx +++ b/website/docs/guides/reference-syntax.mdx @@ -337,8 +337,8 @@ replaces every reference before the bundler or collector sees the config. `$flow.`, `$store.`, `$secret.` and `$contract.` resolve only when the reference is the entire string value. `$var.` and `$env.` also resolve inside a longer -string. `walkeros validate` warns about a whole-value reference that would not -resolve: +string. `walkeros validate` and the bundle fail with `MALFORMED_REFERENCE` on a +whole-value reference that would not resolve: - **Malformed:** `"$flow.server.url/collect"` or `"$secret.api_key"` does not match the reference grammar (`$secret.` names are uppercase letters, digits diff --git a/website/docs/mapping/index.mdx b/website/docs/mapping/index.mdx index b259266be..78a873a4e 100644 --- a/website/docs/mapping/index.mdx +++ b/website/docs/mapping/index.mdx @@ -47,7 +47,7 @@ Everything in walkerOS mapping is built from two building blocks: - **[Mapping.Rule](./rule.mdx)**: the object on a source's or destination's `mapping[entity][action]`. It composes values into a complete rule (`name`, `data`, `condition`, `policy`, `batch`, `ignore`, `silent`, - `settings`). + `settings`). Every destination honours `silent`. When a destination doc shows `data: any`, the value you put there is a `Mapping.Value`. The whole rule around it is a `Mapping.Rule`. diff --git a/website/docs/mapping/rule.mdx b/website/docs/mapping/rule.mdx index bcb62178b..3d4a2a24d 100644 --- a/website/docs/mapping/rule.mdx +++ b/website/docs/mapping/rule.mdx @@ -40,26 +40,21 @@ that take values) accepts any form documented in When an event reaches a destination, the collector picks one rule from `mapping[entity][action]`, where entity and action are the two words of the event name. An array means "try each in order, first matching rule wins". A -`'*'` key on the entity or the action level is a wildcard. The lookup runs in -this order: - -1. If the mapping has a key for the event's entity, only that entity is - searched: the rules under the exact action when that action has a key, - otherwise the rules under the entity's `'*'` action. -2. If the mapping has no key for the entity, the `'*'` entity is searched the - same way: the rules under the exact action when that action has a key, - otherwise the rules under `'*'`. -3. If no rule was found (no key matched, or no condition passed), the `'*'` → - `'*'` rule is tried. - -So a `'*'` → `click` rule is used only for entities that have no key of their -own: with `product: { view: ... }` in the mapping, a `product click` skips it -and falls through to `'*'` → `'*'`. And when every rule under an exact action -fails its condition, the lookup goes straight to `'*'` → `'*'` and skips the -entity's own `'*'` rule. - -In this example the entity's own key wins: `product *` is used and `* click` -is never consulted. +`'*'` key on the entity or the action level is a wildcard. The lookup tries +four tiers, most specific first: + +1. `entity action`, the exact key. +2. `entity *`, the entity's own wildcard action. +3. `* action`, a wildcard entity with the exact action. +4. `* *`, the global wildcard. + +The first tier that has a rule whose `condition` passes wins. A tier that has +rules but whose conditions all fail falls through to the next tier, so a +`product *` rule is still used when every `product view` rule fails its +condition. + +In this example `product click` has no exact rule, so the second tier wins: +`product *` is used and `* click` is never consulted. .skipped` counts the event. + The collector writes a DEBUG line, and Observe shows a skip with the reason + `ignored`. - `silent: true`: the rule matched and the destination's `push()` **is** called - with the rule. On a destination that reads `silent`, `settings.identify`, + with the rule. Every destination honours `silent`: `settings.identify`, `settings.revenue`, `settings.group`, and other side effects still run and only the default forwarding call (e.g. `track()`, `capture()`, `event()`) is suppressed. Use for "identify without an event" style flows. A destination - that does not read `silent`, such as gtag, Meta or the API destinations, - sends the event as usual; the destination's page says whether it supports - it. + has no side effect to run for a rule that maps none, so the rule then sends + nothing and is not a failure. If both are set on the same rule, `ignore` wins. @@ -242,7 +243,7 @@ await getMappingEvent( } ``` -On a destination that reads `silent`, such as Amplitude, the example runs the +On a destination such as Amplitude, the example runs the `identify()` side effect on `user login` events but skips the default forwarding call. @@ -301,7 +302,11 @@ await getMappingValue(event, 'data.id', { collector }); Normalize events before they reach the collector. At a source a rule renames the event (`name`), edits it (`policy`) or drops it (`ignore`); `data`, `include`, `batch`, `settings` and `silent` have no effect there, because only -the edited event moves on. A dropped event's push result is `{ ok: true }`. +the edited event moves on. `walkeros validate` and the bundle reject those +keys (`MAPPING_NO_OP`), naming the path and what applies at a source; if such a +config still runs, the collector logs one ERROR line per step and key when the +source registers and runs the step. A dropped event's push result is +`{ ok: true, dropped: true }`. source.user.email.split('@')[1]` does both steps in one place. For an event field you can also rewrite it with a `policy` diff --git a/website/docs/sources/server/aws.mdx b/website/docs/sources/server/aws.mdx index b5908d886..eecfc5ff4 100644 --- a/website/docs/sources/server/aws.mdx +++ b/website/docs/sources/server/aws.mdx @@ -125,9 +125,19 @@ rides through instead of being dropped. | Status | Meaning | |--------|---------| -| 200 | Event processed | +| 200 | Event processed, or failed for a reason a resend cannot fix (see below) | | 400 | Rejected client input: the pipeline declared the event invalid. The body echoes the validation message, for example `Event name is required` | -| 500 | The pipeline failed to process a valid event, or an unexpected server fault | +| 500 | The pipeline failed to process a valid event for a reason a resend could fix (the body is `Event was not processed`), or an unexpected server fault (`Internal server error`) | + +A failed event is acknowledged or retried by one rule on every source, from the +push result: a failure a resend could fix (`retryable`, such as a timeout or an +unavailable vendor) answers 500, or 207 for the failed events of a batch, with +the fixed text `Event was not processed`. A failure a resend cannot fix (a +missing setting, a `4xx` from the vendor, a destination that is not ready) is +acknowledged with 200: it is in the dead-letter queue and in the log, and +sending the same event again would fail the same way. The body never carries a +destination id or the text of a step's error; the collector's ERROR line does. +An unexpected throw answers 500 `Internal server error`. Invalid input is counted on `collector.status.sources..rejected` rather than inflating `status.failed`, and is logged at warn with the reason instead of as an diff --git a/website/docs/sources/server/express.mdx b/website/docs/sources/server/express.mdx index 767fd31bf..38d6ac18c 100644 --- a/website/docs/sources/server/express.mdx +++ b/website/docs/sources/server/express.mdx @@ -193,13 +193,24 @@ push. | Status | Meaning | |--------|---------| -| 200 | Accepted (respond-first mode: accepted for processing, not proof of delivery) or, in synchronous mode, processed | +| 200 | Accepted (respond-first mode: accepted for processing, not proof of delivery) or, in synchronous mode, processed, or failed for a reason a resend cannot fix (see below) | | 400 | Rejected client input: an unparseable `application/json` body (body echoes the parser message), a batch larger than `maxBatchSize` (default 100), or, on synchronous POST, a single event the pipeline declared invalid (body echoes the validation message, e.g. `Event name is required`) | | 404 | Path, or method on that path, not configured in `paths` (JSON body `{ "success": false, "error": "Not found" }`) | | 405 | `HEAD` request on a path that serves `GET` (no body) | | 413 | Body exceeds the 1mb limit | | 415 | Unsupported charset | -| 500 | On synchronous POST: the pipeline failed to process a valid event. Otherwise: unexpected server fault | +| 500 | On synchronous POST: the pipeline failed to process a valid event for a reason a resend could fix; the body is `Event was not processed`. Otherwise: unexpected server fault (`Internal server error`) | +| 207 | On synchronous batch POST: some events failed and a resend could fix it; the body lists the failed indices | + +A failed event is acknowledged or retried by one rule on every source, from the +push result: a failure a resend could fix (`retryable`, such as a timeout or an +unavailable vendor) answers 500, or 207 for the failed events of a batch, with +the fixed text `Event was not processed`. A failure a resend cannot fix (a +missing setting, a `4xx` from the vendor, a destination that is not ready) is +acknowledged with 200: it is in the dead-letter queue and in the log, and +sending the same event again would fail the same way. The body never carries a +destination id or the text of a step's error; the collector's ERROR line does. +An unexpected throw answers 500 `Internal server error`. GET pixel requests answer 200 with the GIF in every mode, unless a step responded first, because an image endpoint cannot carry a useful 400. Rejected pixel input is visible through the diff --git a/website/docs/sources/server/fetch.mdx b/website/docs/sources/server/fetch.mdx index 71b6f4150..0a7c28c8b 100644 --- a/website/docs/sources/server/fetch.mdx +++ b/website/docs/sources/server/fetch.mdx @@ -103,10 +103,20 @@ Server sources require platform-specific handlers. For containerized deployments | Status | Meaning | |--------|---------| -| 200 | Event processed | +| 200 | Event processed, or failed for a reason a resend cannot fix (see below) | | 400 | Rejected client input: the pipeline declared the event invalid. The body echoes the validation message, for example `Event name is required` | -| 500 | The pipeline failed to process a valid event, or an unexpected server fault | -| 207 | Batch request where some events succeeded and others failed. The body carries per-index errors | +| 500 | The pipeline failed to process a valid event for a reason a resend could fix (the body is `Event was not processed`), or an unexpected server fault (`Internal server error`) | +| 207 | Batch request where some events succeeded and others failed in a way a resend could fix. The body carries per-index errors | + +A failed event is acknowledged or retried by one rule on every source, from the +push result: a failure a resend could fix (`retryable`, such as a timeout or an +unavailable vendor) answers 500, or 207 for the failed events of a batch, with +the fixed text `Event was not processed`. A failure a resend cannot fix (a +missing setting, a `4xx` from the vendor, a destination that is not ready) is +acknowledged with 200: it is in the dead-letter queue and in the log, and +sending the same event again would fail the same way. The body never carries a +destination id or the text of a step's error; the collector's ERROR line does. +An unexpected throw answers 500 `Internal server error`. Invalid input is counted on `collector.status.sources..rejected` rather than inflating `status.failed`, and is logged at warn with the reason instead of as an diff --git a/website/docs/sources/server/gcp.mdx b/website/docs/sources/server/gcp.mdx index 711d16603..d9b90b7e1 100644 --- a/website/docs/sources/server/gcp.mdx +++ b/website/docs/sources/server/gcp.mdx @@ -132,9 +132,19 @@ Send several events in one request with the shared envelope. See | Status | Meaning | |--------|---------| -| 200 | Event processed | +| 200 | Event processed, or failed for a reason a resend cannot fix (see below) | | 400 | Rejected client input: the pipeline declared the event invalid. The body echoes the validation message, for example `Event name is required` | -| 500 | The pipeline failed to process a valid event, or an unexpected server fault | +| 500 | The pipeline failed to process a valid event for a reason a resend could fix (the body is `Event was not processed`), or an unexpected server fault (`Internal server error`) | + +A failed event is acknowledged or retried by one rule on every source, from the +push result: a failure a resend could fix (`retryable`, such as a timeout or an +unavailable vendor) answers 500, or 207 for the failed events of a batch, with +the fixed text `Event was not processed`. A failure a resend cannot fix (a +missing setting, a `4xx` from the vendor, a destination that is not ready) is +acknowledged with 200: it is in the dead-letter queue and in the log, and +sending the same event again would fail the same way. The body never carries a +destination id or the text of a step's error; the collector's ERROR line does. +An unexpected throw answers 500 `Internal server error`. Invalid input is counted on `collector.status.sources..rejected` rather than inflating `status.failed`, and is logged at warn with the reason instead of as an diff --git a/website/docs/sources/server/index.mdx b/website/docs/sources/server/index.mdx index d455c0af4..ba5759d87 100644 --- a/website/docs/sources/server/index.mdx +++ b/website/docs/sources/server/index.mdx @@ -47,3 +47,22 @@ inbound requests and all provide: serving requests. They have no HTTP surface, so none of the request-shaped features above apply, and `config.ingest` does not either: there is no request to adapt. Retries and dead-lettering stay with the queue. + +## Producer acknowledgement + +Every server source tells its producer what happened to an event from the push +result, with one rule: + +| Push result | HTTP sources | SQS and Pub/Sub pull | Pub/Sub push | +|---|---|---|---| +| Delivered, queued, skipped | 200 | ack | 200 | +| Failed, a resend could fix it (`retryable`) | 500, or 207 for the failed events of a batch | nack | 500 | +| Failed, a resend cannot fix it | 200, acknowledged | ack | 200 | +| Invalid input | 400 | ack, counted as `rejected` | 400 | + +A deterministic failure is acknowledged on purpose: resending the message would +fail the same way and loop. It is visible in the destination's `failed` +counter, its dead-letter queue and one ERROR line per cause and minute. A +batched event answers `queued`, so the producer is acknowledged when the event +is queued; a failed flush is reported but the message is not redelivered. The +HTTP body never names a destination or quotes a step's error. diff --git a/website/docs/sources/server/pubsub.mdx b/website/docs/sources/server/pubsub.mdx index c3348cf5c..b53c18c46 100644 --- a/website/docs/sources/server/pubsub.mdx +++ b/website/docs/sources/server/pubsub.mdx @@ -86,7 +86,7 @@ The pull source is event-driven: `init()` opens the streaming subscription and f | `flowControl` | `{ maxMessages?, maxBytes? }` | `{ maxMessages: 100, maxBytes: 10 MB }` | Subscriber pull rate (in-flight cap). | | `ackDeadline` | `number` | `60` | Per-message ack window in seconds. | | `shutdownTimeoutMs` | `number` | `30000` | Graceful drain budget on `destroy()`. | -| `onPushError` | `'nack' \| 'ack'` | `'nack'` | Behavior when forwarding to the collector throws. | +| `onPushError` | `'nack' \| 'ack'` | `'nack'` | Behavior when forwarding to the collector throws or fails for a retryable reason. | ## Push webhook @@ -123,13 +123,13 @@ app.listen(8080);`} Pub/Sub posts each message envelope to the configured endpoint as a JSON body matching the [Pub/Sub push spec](https://cloud.google.com/pubsub/docs/push). The handler decodes the envelope, base64-decodes the `data` field, runs the configured decoder, and forwards to the collector. Status codes: -- `200`: forwarded successfully. +- `200`: forwarded successfully, or failed for a reason a redelivery cannot fix (it is in the destination's dead-letter queue and in the collector's ERROR line). - `400`: the message is permanently invalid, either a malformed envelope (missing `message.data`, invalid base64) or an event the pipeline rejected. A retry cannot fix either, so the answer is a nack. - `401`: OIDC verification failed (when enabled). -- `500`: collector push failed on an otherwise valid message; Pub/Sub will retry - per the subscription's retry policy. +- `500`: collector push failed on an otherwise valid message for a retryable + reason, or threw; Pub/Sub will retry per the subscription's retry policy. Because a permanently invalid message is nacked rather than acknowledged, Pub/Sub keeps redelivering it. Set a @@ -189,7 +189,7 @@ Topic provisioning is owned by the [Pub/Sub destination](/docs/destinations/serv | `text` | `data.toString('utf8')` (forwarded as-is) | | `raw` | The raw `Buffer` is forwarded unchanged | -A decoder that throws on malformed payloads triggers `onPushError` (pull mode): `nack` redelivers, `ack` drops with a debug log. +A decoder that throws on malformed payloads triggers `onPushError` (pull mode): `nack` redelivers, `ack` drops with a debug log. In pull mode a failure a redelivery cannot fix (a missing setting, a `4xx` from the vendor, a destination that is not ready) and invalid input are acknowledged: the failure is in the destination's dead-letter queue and in one ERROR line from the collector, and invalid input is counted as `rejected`. Only a retryable failure (a timeout, an unavailable vendor) and a thrown push follow `onPushError`. ## Emulator diff --git a/website/docs/sources/server/sqs.mdx b/website/docs/sources/server/sqs.mdx index ee5a650d3..2d252adbc 100644 --- a/website/docs/sources/server/sqs.mdx +++ b/website/docs/sources/server/sqs.mdx @@ -113,7 +113,7 @@ The provisioning identity needs `sqs:CreateQueue`, `sqs:GetQueueAttributes`, `sq | `text` | The raw `body` string is forwarded as-is. | | `raw` | The body is wrapped in `{ data: { payload: body } }`. | -A decoder that throws on malformed payloads triggers `onPushError`: `nack` lets visibility expire (SQS redelivers; eventually the DLQ if wired), `ack` deletes the message with a debug log. +A decoder that throws on malformed payloads, or a push that throws, triggers `onPushError`: `nack` lets visibility expire (SQS redelivers; eventually the DLQ if wired), `ack` deletes the message with a debug log. A push that fails for a retryable reason (a timeout, an unavailable vendor) takes the same path. A failure a redelivery cannot fix (a missing setting, a `4xx` from the vendor, a destination that is not ready) and invalid input are acknowledged and the message is deleted: the failure is in the destination's dead-letter queue and in one ERROR line from the collector, and invalid input is counted as `rejected`. ## Backpressure diff --git a/website/docs/sources/web/browser/index.mdx b/website/docs/sources/web/browser/index.mdx index c64eea888..4c51da86c 100644 --- a/website/docs/sources/web/browser/index.mdx +++ b/website/docs/sources/web/browser/index.mdx @@ -104,6 +104,17 @@ await elb('walker init', document.querySelector('#app'));`} See [Commands](/docs/sources/web/browser/commands) for complete API documentation. +## Leaving the page + +When the page goes away (`pagehide`) or turns hidden, the source sends +`walker flush`, so destination batches that have not reached their size or wait +yet are sent before the visitor is gone. The listeners are removed when the +source is destroyed. + +A failing `elbLayer` entry (an event or command the collector could not +process) is logged as one ERROR line that names the event, never its data, and +counts as failed. + ## Single-page apps With `history: true` the source follows route changes by itself. diff --git a/website/docs/sources/web/dataLayer/index.mdx b/website/docs/sources/web/dataLayer/index.mdx index 57ae9b075..16737b62a 100644 --- a/website/docs/sources/web/dataLayer/index.mdx +++ b/website/docs/sources/web/dataLayer/index.mdx @@ -103,6 +103,9 @@ dataLayer.push({ event: 'purchase', value: 99.99 }); language="javascript" /> +The source reads the page's `window` and needs no `env`, in integrated and +bundled mode alike. In tests, `env.window` replaces it. + Every event carries the page context in `source`, like browser source events: `{ type: 'dataLayer', platform: 'web', url, referrer }`, with `url` from `window.location.href` and `referrer` from `document.referrer`. Both are read diff --git a/website/docs/stores/index.mdx b/website/docs/stores/index.mdx index 58698537f..eab553960 100644 --- a/website/docs/stores/index.mdx +++ b/website/docs/stores/index.mdx @@ -187,6 +187,10 @@ const value = await collector.stores.assets.get('key');`} language="typescript" /> +## Failed reads + +A read, write or delete that fails (a 403, a 5xx, a network error) throws instead of looking like a missing key, and a stored value that cannot be decoded throws too. A key that is not stored is a normal miss and returns `undefined`. The collector treats a failed read as a miss for that event and writes one ERROR line per cause, so a failing store is visible without stopping the flow. + ## Setup lifecycle Stores may implement an optional `setup()` lifecycle for one-time, operator-time provisioning, for example creating an S3 bucket. Setup runs only when an operator explicitly invokes `walkeros setup store.`; the runtime never auto-invokes it. Opt in via `config.setup` in the flow config (`true` or an object). diff --git a/website/docs/stores/server/s3.mdx b/website/docs/stores/server/s3.mdx index b481f279e..f2fe7edd1 100644 --- a/website/docs/stores/server/s3.mdx +++ b/website/docs/stores/server/s3.mdx @@ -189,7 +189,7 @@ Set `setup: true` in the component config to enable provisioning with defaults, ### Runtime hard-fail -`storeS3Init` probes `bucketExists()` once when the collector wires the store. On a missing bucket it throws with an actionable message: +The first `get`, `set` or `delete` probes `bucketExists()` once, not the start of the flow. On a missing bucket it throws with an actionable message: ?fields=spreadsheetId` per process per spreadsheet ID. On 404 it throws with an actionable message: +The first operation (`get`, `set` or `delete`) issues a single `GET /spreadsheets/?fields=spreadsheetId` per process per spreadsheet ID. On 404 it throws with an actionable message: . Run "walkeros setup store." to ensure the sheet exists and is shared with the service account.`} diff --git a/website/docs/transformers/fingerprint.mdx b/website/docs/transformers/fingerprint.mdx index c4a173801..f272adfba 100644 --- a/website/docs/transformers/fingerprint.mdx +++ b/website/docs/transformers/fingerprint.mdx @@ -93,11 +93,11 @@ The source must extract `ip` and `userAgent` into `ingest`: Behind a load balancer, `ip` is the balancer's address; extract the client from `headers.x-forwarded-for` instead. An IPv4 client on a dual-stack server (`::ffff:203.0.113.7`) is treated as its IPv4 address. -Set an input to another [mapping value](/docs/mapping/value) to read it from elsewhere, or to `false` to leave it out. When an input resolves empty, the transformer logs one warning and hashes without it. +Set an input to another [mapping value](/docs/mapping/value) to read it from elsewhere, or to `false` to leave it out. When one input resolves empty, the transformer hashes without it; when all of them do, the event fails with `missing-input`. ## Salt -The salt is the HMAC key. Keep it secret and out of the config file (`$env.FINGERPRINT_SALT`), and give each deployment its own. Without a salt, anyone who knows the inputs can recompute the hash, and an IP hash can be reversed by trying every address. The transformer still runs without one, but logs a warning at startup. +The salt is the HMAC key. Keep it secret and out of the config file (`$env.FINGERPRINT_SALT`), and give each deployment its own. Without a salt, anyone who knows the inputs can recompute the hash, and an IP hash can be reversed by trying every address. `settings.salt` is required: without it the transformer does not start. An event whose IP, user agent and fields all resolve empty fails with `missing-input` instead of getting the hash every other visitor gets. ## Rotation diff --git a/website/docs/transformers/ga4.mdx b/website/docs/transformers/ga4.mdx index 23bd4d3c3..8b4017252 100644 --- a/website/docs/transformers/ga4.mdx +++ b/website/docs/transformers/ga4.mdx @@ -145,9 +145,9 @@ The transformer expects the source to populate `ctx.ingest` with these keys: | `path` | `string` | no | Request path. Use it to gate the `before` chain when the source also receives other traffic. | | `body` | `string` | no | Raw POST body. Multi-event batches are `\n`-separated lines. | -`config.ingest` must use the `map` operator with **direct field paths on the request scope** (no `req.` prefix), as shown above. A bare object like `{ "url": "req.url" }` is silently inert: without an operator no field is extracted, `ctx.ingest` stays empty, and the transformer drops the hit. +`config.ingest` must use the `map` operator with **direct field paths on the request scope** (no `req.` prefix), as shown above. A bare object like `{ "url": "req.url" }` has no operator: `walkeros validate` and the bundle reject it (`MAPPING_VALUE_NO_OPERATOR`). If it still runs, the value is dropped with an ERROR line from the collector, `ctx.ingest` stays empty, and the transformer fails every hit with `missing-input`. -If `url` is missing or not a string the transformer drops the event silently. +If `url` is missing or not a string the transformer fails the event with the `missing-input` code instead of dropping it. A request that cannot be decoded fails with the `transformer` code. Both show as one ERROR line per cause and minute, and the source answers 200 because resending the same request would fail again. ### GA4 hits and walkerOS events on one source diff --git a/website/static/schema/flow/v4.json b/website/static/schema/flow/v4.json index 78f384819..f939de8bf 100644 --- a/website/static/schema/flow/v4.json +++ b/website/static/schema/flow/v4.json @@ -251,6 +251,18 @@ "init": { "description": "Optional initialization function. Use $code: prefix for inline JavaScript.", "type": "string" + }, + "pushBatch": { + "description": "Optional batch push function (destinations with config.batch). Use $code: prefix for inline JavaScript.", + "type": "string" + }, + "on": { + "description": "Optional event handler, called with the event type and context (consent, ready, run). Use $code: prefix for inline JavaScript.", + "type": "string" + }, + "destroy": { + "description": "Optional destroy function, called at shutdown. Use $code: prefix for inline JavaScript.", + "type": "string" } }, "required": ["push"], @@ -747,6 +759,46 @@ "description": "Invoke elb('walker ', in) instead of pushing in as an event", "type": "string", "enum": ["config", "consent", "user", "run"] + }, + "outcome": { + "description": "Reported outcome of a failure or queue example (public: false)", + "type": "object", + "properties": { + "status": { + "type": "string", + "enum": ["failed", "queued"], + "description": "Status in the result" + }, + "code": { + "type": "string", + "description": "Reason code, e.g. transport or config" + }, + "retryable": { + "description": "Whether a retry can help", + "type": "boolean" + } + }, + "required": ["status", "code"], + "additionalProperties": false + }, + "respond": { + "description": "What the boundary answers for this example", + "type": "object", + "properties": { + "status": { + "description": "HTTP status to answer", + "type": "number" + }, + "body": { + "description": "Response body to answer" + }, + "error": { + "description": "A network error or a timeout instead of a response", + "type": "string", + "enum": ["network", "timeout"] + } + }, + "additionalProperties": false } }, "additionalProperties": false, From e54103ffda017fac2c33512702cc4f561b057f58 Mon Sep 17 00:00:00 2001 From: alexanderkirtzel Date: Wed, 21 Oct 2026 07:38:27 +0000 Subject: [PATCH 23/41] kit seat fixes --- .../src/dev/__tests__/conformance.test.ts | 37 ++++++++++++++++++- .../__tests__/fixture-package/src/index.ts | 14 +++++++ .../__tests__/fixture-package/src/vendor.ts | 11 ++++++ packages/collector/src/dev/conformance.ts | 31 +++++++++++++--- packages/collector/src/dev/runExamples.ts | 9 ++++- .../web/core/src/__tests__/loadVendor.test.ts | 27 ++++++++++++++ packages/web/core/src/loadVendor.ts | 23 ++++++++++-- 7 files changed, 140 insertions(+), 12 deletions(-) diff --git a/packages/collector/src/dev/__tests__/conformance.test.ts b/packages/collector/src/dev/__tests__/conformance.test.ts index 6770b9c00..0c2e21c5b 100644 --- a/packages/collector/src/dev/__tests__/conformance.test.ts +++ b/packages/collector/src/dev/__tests__/conformance.test.ts @@ -1,7 +1,7 @@ import type { Flow, Store, Trigger, WalkerOS } from '@walkeros/core'; import { isObject } from '@walkeros/core'; import { startFlow } from '../../flow'; -import { conformance } from '../conformance'; +import { assertProfileEnvironment, conformance } from '../conformance'; import { fixtures } from './fixtures'; import { batching, @@ -12,6 +12,7 @@ import { good, grouping, loud, + loudCoded, rawHttps, swallowing, swallowingByDefault, @@ -27,6 +28,7 @@ import { missing, remote } from './fixture-package/src/store'; import { chatty, forwarding } from './fixture-package/src/source'; import type { Vendor } from './fixture-package/src/vendor'; import { + alwaysRevoking, consentAware, consentIgnoring, initCalling, @@ -226,6 +228,30 @@ describe('conformance kit', () => { }); }); + it("fails item 2 for a destination that logs its own coded line next to the collector's", async () => { + await expect( + conformance('destination', loudCoded, { profile: 'http', ...fixtures }), + ).rejects.toMatchObject({ + items: [2], + message: expect.stringContaining( + 'HTTP 500 writes 2 ERROR lines for transport', + ), + }); + }); + + it('refuses a browser profile without a document, and nothing else', () => { + expect(() => assertProfileEnvironment('web-sdk', false)).toThrow( + /profile web-sdk is for web steps and needs a document/, + ); + expect(() => assertProfileEnvironment('vendor-global', false)).toThrow( + /profile vendor-global is for web steps and needs a document/, + ); + expect(() => assertProfileEnvironment('sdk-env', false)).not.toThrow(); + expect(() => assertProfileEnvironment('http', false)).not.toThrow(); + expect(() => assertProfileEnvironment('web-sdk', true)).not.toThrow(); + expect(() => assertProfileEnvironment('vendor-global')).not.toThrow(); + }); + it('refuses fake timers with a clear message', async () => { jest.useFakeTimers(); await expect( @@ -653,6 +679,15 @@ describe('conformance kit, vendor-global profile', () => { ).rejects.toMatchObject({ items: [8] }); }); + it('fails item 8 for a destination that revokes on the grant too', async () => { + await expect( + conformance('destination', alwaysRevoking, options), + ).rejects.toMatchObject({ + items: [8], + message: expect.stringContaining("the grant makes the revoke's calls"), + }); + }); + it('fails item 8 for a destination that ignores the revocation', async () => { await expect( conformance('destination', consentIgnoring, options), diff --git a/packages/collector/src/dev/__tests__/fixture-package/src/index.ts b/packages/collector/src/dev/__tests__/fixture-package/src/index.ts index 4bc37c213..6cdabfb65 100644 --- a/packages/collector/src/dev/__tests__/fixture-package/src/index.ts +++ b/packages/collector/src/dev/__tests__/fixture-package/src/index.ts @@ -170,3 +170,17 @@ export const grouping: Destination.Instance = { await post(requireUrl(context.config), JSON.stringify({ group })); }, }; + +/** Logs its own coded ERROR line next to the collector's (item 2). */ +export const loudCoded: Destination.Instance = { + ...good, + type: 'loud-coded', + push: async (event, context) => { + try { + await good.push(event, context); + } catch (error) { + context.logger.error('Vendor request failed', { code: 'transport' }); + throw error; + } + }, +}; diff --git a/packages/collector/src/dev/__tests__/fixture-package/src/vendor.ts b/packages/collector/src/dev/__tests__/fixture-package/src/vendor.ts index 480ed3612..ec99a1212 100644 --- a/packages/collector/src/dev/__tests__/fixture-package/src/vendor.ts +++ b/packages/collector/src/dev/__tests__/fixture-package/src/vendor.ts @@ -193,3 +193,14 @@ function pageScript(reports: boolean): Destination.Instance { export const pageScriptVendor = pageScript(true); export const pageScriptIgnoring = pageScript(false); + +/** Sends "revoked" to its vendor on every consent change, grant included. */ +export const alwaysRevoking: Destination.Instance = { + ...vendorGlobal, + type: 'always-revoking', + on: (type, { env }) => { + const vendor = env.vendor; + if (type !== 'consent' || !isVendor(vendor)) return; + vendor.consent(false); + }, +}; diff --git a/packages/collector/src/dev/conformance.ts b/packages/collector/src/dev/conformance.ts index bb5cbdbad..01f639479 100644 --- a/packages/collector/src/dev/conformance.ts +++ b/packages/collector/src/dev/conformance.ts @@ -167,18 +167,19 @@ const errorsIn = (logs: LogLine[]) => logs.filter((line) => line.level === Level.ERROR).length; /** - * ERROR lines of the cause under test (item 2): the step's cause-log lines - * with that code, read from their structured context, plus lines without a - * code (the cause log always writes one, so such a line is the step's own - * report of the failure). Lines of other causes do not count. + * ERROR lines of the cause under test (item 2), from their structured + * context: the cause log's line for the step and that code, plus every line + * that does not come from the cause log (it always writes a `step`), so a + * step's own report of the failure counts, coded or not. Cause-log lines of + * other causes do not count. */ function causeLines(logs: LogLine[], code: string): number { const step = stepId('destination', STEP_ID); return logs.filter( ({ level, context }) => level === Level.ERROR && - (context.code === undefined || - (context.code === code && context.step === step)), + (typeof context.step !== 'string' || + (context.step === step && context.code === code)), ).length; } @@ -1456,6 +1457,23 @@ type KitArgs = | ['store', StoreStep] | ['source', SourceStep]; +/** Profiles of steps that run in a browser: they need a document. */ +const BROWSER_PROFILES: Profile[] = ['web-sdk', 'vendor-global']; + +/** + * Refuses a browser profile in a test environment without a document (a + * server package on `web-sdk` would skip item 5 by mistake). + */ +export function assertProfileEnvironment( + profile: Profile, + hasDocument = typeof document !== 'undefined', +): void { + if (BROWSER_PROFILES.includes(profile) && !hasDocument) + throw new Error( + `profile ${profile} is for web steps and needs a document: run it in a jsdom test environment, or use a server profile (http, sdk-env, socket)`, + ); +} + const KIND_PROFILES: Record = { destination: ['http', 'sdk-env', 'web-sdk', 'vendor-global', 'socket'], transformer: ['transformer'], @@ -1517,6 +1535,7 @@ export async function conformance(...given: ConformanceArgs): Promise { throw new Error( `profile ${options.profile} is not a ${kind} profile (${KIND_PROFILES[kind].join(', ')})`, ); + assertProfileEnvironment(options.profile); const args = kitArgs(given); const ctx = contextOf(options); diff --git a/packages/collector/src/dev/runExamples.ts b/packages/collector/src/dev/runExamples.ts index 00d4ad0e6..8144a31c1 100644 --- a/packages/collector/src/dev/runExamples.ts +++ b/packages/collector/src/dev/runExamples.ts @@ -684,11 +684,18 @@ export async function checkDestinationExample( const session = await openDestination(step, setup); try { - if (revoked.length) await session.run({ command: 'consent', data: grant }); + const granted = revoked.length + ? await session.run({ command: 'consent', data: grant }) + : undefined; const { result, effects } = await session.run(input); const problems: string[] = []; const out = compareOut(effects, example); if (out) problems.push(out); + // The grant must not revoke: its calls never equal the revoke's out. + if (granted && example.out?.length && isEqual(granted.effects, example.out)) + problems.push( + `the grant makes the revoke's calls (${show(example.out)}): it must not revoke`, + ); const outcome = compareOutcome(outcomeOf(result), example.outcome); if (outcome) problems.push(outcome); return problems; diff --git a/packages/web/core/src/__tests__/loadVendor.test.ts b/packages/web/core/src/__tests__/loadVendor.test.ts index 20f6e8ff6..0aeb19b1a 100644 --- a/packages/web/core/src/__tests__/loadVendor.test.ts +++ b/packages/web/core/src/__tests__/loadVendor.test.ts @@ -74,6 +74,33 @@ describe('loadVendor', () => { expect(scriptsOf(src)).toHaveLength(1); }); + it("adopts another bundle's script that already loaded: ready at once", async () => { + const src = nextSrc(); + // Another bundle injected and loaded it before this one asks. + const foreign = document.createElement('script'); + foreign.src = src; + foreign.setAttribute(VENDOR_SCRIPT_MARKER, ''); + document.head.appendChild(foreign); + sdk = { track: () => undefined }; + + await expect(loadVendor(src, get).ready()).resolves.toBe(sdk); + expect(scriptsOf(src)).toHaveLength(1); + }); + + it("watches another bundle's script that has not loaded yet", async () => { + const src = nextSrc(); + const foreign = document.createElement('script'); + foreign.src = src; + foreign.setAttribute(VENDOR_SCRIPT_MARKER, ''); + document.head.appendChild(foreign); + + const ready = loadVendor(src, get).ready(); + sdk = { track: () => undefined }; + fire(src, 'load'); + await expect(ready).resolves.toBe(sdk); + expect(scriptsOf(src)).toHaveLength(1); + }); + it('resolves with get() after load', async () => { const src = nextSrc(); const ready = loadVendor(src, get).ready(); diff --git a/packages/web/core/src/loadVendor.ts b/packages/web/core/src/loadVendor.ts index 4ec4069ff..67cea814c 100644 --- a/packages/web/core/src/loadVendor.ts +++ b/packages/web/core/src/loadVendor.ts @@ -59,14 +59,13 @@ function scriptOf(src: string): HTMLScriptElement | undefined { function inject( src: string, timeout: number, - attributes: Record = {}, + attributes: Record, + available: () => boolean, ): Injected { const existing = scriptOf(src); const known = existing && injected.get(existing); if (known) return known; - // A marked script without state (another bundle's) is watched, not doubled. - const script = existing ?? document.createElement('script'); const entry: Injected = { state: 'loading', expired: false, @@ -74,6 +73,16 @@ function inject( waiting: [], }; + // A marked script without state (another bundle's) is adopted, not + // doubled. Its `load` may have fired already: with the vendor there, it is + // ready at once; otherwise it is watched like an own script. + if (existing && available()) { + entry.state = 'loaded'; + injected.set(existing, entry); + return entry; + } + const script = existing ?? document.createElement('script'); + const settle = (outcome: Outcome) => { const waiting = entry.waiting; entry.waiting = []; @@ -136,11 +145,17 @@ export function loadVendor( get: () => T | undefined, options: LoadVendorOptions = {}, ): Vendor { + const available = () => { + const vendor = get(); + return vendor !== undefined && vendor !== null; + }; + const script = src ? inject( src, resolveDestinationTimeout(options.timeout), - options.attributes, + options.attributes ?? {}, + available, ) : undefined; From 4efe0b98afd60175e746f45e49ac59df125abfb8 Mon Sep 17 00:00:00 2001 From: alexanderkirtzel Date: Wed, 21 Oct 2026 07:41:02 +0000 Subject: [PATCH 24/41] source log levels --- .../express/src/__tests__/index.test.ts | 11 +++-- packages/server/sources/express/src/index.ts | 7 +-- .../fetch/src/__tests__/acknowledge.test.ts | 31 ++++++++++++- .../sources/fetch/src/__tests__/index.test.ts | 45 ++++++++++++++++++- packages/server/sources/fetch/src/index.ts | 15 ++++--- 5 files changed, 95 insertions(+), 14 deletions(-) diff --git a/packages/server/sources/express/src/__tests__/index.test.ts b/packages/server/sources/express/src/__tests__/index.test.ts index 95056c88a..21237a956 100644 --- a/packages/server/sources/express/src/__tests__/index.test.ts +++ b/packages/server/sources/express/src/__tests__/index.test.ts @@ -1096,7 +1096,7 @@ describe('sourceExpress', () => { expect(mockLogger.error).not.toHaveBeenCalled(); }); - it('POST async warns when the push settles ok:false after the ack', async () => { + it('POST async traces at DEBUG when the push settles ok:false after the ack', async () => { const mockLogger = createMockLogger(); const okFalsePush = jest.fn().mockResolvedValue({ ok: false }); const source = await sourceExpress( @@ -1119,14 +1119,16 @@ describe('sourceExpress', () => { await new Promise((resolve) => setImmediate(resolve)); expect(res.statusCode).toBe(200); - expect(mockLogger.warn).toHaveBeenCalledWith( + // Per event, so DEBUG: the collector's cause line is the visible one. + expect(mockLogger.debug).toHaveBeenCalledWith( 'Event not processed after ack', { failed: [] }, ); + expect(mockLogger.warn).not.toHaveBeenCalled(); expect(mockLogger.error).not.toHaveBeenCalled(); }); - it('GET async {GET:true} warns when the push settles ok:false after the GIF', async () => { + it('GET async {GET:true} traces at DEBUG when the push settles ok:false after the GIF', async () => { const mockLogger = createMockLogger(); const okFalsePush = jest.fn().mockResolvedValue({ ok: false }); const source = await sourceExpress( @@ -1147,10 +1149,11 @@ describe('sourceExpress', () => { ); await new Promise((resolve) => setImmediate(resolve)); - expect(mockLogger.warn).toHaveBeenCalledWith( + expect(mockLogger.debug).toHaveBeenCalledWith( 'Event not processed after ack', { failed: [] }, ); + expect(mockLogger.warn).not.toHaveBeenCalled(); }); it('POST async stays silent when the push settles ok:true', async () => { diff --git a/packages/server/sources/express/src/index.ts b/packages/server/sources/express/src/index.ts index fa2a9ba5c..59207038f 100644 --- a/packages/server/sources/express/src/index.ts +++ b/packages/server/sources/express/src/index.ts @@ -30,8 +30,9 @@ const toError = (value: unknown): Error => /** * Settle handler for respond-first pushes: the HTTP ack already went out, * so a pipeline decline (ok:false) or a rejection can only surface here. - * Warn on decline; the collector has already error-logged a thrown pipeline - * failure, this line ties the outcome to the request that was acked. + * A decline is a DEBUG trace per event: the collector's cause line already + * reports its failure once per cause; this line ties the outcome to the + * request that was acked. * Invalid-input rejections stay silent here: the collector already warned * and counted them at the pipeline gate. */ @@ -43,7 +44,7 @@ const settleAfterAck = ( .then( (settled) => { if (settled?.ok === false && settled.invalid !== true) { - logger.warn('Event not processed after ack', { + logger.debug('Event not processed after ack', { failed: settled.failed ? Object.keys(settled.failed) : [], }); } diff --git a/packages/server/sources/fetch/src/__tests__/acknowledge.test.ts b/packages/server/sources/fetch/src/__tests__/acknowledge.test.ts index 96da1b020..aad0865fd 100644 --- a/packages/server/sources/fetch/src/__tests__/acknowledge.test.ts +++ b/packages/server/sources/fetch/src/__tests__/acknowledge.test.ts @@ -1,4 +1,4 @@ -import { StepError } from '@walkeros/core'; +import { Level, StepError } from '@walkeros/core'; import type { Destination } from '@walkeros/core'; import { examples } from '../dev'; import type { Result } from '../examples/trigger'; @@ -86,6 +86,35 @@ describe('Fetch acknowledgement', () => { expect(result.body).toMatchObject({ success: true, processed: 2 }); }); + it('traces a not processed event at DEBUG only, single and batch', async () => { + const lines: Array<[Level, string]> = []; + const instance = await examples.createTrigger({ + logger: { + level: 'DEBUG', + handler: (level, message) => void lines.push([level, message]), + }, + sources: { fetch: { code: sourceFetch } }, + destinations: { vendorX: { code: failing(retryable()) } }, + }); + try { + const trigger = instance.trigger(); + const url = 'https://collect.example/collect'; + await trigger({ method: 'POST', url, body: { name: 'page view' } }); + await trigger({ + method: 'POST', + url, + body: { batch: [{ name: 'page view' }] }, + }); + } finally { + await instance.flow?.collector.command('shutdown'); + } + // Per event lines stay at DEBUG; the collector's cause line is the ERROR. + expect(lines).toContainEqual([Level.DEBUG, 'Event processing failed']); + expect(lines).toContainEqual([Level.DEBUG, 'Batch event 0 not processed']); + expect(lines.filter(([level]) => level === Level.WARN)).toEqual([]); + expect(lines.some(([level]) => level === Level.ERROR)).toBe(true); + }); + it('answers a batch with a retryable failure with 207', async () => { const result = await post(failing(retryable()), { batch: [{ name: 'page view' }, { name: 'order complete' }], diff --git a/packages/server/sources/fetch/src/__tests__/index.test.ts b/packages/server/sources/fetch/src/__tests__/index.test.ts index 78a039edc..4c396b4a4 100644 --- a/packages/server/sources/fetch/src/__tests__/index.test.ts +++ b/packages/server/sources/fetch/src/__tests__/index.test.ts @@ -224,6 +224,7 @@ describe('sourceFetch', () => { const errorPush = jest .fn() .mockRejectedValue(new Error('Collector error')); + const logger = createMockLogger(); const source = await sourceFetch( createSourceContext( @@ -232,7 +233,7 @@ describe('sourceFetch', () => { push: errorPush as never, command: mockCommand as never, elb: jest.fn() as never, - logger: createMockLogger(), + logger, }, ), ); @@ -255,6 +256,48 @@ describe('sourceFetch', () => { error: 'Internal server error', }); expect(JSON.stringify(responseBody)).not.toContain('Collector error'); + // The collector never saw it, so this ERROR is the only signal. + expect(logger.error).toHaveBeenCalledWith('Event processing failed', { + error: 'Collector error', + }); + expect(logger.warn).not.toHaveBeenCalled(); + }); + + it('logs a thrown push in a batch at ERROR, never in the answer', async () => { + const errorPush = jest + .fn() + .mockRejectedValue(new Error('Collector error')); + const logger = createMockLogger(); + + const source = await sourceFetch( + createSourceContext( + {}, + { + push: errorPush as never, + command: mockCommand as never, + elb: jest.fn() as never, + logger, + }, + ), + ); + + const response = await source.push( + new Request('https://example.com/collect', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ batch: [{ name: 'page view' }] }), + }), + ); + + expect(response.status).toBe(207); + expect(await response.json()).toMatchObject({ + errors: [{ index: 0, error: 'Internal server error' }], + }); + expect(logger.error).toHaveBeenCalledWith( + 'Batch event 0 processing failed', + { error: 'Collector error' }, + ); + expect(logger.warn).not.toHaveBeenCalled(); }); it('answers 400 with the reason when the push settles an invalid-input rejection', async () => { diff --git a/packages/server/sources/fetch/src/index.ts b/packages/server/sources/fetch/src/index.ts index 153173f15..033f845c4 100644 --- a/packages/server/sources/fetch/src/index.ts +++ b/packages/server/sources/fetch/src/index.ts @@ -201,9 +201,9 @@ export const sourceFetch: Source.Init = async (context) => { // Forward event directly — validation is not the source's responsibility. const result = await processEvent(events[0] ?? {}, envPush, logger); if (result.error) { - // A thrown push is already error-logged inside processEvent; - // this covers settled declines only. - logger.warn('Event processing failed', { error: result.error }); + // A per-request trace, so DEBUG: a decline is reported by the + // collector's cause line, a thrown push at ERROR in processEvent. + logger.debug('Event processing failed', { error: result.error }); return createJsonResponse( { success: false, error: result.error }, result.status ?? 500, @@ -270,11 +270,16 @@ async function processBatch( try { const result = await push(event as WalkerOS.DeepPartialEvent); const outcome = pushResultToOutcome(result); - if (outcome.error) logger.warn(`Batch event ${i} not processed`); + // Per event, so DEBUG: the collector's cause line reports it. + if (outcome.error) logger.debug(`Batch event ${i} not processed`); outcomes.push(outcome); } catch (error) { + // A rejected push never reached the collector's cause log, so this + // ERROR is its only signal; the message is logged, never answered. outcomes.push({ error: INTERNAL_SERVER_ERROR }); - logger.warn(`Batch event ${i} processing failed`, error); + logger.error(`Batch event ${i} processing failed`, { + error: error instanceof Error ? error.message : 'Unknown error', + }); } } From 22be94a7dd03955f6489e9455973dfeb86865425 Mon Sep 17 00:00:00 2001 From: alexanderkirtzel Date: Wed, 21 Oct 2026 07:50:42 +0000 Subject: [PATCH 25/41] amplitude deps --- package-lock.json | 4 ++++ packages/server/destinations/amplitude/package.json | 1 + packages/web/destinations/amplitude/package.json | 3 +++ 3 files changed, 8 insertions(+) diff --git a/package-lock.json b/package-lock.json index 613aec29c..ba73e1b93 100644 --- a/package-lock.json +++ b/package-lock.json @@ -49254,6 +49254,7 @@ ], "license": "MIT", "dependencies": { + "@amplitude/analytics-core": "2.48.2", "@amplitude/analytics-node": "^1.5.53", "@walkeros/core": "4.8.1", "@walkeros/server-core": "4.8.1" @@ -50375,6 +50376,9 @@ ], "license": "MIT", "dependencies": { + "@amplitude/engagement-browser": "^1.0.3", + "@amplitude/plugin-experiment-browser": "1.0.0-beta.28", + "@amplitude/plugin-session-replay-browser": "1.31.0", "@amplitude/unified": "^1.0.16", "@walkeros/core": "4.8.1", "@walkeros/web-core": "4.8.1" diff --git a/packages/server/destinations/amplitude/package.json b/packages/server/destinations/amplitude/package.json index f785c5b36..bfb3ce687 100644 --- a/packages/server/destinations/amplitude/package.json +++ b/packages/server/destinations/amplitude/package.json @@ -35,6 +35,7 @@ "update": "npx npm-check-updates -u && npm update" }, "dependencies": { + "@amplitude/analytics-core": "2.48.2", "@amplitude/analytics-node": "^1.5.53", "@walkeros/core": "4.8.1", "@walkeros/server-core": "4.8.1" diff --git a/packages/web/destinations/amplitude/package.json b/packages/web/destinations/amplitude/package.json index 46287e779..a015fea6c 100644 --- a/packages/web/destinations/amplitude/package.json +++ b/packages/web/destinations/amplitude/package.json @@ -38,6 +38,9 @@ "update": "npx npm-check-updates -u && npm update" }, "dependencies": { + "@amplitude/engagement-browser": "^1.0.3", + "@amplitude/plugin-experiment-browser": "1.0.0-beta.28", + "@amplitude/plugin-session-replay-browser": "1.31.0", "@amplitude/unified": "^1.0.16", "@walkeros/core": "4.8.1", "@walkeros/web-core": "4.8.1" From a30ea7e2c17783b296729a0693f22c6dd17536df Mon Sep 17 00:00:00 2001 From: alexanderkirtzel Date: Wed, 21 Oct 2026 07:55:34 +0000 Subject: [PATCH 26/41] logger seat fixes --- .../unit/bundle/server-entry-logger.test.ts | 77 +++++++++-- packages/cli/src/commands/bundle/bundler.ts | 18 ++- packages/cli/src/commands/setup/index.ts | 2 +- packages/collector/src/report-error.ts | 2 +- packages/core/src/__tests__/node.test.ts | 27 ++++ packages/core/src/node.ts | 123 +++++++++++------- .../src/__tests__/runtime-logging.test.ts | 17 +++ packages/runner/src/runtime-logging.ts | 15 ++- website/docs/collector/logger.mdx | 6 +- 9 files changed, 214 insertions(+), 73 deletions(-) diff --git a/packages/cli/src/__tests__/unit/bundle/server-entry-logger.test.ts b/packages/cli/src/__tests__/unit/bundle/server-entry-logger.test.ts index 329a50d63..eaf829e4c 100644 --- a/packages/cli/src/__tests__/unit/bundle/server-entry-logger.test.ts +++ b/packages/cli/src/__tests__/unit/bundle/server-entry-logger.test.ts @@ -18,7 +18,12 @@ export function wireConfig(data) { if (data.withHandler) config.logger = { ...config.logger, - handler: (level, message) => calls.push(['flow', message]), + handler: (level, message, context, scope, originalHandler) => { + calls.push(['flow', message]); + // The documented custom-handler pattern: forward, then print. + if (data.withHandler === 'original') + originalHandler(level, message, context, scope); + }, }; return config; } @@ -33,29 +38,34 @@ export async function startFlow(config) { } `; +interface HostState { + format: Logger.Format; + debug: boolean; +} + describe('generateServerEntry merges the flow logger with the host logger', () => { const dirs: string[] = []; let errorSpy: jest.SpyInstance; let logSpy: jest.SpyInstance; + let warnSpy: jest.SpyInstance; beforeEach(() => { errorSpy = jest.spyOn(console, 'error').mockImplementation(() => {}); logSpy = jest.spyOn(console, 'log').mockImplementation(() => {}); + warnSpy = jest.spyOn(console, 'warn').mockImplementation(() => {}); }); afterEach(() => { errorSpy.mockRestore(); logSpy.mockRestore(); + warnSpy.mockRestore(); for (const dir of dirs) fs.rmSync(dir, { recursive: true, force: true }); dirs.length = 0; }); async function run( data: Record, - context: ( - format: { current: Logger.Format }, - calls: unknown[], - ) => Record, + context: (host: HostState, calls: unknown[]) => Record, ): Promise<{ ring: Array<[Level, string, LogEntry]>; calls: unknown[] }> { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'server-entry-logger-')); dirs.push(dir); @@ -80,13 +90,15 @@ describe('generateServerEntry merges the flow logger with the host logger', () = const calls: unknown[] = stage.calls; const ring: Array<[Level, string, LogEntry]> = []; - const format: { current: Logger.Format } = { current: 'text' }; - const extra = context(format, calls); + // The runner's side of the contract: the flow's format and its DEBUG + // level, learned through onLogger, drive the host handler. + const host: HostState = { format: 'text', debug: false }; + const extra = context(host, calls); await module.default({ ...extra, logger: createCLILoggerConfig({ - verbose: extra.verbose === true, - format: () => format.current, + verbose: () => extra.verbose === true || host.debug, + format: () => host.format, onLine: (level, message, line) => { ring.push([level, message, line]); calls.push(['host', message]); @@ -102,10 +114,10 @@ describe('generateServerEntry merges the flow logger with the host logger', () = const settings: unknown[] = []; const { ring } = await run( { collector: { logger: { level: 'WARN', format: 'json' } } }, - (format) => ({ + (host) => ({ onLogger: (value: { format?: Logger.Format }) => { settings.push(value); - if (value.format) format.current = value.format; + if (value.format) host.format = value.format; }, }), ); @@ -132,7 +144,7 @@ describe('generateServerEntry merges the flow logger with the host logger', () = it('tells the host the flow settings before the collector starts', async () => { const { calls } = await run( { collector: { logger: { format: 'json' } } }, - (_format, calls) => ({ + (_host, calls) => ({ onLogger: (value: unknown) => calls.push(['onLogger', value]), }), ); @@ -155,6 +167,47 @@ describe('generateServerEntry merges the flow logger with the host logger', () = ]); }); + it("prints the flow's DEBUG lines without --verbose", async () => { + const { ring } = await run( + { collector: { logger: { level: 'DEBUG' } } }, + (host) => ({ + onLogger: (value: { level?: unknown }) => { + host.debug = value.level === 'DEBUG'; + }, + }), + ); + expect(ring.map(([level]) => level)).toEqual([ + Level.DEBUG, + Level.INFO, + Level.WARN, + Level.ERROR, + ]); + expect(logSpy.mock.calls.map((call) => call[0])).toEqual([ + ' [dest] debug line', + '[dest] info line', + '[dest] warn line', + ]); + }); + + it('a flow handler calling originalHandler still prints exactly one masked line per log call', async () => { + const { calls } = await run( + { + collector: { logger: { level: 'WARN' } }, + withHandler: 'original', + }, + () => ({}), + ); + // Two calls pass WARN (warn, error): one line each, both from the host. + expect( + calls.filter((call) => Array.isArray(call) && call[0] === 'flow'), + ).toHaveLength(2); + expect(warnSpy).not.toHaveBeenCalled(); + expect(logSpy.mock.calls).toEqual([['[dest] warn line']]); + expect(errorSpy.mock.calls).toEqual([ + ['[dest] Push failed {"code":"transport","status":503,"retryable":true}'], + ]); + }); + it('keeps the host default when the flow sets no logger', async () => { const { ring } = await run({}, () => ({})); // The host passes every level to its handler, which hides DEBUG unless diff --git a/packages/cli/src/commands/bundle/bundler.ts b/packages/cli/src/commands/bundle/bundler.ts index dcf675403..997cbac51 100644 --- a/packages/cli/src/commands/bundle/bundler.ts +++ b/packages/cli/src/commands/bundle/bundler.ts @@ -1864,24 +1864,28 @@ export { startFlow };`; * ring tap), optionally `context.logLevel` (a level it forces, `--verbose`) * and `context.onLogger` (receives the flow's own `{ level, format }` before * the collector starts). The flow's level and format win over the host's; - * a flow handler runs first, then the host's. + * a flow handler runs first, then the host's, which runs once per line, + * always. The flow handler's `originalHandler` does nothing: the host prints + * every line, masked and in the flow's format. */ const HOST_LOGGER_MERGE = ` // The host's logger (masking, its error ring) joins the flow's own: the // flow's level and format win, unless the host forces a level (--verbose); - // the flow's handler runs first, then the host's, so neither replaces the - // other. The host learns the flow's settings before the collector starts, - // so every line, the first included, has the flow's format. + // the flow's handler runs first, then the host's, once per line, always. + // The flow handler's originalHandler does nothing, so no line prints a + // second time unmasked. The host learns the flow's settings before the + // collector starts, so every line, the first included, has its format. if (context.logger) { const own = config.logger || {}; const host = context.logger; if (context.onLogger) context.onLogger({ level: own.level, format: own.format }); const level = context.logLevel ?? own.level ?? host.level; + const printedByHost = () => {}; const handler = own.handler && host.handler - ? (lvl, message, ctx, scope, original) => { + ? (lvl, message, ctx, scope) => { try { - own.handler(lvl, message, ctx, scope, original); + own.handler(lvl, message, ctx, scope, printedByHost); } finally { - host.handler(lvl, message, ctx, scope, original); + host.handler(lvl, message, ctx, scope, printedByHost); } } : host.handler || own.handler; diff --git a/packages/cli/src/commands/setup/index.ts b/packages/cli/src/commands/setup/index.ts index 982beb275..0dd4e7463 100644 --- a/packages/cli/src/commands/setup/index.ts +++ b/packages/cli/src/commands/setup/index.ts @@ -321,7 +321,7 @@ export async function setupCommand(opts: SetupCommandOptions): Promise { // The init result is classified the same way the collector does: // - object → use as resolved config for setup + destroy // - void → init mutated the input config in place; reuse it - // - false → init explicitly aborted; do NOT run setup + // - false → the step is not ready: setup fails (not-ready) let resolvedConfig: unknown = resolvedInputConfig; if (isLifecycleFn(initFn)) { const target = `${component.kind}.${component.id}`; diff --git a/packages/collector/src/report-error.ts b/packages/collector/src/report-error.ts index 5f7935330..0721fa050 100644 --- a/packages/collector/src/report-error.ts +++ b/packages/collector/src/report-error.ts @@ -93,7 +93,7 @@ export function errorMeta(err: unknown): { * uncaughtException it exists to contain. * * - orphan (`reportError(err)`): bump `status.connectionErrors[stepId]` and - * `logger.error`. Does NOT bump `failed` (no event lost at this instant; + * `logger.warn`. Does NOT bump `failed` (no event lost at this instant; * counting it against `failed` would double-count the next push that hits * the broken writer and gets DLQ'd). * - event-bearing (`reportError(err, event)`): the event goes through the diff --git a/packages/core/src/__tests__/node.test.ts b/packages/core/src/__tests__/node.test.ts index 10caeb904..ba6ba39ed 100644 --- a/packages/core/src/__tests__/node.test.ts +++ b/packages/core/src/__tests__/node.test.ts @@ -241,6 +241,33 @@ describe('createCLILogger format json', () => { expect(String(line.context).startsWith('{"rows":"x\\"')).toBe(true); }); + it('caps a message above half the cap with messageTruncated, and the line still parses', () => { + const long = 'é'.repeat(MAX_JSON_LINE_BYTES); + createCLILogger({ format: 'json' }).error(long, { code: 'transport' }); + + const raw = String(errorSpy.mock.calls[0][0]); + expect(Buffer.byteLength(raw)).toBeLessThanOrEqual(MAX_JSON_LINE_BYTES); + const line = parse(errorSpy.mock.calls[0]); + expect(line.messageTruncated).toBe(true); + expect(String(line.message).startsWith('éé')).toBe(true); + expect(Buffer.byteLength(String(line.message))).toBeLessThanOrEqual( + MAX_JSON_LINE_BYTES / 2, + ); + expect(line.context).toEqual({ code: 'transport' }); + }); + + it('opens the DEBUG gate through a function-form verbose', () => { + let open = false; + const logger = createCLILogger({ format: 'json', verbose: () => open }); + logger.debug('hidden'); + open = true; + logger.debug('shown'); + + expect(logSpy.mock.calls.map((call) => parse(call).message)).toEqual([ + 'shown', + ]); + }); + it('sends ERROR and WARN to stderr, INFO and DEBUG to stdout', () => { const logger = createCLILogger({ format: 'json', verbose: true }); logger.error('e'); diff --git a/packages/core/src/node.ts b/packages/core/src/node.ts index dd9131d9d..b1d9094ed 100644 --- a/packages/core/src/node.ts +++ b/packages/core/src/node.ts @@ -80,12 +80,14 @@ export function getDefaultTmpRoot(): string { * One log line as a structured entry: the shape a JSON line prints and the * `onLine` tap receives, masked like the text line. `message` is * `[scope] message` without the context; `scope` and `context` are omitted - * when empty; `context` is a string prefix of its JSON, with - * `contextTruncated`, when the line would pass {@link MAX_JSON_LINE_BYTES}. + * when empty. When the line would pass {@link MAX_JSON_LINE_BYTES}, a + * message above half the cap keeps a prefix (`messageTruncated`), then + * `context` becomes a string prefix of its JSON (`contextTruncated`). */ export interface LogEntry { severity: 'ERROR' | 'WARNING' | 'INFO' | 'DEBUG'; message: string; + messageTruncated?: true; scope?: string; context?: Record | string; contextTruncated?: true; @@ -99,7 +101,11 @@ export interface LogEntry { export const MAX_JSON_LINE_BYTES = 32 * 1024; export interface CLILoggerOptions { - verbose?: boolean; + /** + * Opens the DEBUG gate. A function is read per line, so a host can open + * it for the level a flow sets once it knows it. + */ + verbose?: boolean | (() => boolean); silent?: boolean; json?: boolean; stderr?: boolean; @@ -182,7 +188,8 @@ function isRecord(value: unknown): value is Record { * and to the serialized context. A context that cannot serialize, or whose * masked form no longer parses, becomes `"[unserializable context]"`. A * line above {@link MAX_JSON_LINE_BYTES} keeps a prefix of its masked - * context JSON, so the cut never splits a secret. + * message (when that is above half the cap) and of its masked context JSON, + * so a cut never splits a secret. */ function buildLogEntry( level: Level, @@ -215,6 +222,20 @@ function buildLogEntry( } let line = JSON.stringify(entry); + if (Buffer.byteLength(line) <= MAX_JSON_LINE_BYTES) return { entry, line }; + + // The message keeps at most half the cap, the context the rest. + const half = MAX_JSON_LINE_BYTES / 2; + if (Buffer.byteLength(entry.message) > half) { + const message = entry.message; + let size = half; + do { + entry.message = message.slice(0, size); + size = Math.floor(size / 2); + } while (Buffer.byteLength(entry.message) > half && size > 0); + entry.messageTruncated = true; + line = JSON.stringify(entry); + } if (contextJson && Buffer.byteLength(line) > MAX_JSON_LINE_BYTES) { entry.contextTruncated = true; let size = MAX_JSON_LINE_BYTES - Buffer.byteLength(entry.message) - 256; @@ -274,6 +295,8 @@ export function createCLILoggerConfig( const readFormat = (): Format => (typeof format === 'function' ? format() : format) ?? 'text'; + const readVerbose = (): boolean => + typeof verbose === 'function' ? verbose() : verbose; // The text line's masking order, shared by the JSON entry: known values // first, so the label can never split one; then the temp root label; then // the pattern rules. @@ -286,6 +309,17 @@ export function createCLILoggerConfig( // --verbose. level: Level.DEBUG, handler: (level, message, context, scope) => { + // Machine output (`json`) prints only its result; silent keeps errors; + // DEBUG needs verbose. A line nobody prints and no tap reads costs + // nothing more. + const shown = + !json && + (level === Level.ERROR || !silent) && + (level !== Level.DEBUG || readVerbose()); + const tap = options.onLine; + if (!shown && !tap) return; + const asJson = shown && readFormat() === 'json'; + // Build formatted message const scopePath = scope.length > 0 ? `[${scope.join(':')}] ` : ''; // Serialize the structured context into the line so error details (gRPC @@ -294,56 +328,55 @@ export function createCLILoggerConfig( // values too; the heartbeat path's 256-char cap stays as the wire // backstop. A context that cannot stringify (circular) must never break // logging. - let meta = ''; - if (Object.keys(context).length > 0) { + const textLine = (): string => { + let meta = ''; + if (Object.keys(context).length > 0) { + try { + meta = ` ${JSON.stringify(context)}`; + } catch { + meta = ' [unserializable context]'; + } + } + // Redact secrets ONCE here, before BOTH the onLine ring tap and the + // console.* output. stderr is shipped directly by the log collector, + // so the heartbeat-egress redactor alone would miss it; doing it in + // the handler scrubs every line routed through this logger (collector + // and steps) on both paths. Length is preserved here (no truncation); + // the heartbeat path applies the 256-char wire cap separately as a + // backstop on already-redacted text. + // Order: known values first, so the label can never split one (a + // `sqlite:/tmp/...` URL, or a value that only touches the root at its + // edge); then the temp root label, since a long per-user root (macOS + // `/var/folders/...`) would otherwise read as a token; then the + // pattern rules. + return mask(`${scopePath}${message}${meta}`); + }; + // The structured entry only where it is used: a JSON line, or the tap. + const built = + asJson || tap + ? buildLogEntry(level, message, context, scope, mask, readKnown()) + : undefined; + const fullMessage = tap || !asJson ? textLine() : ''; + + // Tap every line before the gate so no level is dropped from capture. + if (tap && built) { try { - meta = ` ${JSON.stringify(context)}`; + tap(level, fullMessage, built.entry); } catch { - meta = ' [unserializable context]'; + // Swallow: a throwing consumer must never break logging (and we + // must not call logger methods here to avoid infinite recursion). } } - // Redact secrets ONCE here, before BOTH the onLine ring tap and the - // console.* output. stderr is shipped directly by the log collector, so - // the heartbeat-egress redactor alone would miss it; doing it in the - // handler scrubs every line routed through this logger (collector and - // steps) on both paths. Length is preserved here (no truncation); the - // heartbeat path applies the 256-char wire cap separately as a backstop - // on already-redacted text. - // Order: known values first, so the label can never split one (a - // `sqlite:/tmp/...` URL, or a value that only touches the root at its - // edge); then the temp root label, since a long per-user root (macOS - // `/var/folders/...`) would otherwise read as a token; then the - // pattern rules. - const fullMessage = mask(`${scopePath}${message}${meta}`); - const { entry, line } = buildLogEntry( - level, - message, - context, - scope, - mask, - readKnown(), - ); - // Tap every line before any early return so no level is dropped from capture. - try { - options.onLine?.(level, fullMessage, entry); - } catch { - // Swallow: a throwing consumer must never break logging (and we must - // not call logger methods here to avoid infinite recursion). - } - - // Machine output (`json`) prints only its result; silent keeps errors; - // DEBUG needs verbose. - if (json) return; - if (level !== Level.ERROR && silent) return; - if (level === Level.DEBUG && !verbose) return; + if (!shown) return; // JSON lines: one object per line, no colour; ERROR and WARN on stderr // (where platforms that ship only stderr still see them), INFO and // DEBUG on stdout. - if (readFormat() === 'json') { - if (level === Level.ERROR || level === Level.WARN) console.error(line); - else out(line); + if (asJson && built) { + if (level === Level.ERROR || level === Level.WARN) + console.error(built.line); + else out(built.line); return; } diff --git a/packages/runner/src/__tests__/runtime-logging.test.ts b/packages/runner/src/__tests__/runtime-logging.test.ts index 3c418ee56..c4817d743 100644 --- a/packages/runner/src/__tests__/runtime-logging.test.ts +++ b/packages/runner/src/__tests__/runtime-logging.test.ts @@ -49,6 +49,23 @@ describe('createRuntimeLogging', () => { expect(logSpy).toHaveBeenCalledWith('plain'); }); + it("opens DEBUG for the collector's lines at the flow's DEBUG level, not for the runner's own", () => { + const logging = createRuntimeLogging({}); + logging.flowLogger.onLogger({ level: 'DEBUG' }); + logging.collectorConfig.handler?.( + Level.DEBUG, + 'collector debug', + {}, + ['collector'], + () => {}, + ); + logging.logger.debug('runner debug'); + + expect(logSpy.mock.calls.map((call) => String(call[0]))).toEqual([ + ' [collector] collector debug', + ]); + }); + it('forces DEBUG over the flow level only with --verbose', () => { expect(createRuntimeLogging({ verbose: true }).flowLogger.logLevel).toBe( Level.DEBUG, diff --git a/packages/runner/src/runtime-logging.ts b/packages/runner/src/runtime-logging.ts index 330e4b685..ea9107e60 100644 --- a/packages/runner/src/runtime-logging.ts +++ b/packages/runner/src/runtime-logging.ts @@ -5,8 +5,9 @@ * handler setup (masking, the ring tap) and one output format. The format is * the flow's own `logger.format`: the bundle entry reports the flow's logger * settings through `onLogger` before its collector starts, and every line - * from then on, the runner's own included, uses it. No environment variable - * sets it. + * from then on, the runner's own included, uses it. The flow's level opens + * the DEBUG gate for the collector's lines; the runner's own DEBUG lines + * still need `--verbose`. No environment variable sets either. */ import { Level, type Logger } from '@walkeros/core'; @@ -49,18 +50,24 @@ export interface RuntimeLogging { * (`verbose`, `silent`, `json`), `onLine` and `knownSecrets`. */ export function createRuntimeLogging( - options: Omit, + options: Omit & { verbose?: boolean }, colors: CLILoggerColors = {}, ): RuntimeLogging { let format: Logger.Format = 'text'; + let flowDebug = false; const shared: CLILoggerOptions = { ...options, format: () => format }; return { logger: createCLILogger(shared, colors), - collectorConfig: createCLILoggerConfig(shared, colors), + collectorConfig: createCLILoggerConfig( + { ...shared, verbose: () => options.verbose === true || flowDebug }, + colors, + ), flowLogger: { ...(options.verbose ? { logLevel: Level.DEBUG } : {}), onLogger: (settings) => { if (settings.format) format = settings.format; + flowDebug = + settings.level === Level.DEBUG || settings.level === 'DEBUG'; }, }, format: () => format, diff --git a/website/docs/collector/logger.mdx b/website/docs/collector/logger.mdx index e78d0c198..2513b87ca 100644 --- a/website/docs/collector/logger.mdx +++ b/website/docs/collector/logger.mdx @@ -225,9 +225,9 @@ not in this table because it is never a failure. ## Log format -By default lines print as text. On a server flow, set `format` to `json` to -print one JSON object per line, with `severity`, `message`, `time`, `scope` and -`context`. Errors and warnings go to stderr, info and debug to stdout, which +By default lines print as text. On a server flow run by `runneros`, set +`format` to `json` to print one JSON object per line, with `severity`, +`message`, `time`, `scope` and `context`. Errors and warnings go to stderr, info and debug to stdout, which platforms such as Cloud Run read as structured logs. Date: Wed, 21 Oct 2026 08:02:28 +0000 Subject: [PATCH 27/41] contract seat fixes --- .../src/examples/flow-complete.manifest.ts | 4 + .../src/__tests__/delivery-order.test.ts | 57 ++++++++++ .../src/__tests__/init-timeout.test.ts | 101 +++++++++++++++++- .../src/__tests__/mapping-failures.test.ts | 34 +++++- packages/collector/src/destination.ts | 6 +- packages/collector/src/transformer.ts | 64 +++++++---- packages/core/src/mappingValue.ts | 4 +- packages/core/src/schemas/transformer.ts | 6 ++ packages/core/src/types/transformer.ts | 6 ++ 9 files changed, 252 insertions(+), 30 deletions(-) diff --git a/packages/cli/src/examples/flow-complete.manifest.ts b/packages/cli/src/examples/flow-complete.manifest.ts index ad4486992..f349108ab 100644 --- a/packages/cli/src/examples/flow-complete.manifest.ts +++ b/packages/cli/src/examples/flow-complete.manifest.ts @@ -2035,6 +2035,10 @@ export const flowCompleteCoverage = { state: 'state-set', init: RUNTIME('init'), disabled: RUNTIME('disabled'), + timeout: { + excluded: + 'the transformers here initialize at once; the default 10 s applies', + }, mock: RUNTIME('mock'), chainMocks: RUNTIME('chainMocks'), mapping: 'ga4-consent-map', diff --git a/packages/collector/src/__tests__/delivery-order.test.ts b/packages/collector/src/__tests__/delivery-order.test.ts index 70a6a49cd..5b0779aa1 100644 --- a/packages/collector/src/__tests__/delivery-order.test.ts +++ b/packages/collector/src/__tests__/delivery-order.test.ts @@ -182,3 +182,60 @@ describe('delivery order across a destination init', () => { }, ); }); + +describe('held events keep their own consent', () => { + it.each<['skip' | 'queue']>([['skip'], ['queue']])( + 'consentDenied %s: an event held with its consent granted is delivered from the drain after the collector revoked consent', + async (mode) => { + jest.useFakeTimers({ now: 1_000_000 }); + const delivered: WalkerOS.Event[] = []; + let userSyncDown = true; + const destination: Destination.Instance = { + type: 'held', + config: { consent: { marketing: true }, timeout: 1000 }, + on: (type) => { + if (type === 'user' && userSyncDown) + throw new Error('user sync down'); + }, + push: (event: WalkerOS.Event) => { + delivered.push(event); + }, + }; + const { collector, elb } = await startFlow({ + consentDenied: mode, + consent: { marketing: true }, + logger: { handler: () => undefined }, + destinations: { d: { code: destination } }, + }); + await elb('page view'); // initialized and delivered + await elb('walker user', { id: 'u1' }); // on(user) throws: held + + const held = await elb('a one'); + expect(held.queued?.d.reason?.code).toBe('held'); + + await elb('walker consent', { marketing: false }); + + // The retry window passed and the user sync is back: the next push + // releases the hold and drains the held event. + userSyncDown = false; + jest.setSystemTime(1_000_000 + 1001); + await elb('b two'); + + expect(delivered.map((event) => event.name)).toEqual([ + 'page view', + 'a one', + ]); + expect(delivered[1].consent).toEqual({ marketing: true }); + // The new event has no consent of its own: the revocation applies. + const waiting = (collector.destinations.d.queuePush ?? []).map( + (entry) => [entry.event.name, entry.reason], + ); + if (mode === 'skip') { + expect(waiting).toEqual([]); + expect(collector.status.destinations.d.skipped).toBe(1); + } else { + expect(waiting).toEqual([['b two', 'consent']]); + } + }, + ); +}); diff --git a/packages/collector/src/__tests__/init-timeout.test.ts b/packages/collector/src/__tests__/init-timeout.test.ts index 096314d74..8890e6967 100644 --- a/packages/collector/src/__tests__/init-timeout.test.ts +++ b/packages/collector/src/__tests__/init-timeout.test.ts @@ -12,7 +12,8 @@ import { startFlow } from '..'; * An init that never settles is bounded by the step's timeout, like a push: * the pushes waiting for it answer `failed` with code `timeout` instead of * waiting forever, the next push starts a new attempt, and an attempt that - * settles after its timeout installs nothing. + * settles after its timeout installs nothing. A transformer's push is + * bounded the same way. */ type Line = { level: Level; message: string; context: Logger.LogContext }; @@ -176,8 +177,7 @@ describe('destination init timeout', () => { }); describe('transformer init timeout', () => { - // A transformer has no timeout setting: its init is bounded by the - // default destination timeout. + // Without a `config.timeout`, an init is bounded by the default. const timeout = resolveDestinationTimeout(); it('fails the push with code timeout after the default timeout, retries on the next push, and ignores the stale attempt', async () => { @@ -234,4 +234,99 @@ describe('transformer init timeout', () => { }); expect(delivered).toEqual(['page read']); }); + + it.each<[number, number, 'ok' | 'timeout']>([ + [300, 200, 'ok'], + [15_000, 12_000, 'ok'], + [100, 200, 'timeout'], + ])( + 'honours config.timeout %i ms for an init taking %i ms: %s', + async (configTimeout, initMs, expected) => { + const delivered: string[] = []; + const enrich: Transformer.Init = (context) => ({ + type: 'enrich', + config: context.config, + init: () => new Promise((resolve) => setTimeout(resolve, initMs)), + push: (event) => ({ event }), + }); + const sink: Destination.Instance = { + type: 'sink', + config: {}, + push: (event: WalkerOS.Event) => { + delivered.push(event.name); + }, + }; + const { collector, elb } = await startFlow({ + logger: { handler: () => undefined }, + next: 'enrich', + transformers: { + enrich: { code: enrich, config: { timeout: configTimeout } }, + }, + destinations: { sink: { code: sink } }, + }); + + const answer = track(elb('page view')); + await jest.advanceTimersByTimeAsync(Math.min(configTimeout, initMs)); + + if (expected === 'ok') { + expect(answer()?.ok).toBe(true); + expect(collector.transformers.enrich.config.init).toBe(true); + expect(delivered).toEqual(['page view']); + } else { + expect(answer()?.reason).toEqual( + expect.objectContaining({ code: 'timeout', retryable: true }), + ); + expect(collector.transformers.enrich.config.init).toBeFalsy(); + expect(delivered).toEqual([]); + } + }, + ); +}); + +describe('transformer push timeout', () => { + it('a push that never settles fails the push with code timeout; other events still run', async () => { + const { logger, errors } = capture(); + const delivered: string[] = []; + const enrich: Transformer.Init = (context) => ({ + type: 'enrich', + config: context.config, + push: (event) => (event.name === 'page view' ? never() : { event }), + }); + const sink: Destination.Instance = { + type: 'sink', + config: {}, + push: (event: WalkerOS.Event) => { + delivered.push(event.name); + }, + }; + const { elb } = await startFlow({ + logger, + next: 'enrich', + transformers: { enrich: { code: enrich, config: { timeout: 300 } } }, + destinations: { sink: { code: sink } }, + }); + + const hung = track(elb('page view')); + const next = track(elb('page read')); + await turns(); + // The next event runs while the first one hangs. + expect(next()?.ok).toBe(true); + expect(delivered).toEqual(['page read']); + expect(hung()).toBeUndefined(); + + await jest.advanceTimersByTimeAsync(300); + expect(hung()?.ok).toBe(false); + expect(hung()?.reason).toEqual( + expect.objectContaining({ code: 'timeout', retryable: true }), + ); + const lines = errors('Push failed').filter( + (line) => line.context.step === 'transformer.enrich', + ); + expect(lines).toHaveLength(1); + expect(lines[0].context.code).toBe('timeout'); + + // And after it. + expect((await elb('page scroll')).ok).toBe(true); + expect(delivered).toEqual(['page read', 'page scroll']); + }); }); diff --git a/packages/collector/src/__tests__/mapping-failures.test.ts b/packages/collector/src/__tests__/mapping-failures.test.ts index 770369abf..ef8540ca8 100644 --- a/packages/collector/src/__tests__/mapping-failures.test.ts +++ b/packages/collector/src/__tests__/mapping-failures.test.ts @@ -135,7 +135,7 @@ describe('mapping failures (ruling 2)', () => { expect(collector.status.failed).toBe(0); expect(lines.map((line) => line.context)).toEqual([ expect.objectContaining({ - step: 'mapping', + step: 'mapping.order complete', code: 'mapping', event: 'order complete', mappingKey: 'order complete', @@ -144,4 +144,36 @@ describe('mapping failures (ruling 2)', () => { expect(lines[0].message).toContain('mapping value has no operator (total)'); expect(JSON.stringify(lines)).not.toContain('person@example.com'); }); + + it('value problems in two rules give two lines, one per rule', async () => { + const { lines, logger } = errors(); + const operatorLess: Record = { from: 'data.value' }; + const destination: Destination.Instance = { + type: 'probe', + config: {}, + push: jest.fn(), + }; + const { elb } = await startFlow({ + logger, + destinations: { + d: { + code: destination, + config: { + mapping: { + order: { complete: { data: { map: { total: operatorLess } } } }, + page: { view: { data: { map: { title: operatorLess } } } }, + }, + }, + }, + }, + }); + + await elb('order complete', { value: 42 }); + await elb('page view', { title: 'Home' }); + + expect(lines.map((line) => line.context.step)).toEqual([ + 'mapping.order complete', + 'mapping.page view', + ]); + }); }); diff --git a/packages/collector/src/destination.ts b/packages/collector/src/destination.ts index 1f8c2ff3b..6948f7595 100644 --- a/packages/collector/src/destination.ts +++ b/packages/collector/src/destination.ts @@ -749,11 +749,11 @@ async function deliverToDestinations( // A queued entry goes out with the collector's current consent (full // replace: stale consent must not persist); the entry keeps its // identity, the event inside is copied. An entry a push is waiting - // for was admitted at that push with its own event consent and keeps - // it: on a server the collector's consent is usually empty, and + // for, and a held entry, was admitted with its own event consent and + // keeps it: on a server the collector's consent is usually empty, and // replacing it would deny an event whose consent was granted. const refresh = (entry: Destination.Queued) => { - if (!waiting.has(entry)) + if (!waiting.has(entry) && entry.reason !== 'held') entry.event = { ...entry.event, consent: collector.consent }; return entry; }; diff --git a/packages/collector/src/transformer.ts b/packages/collector/src/transformer.ts index b857ff03c..f383e86a7 100644 --- a/packages/collector/src/transformer.ts +++ b/packages/collector/src/transformer.ts @@ -278,6 +278,25 @@ export async function initTransformers( return result; } +/** + * A transformer's `init` or `push`, bounded by its `config.timeout` (default + * 10000 ms), as a destination's. Work that does not settle in time rejects + * with `DestinationTimeoutError` (code `timeout`); its late result is ignored. + */ +function withTransformerTimeout( + transformer: Transformer.Instance, + transformerId: string, + step: 'init' | 'push', + work: WalkerOS.PromiseOrValue, +): Promise { + const timeoutMs = resolveDestinationTimeout(transformer.config.timeout); + return withTimeout( + Promise.resolve(work), + timeoutMs, + `Transformer "${transformerId}" ${step} timed out after ${timeoutMs}ms`, + ); +} + /** * Initializes a transformer if it hasn't been initialized yet. * Called lazily before first push. @@ -316,22 +335,18 @@ export async function transformerInit( transformerLogger.debug('init'); - // A transformer has no timeout setting: its init is bounded by the - // default destination timeout. One that does not settle in time throws - // `DestinationTimeoutError` (code `timeout`); the config below is only - // installed from an attempt that settled in time. - const timeoutMs = resolveDestinationTimeout(); - const configResult = await withTimeout( - Promise.resolve( - useHooks( - transformer.init, - 'TransformerInit', - collector.hooks, - collector.logger, - )(context), - ), - timeoutMs, - `Transformer "${transformerId}" init timed out after ${timeoutMs}ms`, + // The config below is only installed from an attempt that settled in + // time. + const configResult = await withTransformerTimeout( + transformer, + transformerId, + 'init', + useHooks( + transformer.init, + 'TransformerInit', + collector.hooks, + collector.logger, + )(context), ); // Check for initialization failure @@ -411,12 +426,17 @@ export async function transformerPush( emitStep(collector, inState); try { - const result = await useHooks( - transformer.push, - 'TransformerPush', - collector.hooks, - collector.logger, - )(event, context); + const result = await withTransformerTimeout( + transformer, + transformerId, + 'push', + useHooks( + transformer.push, + 'TransformerPush', + collector.hooks, + collector.logger, + )(event, context), + ); const finished = Date.now(); const outState = stepState( diff --git a/packages/core/src/mappingValue.ts b/packages/core/src/mappingValue.ts index 3acbeebd8..c727fd0f2 100644 --- a/packages/core/src/mappingValue.ts +++ b/packages/core/src/mappingValue.ts @@ -54,10 +54,12 @@ function reportValue( err?: unknown, ): void { const name = context.event.name; + // Keyed on the rule, so a second rule's problem gets its own line. + const step = trace?.mappingKey ? `mapping.${trace.mappingKey}` : 'mapping'; reportCause( context.collector, context.logger, - { step: 'mapping', code: 'mapping' }, + { step, code: 'mapping' }, message, { ...(typeof name === 'string' ? { event: name } : {}), diff --git a/packages/core/src/schemas/transformer.ts b/packages/core/src/schemas/transformer.ts index e36b24095..a2c29a14c 100644 --- a/packages/core/src/schemas/transformer.ts +++ b/packages/core/src/schemas/transformer.ts @@ -75,6 +75,12 @@ export const ConfigSchema = z .boolean() .describe('Completely skip this transformer in chains') .optional(), + timeout: z + .number() + .optional() + .describe( + 'Timeout in ms (default 10000) for init and for each push; one that does not settle in this window fails the push with code timeout, and the next push retries the init.', + ), mock: z .unknown() .meta({ diff --git a/packages/core/src/types/transformer.ts b/packages/core/src/types/transformer.ts index 6bcec3623..ec304b372 100644 --- a/packages/core/src/types/transformer.ts +++ b/packages/core/src/types/transformer.ts @@ -148,6 +148,12 @@ export interface Config { state?: import('./state').State | import('./state').State[]; init?: boolean; // Track init state (like Destination) disabled?: boolean; // Completely skip this transformer in chains + /** + * Timeout in ms (default 10000) for `init` and for each `push`: one that + * does not settle within this window fails the push with code `timeout`; + * the next push retries the init. + */ + timeout?: number; /** Return this value instead of calling push(). Global mock for all chains. */ mock?: unknown; /** Path-specific mock values keyed by chain path (e.g., "destination.ga4.before"). Takes precedence over global mock. */ From 26434cedf05c1b26ce83fa85d10306c7566d3688 Mon Sep 17 00:00:00 2001 From: alexanderkirtzel Date: Wed, 21 Oct 2026 08:14:00 +0000 Subject: [PATCH 28/41] server seat fixes --- .../server/destinations/bing/src/config.ts | 2 +- packages/server/destinations/bing/src/push.ts | 2 +- .../server/destinations/criteo/src/config.ts | 2 +- .../server/destinations/criteo/src/push.ts | 15 +++- .../destinations/customerio/src/config.ts | 2 +- .../destinations/customerio/src/index.ts | 10 +-- .../destinations/customerio/src/push.ts | 4 +- .../datamanager/src/__tests__/index.test.ts | 2 +- .../destinations/datamanager/src/config.ts | 2 +- .../destinations/datamanager/src/push.ts | 11 ++- .../server/destinations/klaviyo/src/config.ts | 2 +- .../server/destinations/klaviyo/src/index.ts | 2 +- .../server/destinations/klaviyo/src/push.ts | 2 +- .../destinations/linkedin/src/config.ts | 2 +- .../server/destinations/meta/src/config.ts | 2 +- .../src/__tests__/conformance.test.ts | 45 ++++++++++++ .../destinations/mixpanel/src/config.ts | 2 +- .../server/destinations/mixpanel/src/push.ts | 35 ++++----- .../destinations/mparticle/src/config.ts | 2 +- .../server/destinations/mparticle/src/push.ts | 11 ++- .../server/destinations/piwikpro/src/push.ts | 4 +- .../server/destinations/reddit/src/config.ts | 2 +- .../server/destinations/slack/src/config.ts | 4 +- .../server/destinations/slack/src/index.ts | 2 +- .../server/destinations/slack/src/push.ts | 72 ++++++++++++------- .../destinations/slack/src/types/index.ts | 6 +- .../destinations/snapchat/src/config.ts | 2 +- .../server/destinations/snapchat/src/push.ts | 2 +- .../server/destinations/twitter/src/config.ts | 2 +- .../server/destinations/twitter/src/push.ts | 15 +++- 30 files changed, 187 insertions(+), 81 deletions(-) diff --git a/packages/server/destinations/bing/src/config.ts b/packages/server/destinations/bing/src/config.ts index 7fab93fa0..4932bd491 100644 --- a/packages/server/destinations/bing/src/config.ts +++ b/packages/server/destinations/bing/src/config.ts @@ -2,7 +2,7 @@ import type { Config, Settings, PartialConfig } from './types'; import { StepError } from '@walkeros/core'; export function getConfig(partialConfig: PartialConfig = {}): Config { - const settings = (partialConfig.settings || {}) as Partial; + const settings: Partial = partialConfig.settings || {}; const { accessToken, tagId } = settings; if (!accessToken || !tagId) { diff --git a/packages/server/destinations/bing/src/push.ts b/packages/server/destinations/bing/src/push.ts index 0a9645197..623f899e1 100644 --- a/packages/server/destinations/bing/src/push.ts +++ b/packages/server/destinations/bing/src/push.ts @@ -32,7 +32,7 @@ const send: PushFn = async function ( continueOnValidationError, ip, userAgent, - } = config.settings as Settings; + } = config.settings!; const eventData = isObject(data) ? data : {}; const configData = config.data diff --git a/packages/server/destinations/criteo/src/config.ts b/packages/server/destinations/criteo/src/config.ts index ac2f646c0..da419bb92 100644 --- a/packages/server/destinations/criteo/src/config.ts +++ b/packages/server/destinations/criteo/src/config.ts @@ -4,7 +4,7 @@ import { StepError } from '@walkeros/core'; export const DEFAULT_URL = 'https://widget.criteo.com/m/event?version=s2s_v0'; export function getConfig(partialConfig: PartialConfig = {}): Config { - const settings = (partialConfig.settings || {}) as Partial; + const settings: Partial = partialConfig.settings || {}; const { partnerId, callerId } = settings; if (!partnerId || !callerId) { diff --git a/packages/server/destinations/criteo/src/push.ts b/packages/server/destinations/criteo/src/push.ts index 8d9386094..5fbe7bcae 100644 --- a/packages/server/destinations/criteo/src/push.ts +++ b/packages/server/destinations/criteo/src/push.ts @@ -9,7 +9,12 @@ import type { Settings, } from './types'; import type { Collector, Mapping as WalkerOSMapping } from '@walkeros/core'; -import { createMappingRoot, getMappingValue, isObject } from '@walkeros/core'; +import { + createMappingRoot, + getMappingValue, + isObject, + StepError, +} from '@walkeros/core'; import { getHashServer, sendServer } from '@walkeros/server-core'; import { DEFAULT_URL } from './config'; @@ -110,7 +115,13 @@ const send: PushFn = async function ( user_data, ip, userAgent, - } = config.settings as Settings; + } = config.settings ?? {}; + + // init() validates these; a destination that skipped it is not ready. + if (!partnerId || !callerId) + throw new StepError('Config settings missing, init() not run', { + code: 'not-ready', + }); const eventData = isObject(data) ? data : {}; const userDataCustom = user_data diff --git a/packages/server/destinations/customerio/src/config.ts b/packages/server/destinations/customerio/src/config.ts index 37f0ee5a6..d7bf475a3 100644 --- a/packages/server/destinations/customerio/src/config.ts +++ b/packages/server/destinations/customerio/src/config.ts @@ -2,7 +2,7 @@ import type { Config, Settings, PartialConfig } from './types'; import { StepError } from '@walkeros/core'; export function getConfig(partialConfig: PartialConfig = {}): Config { - const settings = (partialConfig.settings || {}) as Partial; + const settings: Partial = partialConfig.settings || {}; const { siteId, apiKey } = settings; if (!siteId || !apiKey) { diff --git a/packages/server/destinations/customerio/src/index.ts b/packages/server/destinations/customerio/src/index.ts index a7ec7562d..ce7076060 100644 --- a/packages/server/destinations/customerio/src/index.ts +++ b/packages/server/destinations/customerio/src/index.ts @@ -18,7 +18,7 @@ export const destinationCustomerIo: Destination = { init({ config: partialConfig, env }) { const config = getConfig(partialConfig); - const settings = config.settings as Settings; + const settings = config.settings!; // Use env mocks if provided (testing), otherwise create real SDK clients const envTyped = env as @@ -77,10 +77,12 @@ export const destinationCustomerIo: Destination = { }, async destroy({ config }) { - const settings = (config?.settings || {}) as Settings; + const settings = config?.settings; // TrackClient has no flush -- just clear references - settings._trackClient = undefined; - settings._apiClient = undefined; + if (settings) { + settings._trackClient = undefined; + settings._apiClient = undefined; + } }, }; diff --git a/packages/server/destinations/customerio/src/push.ts b/packages/server/destinations/customerio/src/push.ts index 45606604d..5d971df45 100644 --- a/packages/server/destinations/customerio/src/push.ts +++ b/packages/server/destinations/customerio/src/push.ts @@ -40,7 +40,7 @@ export const push: PushFn = async function ( event, { config, rule, data, collector, env }, ) { - const settings = config.settings as Settings; + const settings = config.settings!; const envTyped = env as | { trackClient?: CustomerIoTrackClientMock; @@ -151,7 +151,7 @@ export const push: PushFn = async function ( customerId, deviceId, platform, - isObject(r.data) ? (r.data as Record) : undefined, + isObject(r.data) ? r.data : undefined, ), ); } diff --git a/packages/server/destinations/datamanager/src/__tests__/index.test.ts b/packages/server/destinations/datamanager/src/__tests__/index.test.ts index 8c087615c..276541ddf 100644 --- a/packages/server/destinations/datamanager/src/__tests__/index.test.ts +++ b/packages/server/destinations/datamanager/src/__tests__/index.test.ts @@ -276,7 +276,7 @@ describe('Server Destination Data Manager', () => { test('sends event to Data Manager API', async () => { const event = getEvent('order complete'); (event.data as Record).id = 'ORDER-123'; - (event.data as Record).value = 99.99; + event.data.value = 99.99; (event.data as Record).currency = 'USD'; const config: Config = { diff --git a/packages/server/destinations/datamanager/src/config.ts b/packages/server/destinations/datamanager/src/config.ts index 1f3531d0f..64c64b466 100644 --- a/packages/server/destinations/datamanager/src/config.ts +++ b/packages/server/destinations/datamanager/src/config.ts @@ -7,7 +7,7 @@ import type { import { StepError } from '@walkeros/core'; export function getConfig(partialConfig: PartialConfig = {}): ValidatedConfig { - const settings = (partialConfig.settings || {}) as Partial; + const settings: Partial = partialConfig.settings || {}; const { destinations, eventSource = 'WEB' } = settings; if (!destinations || destinations.length === 0) diff --git a/packages/server/destinations/datamanager/src/push.ts b/packages/server/destinations/datamanager/src/push.ts index 6b2f9c004..2a991e4ef 100644 --- a/packages/server/destinations/datamanager/src/push.ts +++ b/packages/server/destinations/datamanager/src/push.ts @@ -175,7 +175,16 @@ const send: PushFn = async function ( ); } - const accessToken = await getAccessToken(authClient); + // The auth error stays on `cause`; the message names only the step. + let accessToken: string; + try { + accessToken = await getAccessToken(authClient); + } catch (error) { + throw new StepError('Data Manager authentication failed', { + code: 'transport', + cause: error, + }); + } const fetchFn = env?.fetch || fetch; const endpoint = `${url}/events:ingest`; diff --git a/packages/server/destinations/klaviyo/src/config.ts b/packages/server/destinations/klaviyo/src/config.ts index 4893b81bb..06d4e30d1 100644 --- a/packages/server/destinations/klaviyo/src/config.ts +++ b/packages/server/destinations/klaviyo/src/config.ts @@ -2,7 +2,7 @@ import type { Config, Settings, PartialConfig } from './types'; import { StepError } from '@walkeros/core'; export function getConfig(partialConfig: PartialConfig = {}): Config { - const settings = (partialConfig.settings || {}) as Partial; + const settings: Partial = partialConfig.settings || {}; const { apiKey } = settings; if (!apiKey) { diff --git a/packages/server/destinations/klaviyo/src/index.ts b/packages/server/destinations/klaviyo/src/index.ts index 2f4f2118c..c80efe36c 100644 --- a/packages/server/destinations/klaviyo/src/index.ts +++ b/packages/server/destinations/klaviyo/src/index.ts @@ -18,7 +18,7 @@ export const destinationKlaviyo: Destination = { init({ config: partialConfig, env }) { const config = getConfig(partialConfig); - const settings = config.settings as Settings; + const settings = config.settings!; // Use env mocks if provided (testing), otherwise create real SDK instances const envTyped = env as diff --git a/packages/server/destinations/klaviyo/src/push.ts b/packages/server/destinations/klaviyo/src/push.ts index 5bd92c2cf..32269d7c2 100644 --- a/packages/server/destinations/klaviyo/src/push.ts +++ b/packages/server/destinations/klaviyo/src/push.ts @@ -48,7 +48,7 @@ export const push: PushFn = async function ( event, { config, rule, data, collector, env }, ) { - const settings = config.settings as Settings; + const settings = config.settings!; // Resolve SDK instances: prefer env mocks (testing), fall back to real const envTyped = env as diff --git a/packages/server/destinations/linkedin/src/config.ts b/packages/server/destinations/linkedin/src/config.ts index 995d51642..8e481f510 100644 --- a/packages/server/destinations/linkedin/src/config.ts +++ b/packages/server/destinations/linkedin/src/config.ts @@ -2,7 +2,7 @@ import type { Config, Settings, PartialConfig } from './types'; import { StepError } from '@walkeros/core'; export function getConfig(partialConfig: PartialConfig = {}): Config { - const settings = (partialConfig.settings || {}) as Partial; + const settings: Partial = partialConfig.settings || {}; const { accessToken, conversionRuleId } = settings; if (!accessToken || !conversionRuleId) { diff --git a/packages/server/destinations/meta/src/config.ts b/packages/server/destinations/meta/src/config.ts index 7ab7143f4..8b30d8206 100644 --- a/packages/server/destinations/meta/src/config.ts +++ b/packages/server/destinations/meta/src/config.ts @@ -2,7 +2,7 @@ import type { Config, Settings, PartialConfig } from './types'; import { StepError } from '@walkeros/core'; export function getConfig(partialConfig: PartialConfig = {}): Config { - const settings = (partialConfig.settings || {}) as Partial; + const settings: Partial = partialConfig.settings || {}; const { accessToken, pixelId } = settings; if (!accessToken || !pixelId) { diff --git a/packages/server/destinations/mixpanel/src/__tests__/conformance.test.ts b/packages/server/destinations/mixpanel/src/__tests__/conformance.test.ts index 718decb69..804a67eea 100644 --- a/packages/server/destinations/mixpanel/src/__tests__/conformance.test.ts +++ b/packages/server/destinations/mixpanel/src/__tests__/conformance.test.ts @@ -1,5 +1,8 @@ import path from 'path'; +import { startFlow } from '@walkeros/collector'; import { conformance } from '@walkeros/collector/dev'; +import type { Logger } from '@walkeros/core'; +import { clone, createEvent, Level } from '@walkeros/core'; import { destinationMixpanel } from '..'; import { examples, schemas } from '../dev'; @@ -19,3 +22,45 @@ describe('mixpanel conformance', () => { }); }); }); + +describe('mixpanel declared skips', () => { + it('write one DEBUG line naming the event and the mapping key', async () => { + const lines: Array<[Level, string, unknown]> = []; + const handler: Logger.Handler = (level, message, context) => { + lines.push([level, message, context]); + }; + const event = createEvent({ name: 'company update' }); + const { collector } = await startFlow({ + logger: { level: 'DEBUG', handler }, + destinations: { + mixpanel: { + code: { ...destinationMixpanel, env: clone(examples.env.push) }, + config: { + settings: { apiKey: 'token' }, + mapping: { + company: { + update: { + silent: true, + settings: { + groupProfile: { + map: { key: 'data.none', id: 'data.none' }, + }, + }, + }, + }, + }, + }, + }, + }, + }); + await collector.push(event); + + const skipped = lines.filter(([, message]) => message.includes('skipped')); + expect(skipped).toHaveLength(1); + expect(skipped[0][0]).toBe(Level.DEBUG); + expect(skipped[0][1]).toBe( + 'Mixpanel skipped groupProfile: groupProfile resolved to nothing', + ); + expect(skipped[0][2]).toMatchObject({ mappingKey: 'groupProfile' }); + }); +}); diff --git a/packages/server/destinations/mixpanel/src/config.ts b/packages/server/destinations/mixpanel/src/config.ts index 40811e8c2..ff2c7b733 100644 --- a/packages/server/destinations/mixpanel/src/config.ts +++ b/packages/server/destinations/mixpanel/src/config.ts @@ -14,7 +14,7 @@ export function getConfig( env: Env | undefined, logger: Logger.Instance, ): Config { - const settings = (partialConfig.settings || {}) as Partial; + const settings: Partial = partialConfig.settings || {}; const { apiKey } = settings; if (!apiKey) diff --git a/packages/server/destinations/mixpanel/src/push.ts b/packages/server/destinations/mixpanel/src/push.ts index af524e933..3f91c2053 100644 --- a/packages/server/destinations/mixpanel/src/push.ts +++ b/packages/server/destinations/mixpanel/src/push.ts @@ -5,9 +5,7 @@ import { isObject, isString, isThenable, - reportCause, StepError, - stepId, } from '@walkeros/core'; type PeopleObjectOp = @@ -81,7 +79,7 @@ async function applyPeople( props: Record, cb: MixpanelCallback, ) => void - )(distinctId, bag as Record, cb), + )(distinctId, bag, cb), ), ); } @@ -143,7 +141,7 @@ async function applyGroupProfile( props: Record, cb: MixpanelCallback, ) => void - )(key, id, bag as Record, cb), + )(key, id, bag, cb), ), ); } @@ -179,7 +177,7 @@ async function applyGroupProfile( export const push: PushFn = async function ( event, - { config, rule, data, collector, logger, id }, + { config, rule, data, collector, logger }, ) { const settings = config.settings!; const client = settings.client; @@ -216,16 +214,22 @@ export const push: PushFn = async function ( } // 3. People operations - require distinct_id - if (distinctId && mappingSettings.people !== undefined) { - const resolved = await getMappingValue(event, mappingSettings.people, { - collector, - }); - if (isObject(resolved)) { + if (mappingSettings.people !== undefined) { + const resolved = distinctId + ? await getMappingValue(event, mappingSettings.people, { collector }) + : undefined; + if (distinctId && isObject(resolved)) { await applyPeople( client, distinctId, resolved as Record, ); + } else { + // Nothing to address: a deliberate skip, named by its mapping key. + logger.debug('Mixpanel skipped people: identify resolved to nothing', { + event: event.name, + mappingKey: distinctId ? 'people' : 'identify', + }); } } @@ -256,13 +260,10 @@ export const push: PushFn = async function ( ); if (isObject(resolved)) { // A group profile without key and id has nothing to address: the event - // still goes out, and the cause log reports the dropped operation once. - await applyGroupProfile(client, resolved as Record, () => - reportCause( - collector, - logger, - { step: stepId('destination', id), code: 'mapping' }, - 'Mixpanel group profile skipped: key and id must resolve to strings', + // still goes out, and the skip is a deliberate decision (DEBUG). + await applyGroupProfile(client, resolved, () => + logger.debug( + 'Mixpanel skipped groupProfile: groupProfile resolved to nothing', { event: event.name, mappingKey: 'groupProfile' }, ), ); diff --git a/packages/server/destinations/mparticle/src/config.ts b/packages/server/destinations/mparticle/src/config.ts index 4b54b61da..29bbb48f2 100644 --- a/packages/server/destinations/mparticle/src/config.ts +++ b/packages/server/destinations/mparticle/src/config.ts @@ -2,7 +2,7 @@ import type { Config, PartialConfig, Settings } from './types'; import { StepError } from '@walkeros/core'; export function getConfig(partialConfig: PartialConfig = {}): Config { - const settings = (partialConfig.settings || {}) as Partial; + const settings: Partial = partialConfig.settings || {}; const { apiKey, apiSecret } = settings; if (!apiKey || !apiSecret) { diff --git a/packages/server/destinations/mparticle/src/push.ts b/packages/server/destinations/mparticle/src/push.ts index a4b6cbb6e..a7930f2d3 100644 --- a/packages/server/destinations/mparticle/src/push.ts +++ b/packages/server/destinations/mparticle/src/push.ts @@ -3,7 +3,7 @@ import type { WalkerOS, Mapping as WalkerOSMapping, } from '@walkeros/core'; -import { getMappingValue, isObject } from '@walkeros/core'; +import { getMappingValue, isObject, StepError } from '@walkeros/core'; import { sendServer } from '@walkeros/server-core'; import type { CommerceEventData, @@ -32,7 +32,6 @@ const send: PushFn = async function ( event, { config, rule, data, env, logger, collector }, ) { - const settings = config.settings as Settings; const { apiKey, apiSecret, @@ -43,7 +42,13 @@ const send: PushFn = async function ( consent, ip: ipSetting, sourceRequestId: sourceRequestIdSetting, - } = settings; + } = config.settings ?? {}; + + // init() validates these; a destination that skipped it is not ready. + if (!apiKey || !apiSecret) + throw new StepError('Config settings missing, init() not run', { + code: 'not-ready', + }); const ruleSettings: Mapping = (rule?.settings || {}) as Mapping; diff --git a/packages/server/destinations/piwikpro/src/push.ts b/packages/server/destinations/piwikpro/src/push.ts index cffdaa795..e43a2d171 100644 --- a/packages/server/destinations/piwikpro/src/push.ts +++ b/packages/server/destinations/piwikpro/src/push.ts @@ -12,7 +12,7 @@ function getSettings(settings: Partial | undefined): Settings { const { url, appId } = settings || {}; if (!url || !appId) throw new StepError('Config settings missing, init() not run', { - code: 'config', + code: 'not-ready', }); return { ...settings, url: normalizeUrl(url), appId }; } @@ -37,7 +37,7 @@ export const push: PushFn = async function ( if ('invalid' in result) throw invalid(result.invalid); - // A skip is deliberate (no rule for the event, a silent rule): no request. + // A skip is deliberate (a silent rule with nothing else to send): no request. if ('hits' in result) await sendHits(result.hits, settings, env, logger); }; diff --git a/packages/server/destinations/reddit/src/config.ts b/packages/server/destinations/reddit/src/config.ts index 7ab7143f4..8b30d8206 100644 --- a/packages/server/destinations/reddit/src/config.ts +++ b/packages/server/destinations/reddit/src/config.ts @@ -2,7 +2,7 @@ import type { Config, Settings, PartialConfig } from './types'; import { StepError } from '@walkeros/core'; export function getConfig(partialConfig: PartialConfig = {}): Config { - const settings = (partialConfig.settings || {}) as Partial; + const settings: Partial = partialConfig.settings || {}; const { accessToken, pixelId } = settings; if (!accessToken || !pixelId) { diff --git a/packages/server/destinations/slack/src/config.ts b/packages/server/destinations/slack/src/config.ts index 05bf62ac2..122e43687 100644 --- a/packages/server/destinations/slack/src/config.ts +++ b/packages/server/destinations/slack/src/config.ts @@ -6,7 +6,7 @@ export function getConfig( partialConfig: PartialConfig = {}, logger: Logger.Instance, ): Config { - const settings = (partialConfig.settings || {}) as Partial; + const settings: Partial = partialConfig.settings || {}; const { token, webhookUrl, channel } = settings; if (!token && !webhookUrl) @@ -27,6 +27,6 @@ export function getConfig( return { ...partialConfig, - settings: { ...settings } as Settings, + settings, }; } diff --git a/packages/server/destinations/slack/src/index.ts b/packages/server/destinations/slack/src/index.ts index 2b379ca45..f87e8dfa0 100644 --- a/packages/server/destinations/slack/src/index.ts +++ b/packages/server/destinations/slack/src/index.ts @@ -19,7 +19,7 @@ export const destinationSlack: Destination = { init({ config: partialConfig, logger, env }) { const config = getConfig(partialConfig, logger); - const settings = config.settings as Settings; + const settings = config.settings!; // Webhook mode -- nothing to initialize. if (settings.webhookUrl) return config; diff --git a/packages/server/destinations/slack/src/push.ts b/packages/server/destinations/slack/src/push.ts index a124e4027..66c8d8011 100644 --- a/packages/server/destinations/slack/src/push.ts +++ b/packages/server/destinations/slack/src/push.ts @@ -1,8 +1,24 @@ import { StepError } from '@walkeros/core'; import { sendServer } from '@walkeros/server-core'; -import type { Mapping, PushFn, Settings, SlackClientMock } from './types'; +import type { + Mapping, + PushFn, + Settings, + SlackBlock, + SlackClientMock, +} from './types'; import { buildMessage } from './message'; +/** What a message carries: the text, the blocks, or both. */ +type Content = + | { blocks: SlackBlock[]; text?: string } + | { text: string; blocks?: SlackBlock[] }; + +/** The thread fields Slack accepts together. */ +type Thread = + | { thread_ts: string; reply_broadcast: true } + | { thread_ts?: string }; + // A silent rule has no Slack message of its own: nothing to send. export const push: PushFn = async function (event, context) { if (!context.rule?.silent) await send(event, context); @@ -10,29 +26,35 @@ export const push: PushFn = async function (event, context) { const send: PushFn = async function (event, ctx) { const { config, rule, env } = ctx; - const settings = config.settings as Settings; - const ruleSettings = (rule?.settings || {}) as Partial; + const settings = config.settings!; + const ruleSettings: Partial = rule?.settings || {}; const message = buildMessage(event, settings, ruleSettings); // A message without text and blocks has nothing to show in Slack. - if (!message.text && !message.blocks) + const content: Content | undefined = message.blocks + ? { blocks: message.blocks, text: message.text } + : message.text + ? { text: message.text } + : undefined; + if (!content) throw new StepError('Slack message has no text or blocks', { code: 'missing-input', }); // Webhook mode if (settings.webhookUrl) { - await pushWebhook(message, settings, env); + await pushWebhook(content, settings, settings.webhookUrl, env); } else { // Web API mode - await pushWebApi(message, settings, ruleSettings, env); + await pushWebApi(content, settings, ruleSettings, env); } }; async function pushWebhook( - message: { text?: string; blocks?: unknown[] }, + message: Content, settings: Settings, + webhookUrl: string, env: { sendServer?: typeof sendServer } | undefined, ): Promise { const body = { @@ -44,18 +66,16 @@ async function pushWebhook( }; const send = env?.sendServer || sendServer; - await send(settings.webhookUrl as string, JSON.stringify(body)); + await send(webhookUrl, JSON.stringify(body)); } async function pushWebApi( - message: { text?: string; blocks?: unknown[] }, + message: Content, settings: Settings, rule: Partial, env: { slackClient?: SlackClientMock } | undefined, ): Promise { - const client = - (env?.slackClient as SlackClientMock | undefined) || - (settings._client as unknown as SlackClientMock | undefined); + const client = env?.slackClient || settings._client; if (!client) throw new StepError('Slack WebClient not initialized', { @@ -72,11 +92,7 @@ async function pushWebApi( code: 'config', }); const opened = await client.conversations.open({ users: rule.user }); - const dmChannel = ( - opened as unknown as { - channel?: { id?: string }; - } - ).channel?.id; + const dmChannel = opened.channel?.id; if (!dmChannel) throw new StepError('Slack conversations.open returned no channel id', { code: 'transport', @@ -91,20 +107,22 @@ async function pushWebApi( { code: 'config' }, ); - const baseArgs: Record = { + const threadTs = rule.threadTs ?? settings.threadTs; + // A broadcast only means something on a thread reply. + const thread: Thread = threadTs + ? rule.replyBroadcast + ? { thread_ts: threadTs, reply_broadcast: true } + : { thread_ts: threadTs } + : {}; + const baseArgs = { channel, - ...(message.text ? { text: message.text } : {}), - ...(message.blocks ? { blocks: message.blocks } : {}), + ...message, unfurl_links: settings.unfurlLinks ?? false, unfurl_media: settings.unfurlMedia ?? false, mrkdwn: settings.mrkdwn ?? true, + ...thread, }; - if (rule.threadTs ?? settings.threadTs) { - baseArgs.thread_ts = rule.threadTs ?? settings.threadTs; - } - if (rule.replyBroadcast) baseArgs.reply_broadcast = true; - // Ephemeral path if (rule.ephemeral) { if (!rule.user) @@ -114,8 +132,8 @@ async function pushWebApi( await client.chat.postEphemeral({ ...baseArgs, user: rule.user, - } as never); + }); } else { - await client.chat.postMessage(baseArgs as never); + await client.chat.postMessage(baseArgs); } } diff --git a/packages/server/destinations/slack/src/types/index.ts b/packages/server/destinations/slack/src/types/index.ts index 47ccc6cbd..a7b7ed10e 100644 --- a/packages/server/destinations/slack/src/types/index.ts +++ b/packages/server/destinations/slack/src/types/index.ts @@ -4,6 +4,8 @@ import type { } from '@walkeros/core'; import type { DestinationServer, sendServer } from '@walkeros/server-core'; import type { + Block, + KnownBlock, WebClient, ChatPostMessageArguments, ChatPostMessageResponse, @@ -13,8 +15,8 @@ import type { ConversationsOpenResponse, } from '@slack/web-api'; -/** A single Block Kit block (kept loose -- Slack does not export a union type). */ -export type SlackBlock = Record; +/** A single Block Kit block, as the Slack SDK types it. */ +export type SlackBlock = KnownBlock | Block; export interface Settings { // --- Auth (exactly one required) --- diff --git a/packages/server/destinations/snapchat/src/config.ts b/packages/server/destinations/snapchat/src/config.ts index d1aa786bd..2f1e084a5 100644 --- a/packages/server/destinations/snapchat/src/config.ts +++ b/packages/server/destinations/snapchat/src/config.ts @@ -2,7 +2,7 @@ import type { Config, Settings, PartialConfig } from './types'; import { StepError } from '@walkeros/core'; export function getConfig(partialConfig: PartialConfig = {}): Config { - const settings = (partialConfig.settings || {}) as Partial; + const settings: Partial = partialConfig.settings || {}; const { accessToken, pixelId } = settings; if (!accessToken || !pixelId) { diff --git a/packages/server/destinations/snapchat/src/push.ts b/packages/server/destinations/snapchat/src/push.ts index ea3468bfa..3525b54da 100644 --- a/packages/server/destinations/snapchat/src/push.ts +++ b/packages/server/destinations/snapchat/src/push.ts @@ -31,7 +31,7 @@ const send: PushFn = async function ( testMode, ip, userAgent, - } = config.settings as Settings; + } = config.settings!; const eventData = isObject(data) ? data : {}; const configData = config.data diff --git a/packages/server/destinations/twitter/src/config.ts b/packages/server/destinations/twitter/src/config.ts index 417de0881..1fd463856 100644 --- a/packages/server/destinations/twitter/src/config.ts +++ b/packages/server/destinations/twitter/src/config.ts @@ -2,7 +2,7 @@ import type { Config, Settings, PartialConfig } from './types'; import { StepError } from '@walkeros/core'; export function getConfig(partialConfig: PartialConfig = {}): Config { - const settings = (partialConfig.settings || {}) as Partial; + const settings: Partial = partialConfig.settings || {}; const { pixelId, eventId, diff --git a/packages/server/destinations/twitter/src/push.ts b/packages/server/destinations/twitter/src/push.ts index a3ac167b2..931ad2bcc 100644 --- a/packages/server/destinations/twitter/src/push.ts +++ b/packages/server/destinations/twitter/src/push.ts @@ -50,7 +50,20 @@ const send: PushFn = async function ( user_data, ip, userAgent: userAgentSetting, - } = config.settings as Settings; + } = config.settings ?? {}; + + // init() validates these; a destination that skipped it is not ready. + if ( + !pixelId || + !defaultEventId || + !consumerKey || + !consumerSecret || + !accessToken || + !accessTokenSecret + ) + throw new StepError('Config settings missing, init() not run', { + code: 'not-ready', + }); // Resolve user data from settings-level mapping const userDataCustom = user_data From 52db3d59f2c10d62527e296bc1e7288dca1a1737 Mon Sep 17 00:00:00 2001 From: alexanderkirtzel Date: Wed, 21 Oct 2026 08:35:11 +0000 Subject: [PATCH 29/41] web seat fixes --- packages/core/src/__tests__/stepError.test.ts | 65 +++++++- packages/core/src/index.ts | 7 +- packages/core/src/stepError.ts | 20 +++ .../clarity/src/__tests__/index.test.ts | 29 +++- .../destinations/clarity/src/examples/step.ts | 17 +- .../web/destinations/clarity/src/index.ts | 59 ++----- .../clarity/src/schemas/settings.ts | 2 +- packages/web/destinations/d8a/src/index.ts | 33 ++-- .../d8a/src/schemas/primitives.ts | 4 +- .../fullstory/src/examples/step.ts | 27 +--- .../web/destinations/fullstory/src/index.ts | 43 ++--- .../gtag/src/__tests__/index.test.ts | 39 ++++- packages/web/destinations/gtag/src/index.ts | 145 +++++++++-------- .../web/destinations/gtag/src/shared/gtag.ts | 17 -- packages/web/destinations/heap/src/index.ts | 20 +-- .../destinations/hotjar/src/examples/step.ts | 12 +- packages/web/destinations/hotjar/src/index.ts | 50 +++--- .../src/__tests__/stepExamples.test.ts | 50 +++--- .../destinations/linkedin/src/examples/env.ts | 4 +- .../web/destinations/linkedin/src/index.ts | 99 +++++------- packages/web/destinations/matomo/src/index.ts | 15 +- packages/web/destinations/meta/src/index.ts | 26 ++- .../mixpanel/src/__tests__/consent.test.ts | 33 ++++ .../mixpanel/src/examples/step.ts | 18 +-- .../web/destinations/mixpanel/src/index.ts | 95 ++++------- .../mixpanel/src/schemas/settings.ts | 4 +- .../optimizely/src/__tests__/consent.test.ts | 47 ++++++ .../optimizely/src/examples/step.ts | 21 ++- .../web/destinations/optimizely/src/index.ts | 55 ++----- packages/web/destinations/piano/src/index.ts | 35 ++-- .../pinterest/src/__tests__/consent.test.ts | 40 +++++ .../pinterest/src/__tests__/init.test.ts | 17 +- .../src/__tests__/stepExamples.test.ts | 48 ++++-- .../pinterest/src/examples/env.ts | 42 +---- .../pinterest/src/examples/step.ts | 35 ++-- .../web/destinations/pinterest/src/index.ts | 151 +++++++----------- .../web/destinations/piwikpro/src/index.ts | 15 +- .../web/destinations/plausible/src/index.ts | 15 +- .../posthog/src/__tests__/consent.test.ts | 33 ++++ .../destinations/posthog/src/examples/step.ts | 18 +-- .../web/destinations/posthog/src/index.ts | 84 ++++------ .../posthog/src/schemas/settings.ts | 4 +- .../destinations/segment/src/examples/step.ts | 13 +- .../web/destinations/segment/src/index.ts | 113 +++++-------- .../segment/src/schemas/settings.ts | 130 ++++++++------- .../src/__tests__/conformance.test.ts | 1 + .../snowplow/src/__tests__/consent.test.ts | 60 +++++++ .../src/__tests__/stepExamples.test.ts | 4 +- .../web/destinations/snowplow/src/adapter.ts | 13 ++ .../snowplow/src/examples/step.ts | 10 +- .../web/destinations/snowplow/src/index.ts | 86 +++++++--- .../destinations/snowplow/src/types/index.ts | 5 + .../tiktok/src/__tests__/consent.test.ts | 35 ++++ .../tiktok/src/__tests__/stepExamples.test.ts | 2 +- .../destinations/tiktok/src/examples/env.ts | 66 ++++---- .../destinations/tiktok/src/examples/step.ts | 5 +- packages/web/destinations/tiktok/src/index.ts | 131 +++++++-------- .../tiktok/src/schemas/settings.ts | 4 +- packages/web/destinations/tiktok/src/setup.ts | 87 +++++----- .../destinations/tiktok/src/types/index.ts | 20 ++- 60 files changed, 1242 insertions(+), 1136 deletions(-) create mode 100644 packages/web/destinations/mixpanel/src/__tests__/consent.test.ts create mode 100644 packages/web/destinations/optimizely/src/__tests__/consent.test.ts create mode 100644 packages/web/destinations/pinterest/src/__tests__/consent.test.ts create mode 100644 packages/web/destinations/posthog/src/__tests__/consent.test.ts create mode 100644 packages/web/destinations/snowplow/src/__tests__/consent.test.ts create mode 100644 packages/web/destinations/tiktok/src/__tests__/consent.test.ts diff --git a/packages/core/src/__tests__/stepError.test.ts b/packages/core/src/__tests__/stepError.test.ts index d7cc19463..a4095179a 100644 --- a/packages/core/src/__tests__/stepError.test.ts +++ b/packages/core/src/__tests__/stepError.test.ts @@ -1,4 +1,9 @@ -import { networkErrorMessage, StepError, toFailureReason } from '../stepError'; +import { + networkErrorMessage, + StepError, + stepCall, + toFailureReason, +} from '../stepError'; describe('StepError', () => { it('carries code, status, retryable and the response body', () => { @@ -120,3 +125,61 @@ describe('networkErrorMessage', () => { expect(networkErrorMessage(error)).toBe(expected); }); }); + +describe('stepCall', () => { + it('answers what the call returns, sync or async', async () => { + await expect(stepCall(() => 1, 'transport', 'Call failed')).resolves.toBe( + 1, + ); + await expect( + stepCall(() => Promise.resolve('done'), 'transport', 'Call failed'), + ).resolves.toBe('done'); + }); + + it.each<[string, () => unknown]>([ + [ + 'a throw', + () => { + throw new Error('Request failed: jane@example.com'); + }, + ], + [ + 'a rejection', + () => Promise.reject(new Error('Request failed: jane@example.com')), + ], + ])( + 'wraps %s in its code and message, the original as cause only', + async (_, call) => { + const error = await stepCall( + call, + 'transport', + 'Vendor call failed', + ).then( + () => undefined, + (thrown: unknown) => thrown, + ); + expect(error).toBeInstanceOf(StepError); + expect(error).toMatchObject({ + code: 'transport', + message: 'Vendor call failed', + retryable: true, + }); + expect(error instanceof Error && error.cause).toBeInstanceOf(Error); + }, + ); + + it('passes a StepError through unchanged, its own code winning', async () => { + const own = new StepError('Vendor script failed to load', { + code: 'vendor-not-loaded', + }); + await expect( + stepCall( + () => { + throw own; + }, + 'transport', + 'Vendor call failed', + ), + ).rejects.toBe(own); + }); +}); diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index d2ec3c4d3..b4cbc2599 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -36,7 +36,12 @@ export * from './trim'; export * from './tryCatch'; export * from './fatalError'; export * from './invalidEventError'; -export { networkErrorMessage, StepError, toFailureReason } from './stepError'; +export { + networkErrorMessage, + StepError, + stepCall, + toFailureReason, +} from './stepError'; export type { FailureReason, QueueReason, diff --git a/packages/core/src/stepError.ts b/packages/core/src/stepError.ts index 976eaaf6b..d73fdc4ca 100644 --- a/packages/core/src/stepError.ts +++ b/packages/core/src/stepError.ts @@ -81,6 +81,26 @@ export class StepError extends Error { } } +/** + * Runs one vendor or SDK call, sync or async, and waits for it. A + * `StepError` it throws passes through unchanged (the step's own code wins); + * anything else becomes a `StepError` with `code` and `message`, the + * original kept as `cause` and never in the message, which a vendor can fill + * with the values it was sent. + */ +export async function stepCall( + call: () => T | PromiseLike, + code: string, + message: string, +): Promise> { + try { + return await call(); + } catch (error) { + if (error instanceof StepError) throw error; + throw new StepError(message, { code, cause: error }); + } +} + /** A message capped to what a reason or a log line may carry. */ export function capMessage(message: string): string { return message.length > MAX_MESSAGE_LENGTH diff --git a/packages/web/destinations/clarity/src/__tests__/index.test.ts b/packages/web/destinations/clarity/src/__tests__/index.test.ts index 79e423290..ec795cf39 100644 --- a/packages/web/destinations/clarity/src/__tests__/index.test.ts +++ b/packages/web/destinations/clarity/src/__tests__/index.test.ts @@ -43,12 +43,39 @@ describe('clarity destination', () => { expect(calls).toEqual([ ['init', 'gated'], - ['consentV2', { analytics_Storage: 'granted', ad_Storage: 'granted' }], + ['consentV2', { analytics_Storage: 'granted' }], ['event', 'page view'], ['consentV2', { analytics_Storage: 'denied', ad_Storage: 'denied' }], ]); }); + it('never grants ad storage from an analytics-only grant without settings.consent', async () => { + const calls: unknown[][] = []; + const { elb } = await startFlow({ + destinations: { + clarity: { + code: { + ...destinationClarity, + env: { clarity: recordingClarity(calls) }, + }, + config: { + consent: { analytics: true, marketing: true }, + settings: { apiKey: 'analytics-only' }, + }, + }, + }, + }); + + await elb('walker consent', { analytics: true, marketing: false }); + await elb(getEvent('page view')); + await elb('walker consent', { analytics: true }); + + const consentCalls = calls.filter(([name]) => name === 'consentV2'); + expect(consentCalls.length).toBeGreaterThan(0); + for (const [, v2] of consentCalls) + expect(v2).not.toMatchObject({ ad_Storage: 'granted' }); + }); + it('fails pushes with vendor-not-loaded once the Clarity script failed to load', async () => { const { elb } = await startFlow({ destinations: { diff --git a/packages/web/destinations/clarity/src/examples/step.ts b/packages/web/destinations/clarity/src/examples/step.ts index 5abe7512c..d5abb6fdd 100644 --- a/packages/web/destinations/clarity/src/examples/step.ts +++ b/packages/web/destinations/clarity/src/examples/step.ts @@ -335,23 +335,14 @@ export const clarityUnreachable: ClarityStepExample = { /** * The silent tag rule of pageViewSilent on a page view without data: no tag - * resolves and the rule suppresses the event, so nothing is sent. + * resolves and the rule suppresses the event, so nothing is sent. A declared + * skip: an unmet mapping, not a failure. */ export const pageViewSilentWithoutData: ClarityStepExample = { public: false, description: - 'A silent rule whose tag mapping finds no data sends nothing, so the push fails with missing-input instead of counting as delivered.', + 'A silent rule whose tag mapping finds no data sends nothing: an unmet mapping, not a failure, so the push returns normally.', in: { name: 'page view', entity: 'page', action: 'view' }, - mapping: { - silent: true, - settings: { - set: { - map: { - page_id: 'data.id', - }, - }, - }, - }, + mapping: pageViewSilent.mapping, out: [], - outcome: { status: 'failed', code: 'missing-input' }, }; diff --git a/packages/web/destinations/clarity/src/index.ts b/packages/web/destinations/clarity/src/index.ts index 979a198b4..429701d46 100644 --- a/packages/web/destinations/clarity/src/index.ts +++ b/packages/web/destinations/clarity/src/index.ts @@ -5,6 +5,7 @@ import { isObject, isString, StepError, + stepCall, } from '@walkeros/core'; import { getEnv, loadVendor } from '@walkeros/web-core'; import Clarity from '@microsoft/clarity'; @@ -110,22 +111,8 @@ function getClarity( throw new StepError('env.clarity is not a Clarity SDK', { code: 'config' }); } -/** Runs one Clarity call; a throw or a rejection becomes a StepError. */ -async function call( - fn: () => unknown, - code: string, - message: string, -): Promise { - try { - await fn(); - } catch (error) { - if (error instanceof StepError) throw error; - throw new StepError(message, { code, cause: error }); - } -} - const send = (fn: () => unknown) => - call(fn, 'transport', 'Clarity call failed'); + stepCall(fn, 'transport', 'Clarity call failed'); function coerceTagValue(value: unknown): string | string[] | undefined { if (value == null || value === '') return undefined; @@ -180,8 +167,9 @@ function isCategory(value: unknown): value is Category { /** * Clarity's ConsentV2 update for one consent change, or none to send. With * `settings.consent` each mapped key present in the change sets its - * category; without it the destination's required consent decides, as it - * gates the pushes, and both categories follow it. + * category. Without it the destination's required consent decides, as it + * gates the pushes: a grant grants analytics storage only, a revocation + * denies both. Ad storage is granted only through `settings.consent`. */ function consentOf( settings: unknown, @@ -201,9 +189,9 @@ function consentOf( if (!required || !Object.keys(required).some((key) => key in change)) return undefined; - const granted = - getGrantedConsent(required, state) !== false ? 'granted' : 'denied'; - return { analytics_Storage: granted, ad_Storage: granted }; + return getGrantedConsent(required, state) !== false + ? { analytics_Storage: 'granted' } + : { analytics_Storage: 'denied', ad_Storage: 'denied' }; } export const destinationClarity: Destination = { @@ -216,7 +204,7 @@ export const destinationClarity: Destination = { if (!apiKey) throw new StepError('Clarity needs settings.apiKey', { code: 'config' }); - await call( + await stepCall( () => getClarity(env, apiKey, config.timeout).init(apiKey), 'vendor-not-ready', 'Clarity init failed', @@ -227,11 +215,6 @@ export const destinationClarity: Destination = { const settings: Partial = config.settings || {}; const clarity = getClarity(env, settings.apiKey ?? '', config.timeout); const mappingSettings: Mapping = rule?.settings || {}; - let calls = 0; - const sent = (fn: () => unknown) => { - calls++; - return send(fn); - }; // 1. Identify - rule-level override wins over destination-level. // Fires first per Clarity's own guidance so subsequent tag/event @@ -245,7 +228,7 @@ export const destinationClarity: Destination = { const { customId, customSessionId, customPageId, friendlyName } = resolved; if (isString(customId) && customId) { - await sent(() => + await send(() => callIdentify( clarity, customId, @@ -260,7 +243,7 @@ export const destinationClarity: Destination = { // 2. Include tags - pre-flattened by core into context.data for (const [key, value] of tagsOf(data)) - await sent(() => clarity.setTag(key, value)); + await send(() => clarity.setTag(key, value)); // 3. Explicit tags via mapping.settings.set if (mappingSettings.set !== undefined) { @@ -268,7 +251,7 @@ export const destinationClarity: Destination = { collector, }); for (const [key, value] of tagsOf(resolved)) - await sent(() => clarity.setTag(key, value)); + await send(() => clarity.setTag(key, value)); } // 4. Session priority upgrade @@ -276,25 +259,17 @@ export const destinationClarity: Destination = { const reason = await getMappingValue(event, mappingSettings.upgrade, { collector, }); - if (isString(reason) && reason) await sent(() => clarity.upgrade(reason)); + if (isString(reason) && reason) await send(() => clarity.upgrade(reason)); } // 5. Default event forwarding - unless rule.silent is set if (rule?.silent !== true) { const eventName = isString(rule?.name) ? rule.name : event.name; - await sent(() => clarity.event(eventName)); + await send(() => clarity.event(eventName)); } - // A silent rule whose own settings resolved nothing sent nothing. - const ruleSettings = - mappingSettings.identify !== undefined || - mappingSettings.set !== undefined || - mappingSettings.upgrade !== undefined; - if (!calls && ruleSettings) - throw new StepError( - `Clarity: the silent rule of "${event.name}" resolved no identify, tag or upgrade`, - { code: 'missing-input' }, - ); + // A silent rule whose settings resolve nothing sends nothing: a + // declared skip, not a failure. }, async on(type, { config, data, env, collector }) { @@ -308,7 +283,7 @@ export const destinationClarity: Destination = { if (!v2) return; const apiKey = isObject(config.settings) ? config.settings.apiKey : ''; - await call( + await stepCall( () => getClarity( env, diff --git a/packages/web/destinations/clarity/src/schemas/settings.ts b/packages/web/destinations/clarity/src/schemas/settings.ts index 422055718..1a2ba3156 100644 --- a/packages/web/destinations/clarity/src/schemas/settings.ts +++ b/packages/web/destinations/clarity/src/schemas/settings.ts @@ -11,7 +11,7 @@ export const SettingsSchema = z consent: z .record(z.string(), z.enum(['analytics_Storage', 'ad_Storage'])) .describe( - 'Translation table from walkerOS consent keys to Clarity ConsentV2 categories. Example: { "analytics": "analytics_Storage", "marketing": "ad_Storage" }. Required to get meaningful consent behavior, since Clarity expects its own category names.', + 'Translation table from walkerOS consent keys to Clarity ConsentV2 categories. Example: { "analytics": "analytics_Storage", "marketing": "ad_Storage" }. Without it, the consent the destination requires (config.consent) decides: a grant grants analytics_Storage only, a revocation denies both categories; ad_Storage is granted only through this table.', ) .optional(), identify: z diff --git a/packages/web/destinations/d8a/src/index.ts b/packages/web/destinations/d8a/src/index.ts index d4336939c..37ab32a1a 100644 --- a/packages/web/destinations/d8a/src/index.ts +++ b/packages/web/destinations/d8a/src/index.ts @@ -1,5 +1,5 @@ import type { WalkerOS } from '@walkeros/core'; -import { isObject, StepError } from '@walkeros/core'; +import { isObject, StepError, stepCall } from '@walkeros/core'; import { getEnv } from '@walkeros/web-core'; import type { D8aConfigParams, D8aFn } from '@d8a-tech/wt'; import { installD8a } from '@d8a-tech/wt'; @@ -29,19 +29,6 @@ function getD8a( return windowRef[name] as D8aFn; } -/** Runs one d8a call; a throw or a rejection becomes a StepError of `code`. */ -async function call( - fn: () => unknown, - code: string, - message: string, -): Promise { - try { - await fn(); - } catch (error) { - throw new StepError(message, { code, cause: error }); - } -} - /** The installed d8a function, or a StepError when it is not there. */ function requireD8a( windowRef: Window & Record, @@ -79,11 +66,11 @@ export const destinationD8a: Destination = { const { property_id, server_container_url } = settings; const missing = [ - !property_id && 'property_id', - !server_container_url && 'server_container_url', - ].filter(Boolean); + ...(property_id ? [] : ['settings.property_id']), + ...(server_container_url ? [] : ['settings.server_container_url']), + ]; if (missing.length) - throw new StepError(`d8a needs settings.${missing.join(', ')}`, { + throw new StepError(`d8a needs ${missing.join(' and ')}`, { code: 'config', }); @@ -102,12 +89,12 @@ export const destinationD8a: Destination = { window as Window & Record, globalName, ); - await call( + await stepCall( () => d8a('js', new Date()), 'vendor-not-ready', 'd8a init call failed', ); - await call( + await stepCall( () => d8a('config', property_id, buildConfigParams(settings)), 'vendor-not-ready', 'd8a config call failed', @@ -144,7 +131,7 @@ export const destinationD8a: Destination = { globalName, ); - await call( + await stepCall( () => d8a('event', eventName, eventParams), 'transport', 'd8a event call failed', @@ -179,7 +166,7 @@ export const destinationD8a: Destination = { allParams.forEach((param) => { defaultConsent[param] = 'denied'; }); - await call( + await stepCall( () => d8a('consent', 'default', defaultConsent), 'consent-not-applied', 'd8a did not take the consent default', @@ -203,7 +190,7 @@ export const destinationD8a: Destination = { if (Object.keys(d8aConsent).length === 0) return; - await call( + await stepCall( () => d8a('consent', 'update', d8aConsent), 'consent-not-applied', 'd8a did not take the consent update', diff --git a/packages/web/destinations/d8a/src/schemas/primitives.ts b/packages/web/destinations/d8a/src/schemas/primitives.ts index d3536cfa6..6d58a26c6 100644 --- a/packages/web/destinations/d8a/src/schemas/primitives.ts +++ b/packages/web/destinations/d8a/src/schemas/primitives.ts @@ -85,6 +85,8 @@ export const SettingsSchema = z file_downloads_enabled: z.boolean().optional(), file_download_extensions: csvOrArray.optional(), }) - .strict(); + // Pass-through settings are loose: every other d8a config key is + // forwarded to d8a('config', ...) as given. + .loose(); export type Settings = z.infer; diff --git a/packages/web/destinations/fullstory/src/examples/step.ts b/packages/web/destinations/fullstory/src/examples/step.ts index 022d0dd0f..d24aa617f 100644 --- a/packages/web/destinations/fullstory/src/examples/step.ts +++ b/packages/web/destinations/fullstory/src/examples/step.ts @@ -339,39 +339,28 @@ export const fullstoryUnreachable: FullStoryStepExample = { /** * The silent page rule of setPageProperties on a page view without data: no - * property resolves and the rule suppresses the event, so nothing is sent. + * property resolves and the rule suppresses the event, so nothing is sent. A + * declared skip: an unmet mapping, not a failure. */ export const setPagePropertiesWithoutData: FullStoryStepExample = { public: false, description: - 'A silent rule whose property mapping finds no data sends nothing, so the push fails with missing-input instead of counting as delivered.', + 'A silent rule whose property mapping finds no data sends nothing: an unmet mapping, not a failure, so the push returns normally.', in: { name: 'page view', entity: 'page', action: 'view' }, - mapping: { - silent: true, - settings: { - set: { map: { pageName: 'data.title' } }, - setType: 'page', - }, - }, + mapping: setPageProperties.mapping, out: [], - outcome: { status: 'failed', code: 'missing-input' }, }; /** * The silent identify rule of silentWithIdentify on a login without data: no - * uid resolves and the rule suppresses the event, so nothing is sent. + * uid resolves and the rule suppresses the event, so nothing is sent. A + * declared skip. */ export const silentIdentifyWithoutData: FullStoryStepExample = { public: false, description: - 'A silent rule whose identify mapping finds no uid sends nothing, so the push fails with missing-input instead of counting as delivered.', + 'A silent rule whose identify mapping finds no uid sends nothing: an unmet mapping, not a failure, so the push returns normally.', in: { name: 'user login', entity: 'user', action: 'login' }, - mapping: { - silent: true, - settings: { - identify: { map: { uid: 'data.id' } }, - }, - }, + mapping: silentWithIdentify.mapping, out: [], - outcome: { status: 'failed', code: 'missing-input' }, }; diff --git a/packages/web/destinations/fullstory/src/index.ts b/packages/web/destinations/fullstory/src/index.ts index f716ee1b3..8b8779ac9 100644 --- a/packages/web/destinations/fullstory/src/index.ts +++ b/packages/web/destinations/fullstory/src/index.ts @@ -5,6 +5,7 @@ import { isObject, isString, StepError, + stepCall, } from '@walkeros/core'; import { init as fsInit, FullStory } from '@fullstory/browser'; import type { Destination, FullStorySDK, Mapping, Settings } from './types'; @@ -74,22 +75,8 @@ function initOptions(settings: unknown): InitOptions | undefined { return options; } -/** Runs one FullStory call; a throw or a rejection becomes a StepError. */ -async function call( - fn: () => unknown, - code: string, - message: string, -): Promise { - try { - await fn(); - } catch (error) { - if (error instanceof StepError) throw error; - throw new StepError(message, { code, cause: error }); - } -} - const send = (fn: () => unknown) => - call(fn, 'transport', 'FullStory call failed'); + stepCall(fn, 'transport', 'FullStory call failed'); /** The calls a consent change makes, or none. */ function consentCalls( @@ -133,7 +120,7 @@ export const destinationFullStory: Destination = { throw new StepError('FullStory needs settings.orgId', { code: 'config', }); - await call( + await stepCall( () => getFS(env).init(options), 'vendor-not-ready', 'FullStory init failed', @@ -144,11 +131,6 @@ export const destinationFullStory: Destination = { const fs = getFS(env); const settings: Partial = config.settings || {}; const mappingSettings: Mapping = rule?.settings || {}; - let calls = 0; - const sent = (fn: () => unknown) => { - calls++; - return send(fn); - }; // 1. Identify -- rule-level override wins over destination-level. const identifyMapping = mappingSettings.identify ?? settings.identify; @@ -164,7 +146,7 @@ export const destinationFullStory: Destination = { properties?: Record; } = { uid }; if (isObject(properties)) identityOpts.properties = properties; - await sent(() => fs.setIdentity(identityOpts)); + await send(() => fs.setIdentity(identityOpts)); } } } @@ -176,7 +158,7 @@ export const destinationFullStory: Destination = { }); if (isObject(resolved) && Object.keys(resolved).length) { const propertyType = mappingSettings.setType || 'user'; - await sent(() => + await send(() => fs.setProperties({ type: propertyType, properties: resolved }), ); } @@ -186,18 +168,11 @@ export const destinationFullStory: Destination = { if (rule?.silent !== true) { const eventName = isString(rule?.name) ? rule.name : event.name; const properties = isObject(data) ? data : {}; - await sent(() => fs.trackEvent({ name: eventName, properties })); + await send(() => fs.trackEvent({ name: eventName, properties })); } - // A silent rule whose own settings resolved nothing sent nothing. - const ruleSettings = - mappingSettings.identify !== undefined || - mappingSettings.set !== undefined; - if (!calls && ruleSettings) - throw new StepError( - `FullStory: the silent rule of "${event.name}" resolved no identity or properties`, - { code: 'missing-input' }, - ); + // A silent rule whose settings resolve nothing sends nothing: a + // declared skip, not a failure. }, async on(type, { config, data, env, collector }) { @@ -211,7 +186,7 @@ export const destinationFullStory: Destination = { ); for (const fn of calls) - await call( + await stepCall( fn, 'consent-not-applied', 'FullStory did not take the consent update', diff --git a/packages/web/destinations/gtag/src/__tests__/index.test.ts b/packages/web/destinations/gtag/src/__tests__/index.test.ts index 72af4bad5..fa967d78f 100644 --- a/packages/web/destinations/gtag/src/__tests__/index.test.ts +++ b/packages/web/destinations/gtag/src/__tests__/index.test.ts @@ -701,7 +701,7 @@ describe('Unified Gtag Destination', () => { expect(pushGA4Event).toHaveBeenCalled(); }); - it('should call gtag consent update on subsequent consent changes', () => { + it('should call gtag consent update on subsequent consent changes', async () => { // Set up destination with default consent mode const destination = { ...destinationGtag }; destination.config = { @@ -710,7 +710,7 @@ describe('Unified Gtag Destination', () => { destination.env = mockEnvWithGtag; // First consent call (should use 'default' then 'update') - destination.on?.('consent', { + await destination.on?.('consent', { id: contextId, collector: mockCollector, config: destination.config, @@ -737,7 +737,7 @@ describe('Unified Gtag Destination', () => { mockGtag.mockClear(); // Second consent call (should only use 'update') - destination.on?.('consent', { + await destination.on?.('consent', { id: contextId, collector: mockCollector, config: destination.config, @@ -1000,6 +1000,39 @@ describe('Unified Gtag Destination', () => { }); }); + it('sends the denied default again after an init whose default call failed', async () => { + let fail = true; + gtag.mockImplementation((...args: unknown[]) => { + if (fail && args[0] === 'consent' && args[1] === 'default') { + fail = false; + throw new Error('gtag wrapper failed'); + } + }); + const config = { + consent: { marketing: true }, + settings: { ga4: { measurementId: 'G-X' } }, + }; + const context = { + id: contextId, + config, + env: orderEnv, + collector: mockCollector, + logger: createThrowingLogger(), + }; + + await expect(destinationGtag.init!(context)).rejects.toMatchObject({ + code: 'vendor-not-ready', + }); + gtag.mockClear(); + await destinationGtag.init!(context); + + expect(gtag.mock.calls[0]).toEqual([ + 'consent', + 'default', + expect.objectContaining({ ad_storage: 'denied' }), + ]); + }); + it('does NOT emit a default when there is no consent signal', async () => { const config = { settings: { ga4: { measurementId: 'G-X' } } }; diff --git a/packages/web/destinations/gtag/src/index.ts b/packages/web/destinations/gtag/src/index.ts index 4f3ef4609..e609fc2bb 100644 --- a/packages/web/destinations/gtag/src/index.ts +++ b/packages/web/destinations/gtag/src/index.ts @@ -10,9 +10,15 @@ import { hasConsentSignal, resolveConsentMapping, } from './shared/consent'; -import { checkGtag, initializeGtag, settle } from './shared/gtag'; +import { checkGtag, initializeGtag } from './shared/gtag'; import { hasTool, TOOL_PATHS } from './shared/tools'; -import { isObject, reportCause, stepId, StepError } from '@walkeros/core'; +import { + isObject, + reportCause, + stepCall, + stepId, + StepError, +} from '@walkeros/core'; import { getEnv } from '@walkeros/web-core'; // Types @@ -42,7 +48,7 @@ function setupConsentDefault(config: Config, env: Env | undefined): unknown[] { const gtag = initializeGtag(window); // ensure stub exists before any config if (!gtag) return []; - defaultConsentSet = true; + // defaultConsentSet is set by the caller once this call settled. return [gtag('consent', 'default', defaultConsent)]; } @@ -83,26 +89,33 @@ export const destinationGtag: Destination = { // installs a stub, so a later `js` is only sent if walkerOS bootstrapped. const gtagExternal = !!getEnv(env).window.gtag; - await settle( - () => [ + let defaultSent = false; + await stepCall( + () => { // Emit the denied consent default before any tool's config call. - ...setupConsentDefault(config, env), - // Initialize GA4 if configured - ...(ga4?.measurementId - ? initGA4(ga4, loadScript, env, logger, gtagExternal, timeout) - : []), - // Initialize Google Ads if configured - ...(ads?.conversionId - ? initAds(ads, loadScript, env, logger, gtagExternal, timeout) - : []), - // Initialize GTM if configured - ...(gtm?.containerId - ? initGTM(gtm, loadScript, env, logger, timeout) - : []), - ], + const defaultCall = setupConsentDefault(config, env); + defaultSent = defaultCall.length > 0; + return Promise.all([ + ...defaultCall, + // Initialize GA4 if configured + ...(ga4?.measurementId + ? initGA4(ga4, loadScript, env, logger, gtagExternal, timeout) + : []), + // Initialize Google Ads if configured + ...(ads?.conversionId + ? initAds(ads, loadScript, env, logger, gtagExternal, timeout) + : []), + // Initialize GTM if configured + ...(gtm?.containerId + ? initGTM(gtm, loadScript, env, logger, timeout) + : []), + ]); + }, 'vendor-not-ready', 'Google tag did not take its setup calls', ); + // Only a default gtag took counts: a failed init sends it again. + if (defaultSent) defaultConsentSet = true; return config; }, @@ -149,43 +162,44 @@ export const destinationGtag: Destination = { adsUserData = await resolveUserData(event, ads, collector); } - await settle( - () => [ - // Push to GA4 if configured - ...(send && ga4?.measurementId - ? [ - pushGA4Event(event, ga4, ga4Data, env, logger, (message, key) => - // One line per cause through the collector's cause log; the - // context names the mapping key, never its value. - reportCause( - collector, - logger, - { step: stepId('destination', id), code: 'mapping' }, - message, - { event: event.name, mappingKey: key }, + await stepCall( + () => + Promise.all([ + // Push to GA4 if configured + ...(send && ga4?.measurementId + ? [ + pushGA4Event(event, ga4, ga4Data, env, logger, (message, key) => + // One line per cause through the collector's cause log; the + // context names the mapping key, never its value. + reportCause( + collector, + logger, + { step: stepId('destination', id), code: 'mapping' }, + message, + { event: event.name, mappingKey: key }, + ), ), - ), - ] - : []), - // @TODO: Fix condition - should check for rule.settings?.ads?.label || rule.name - // Currently requires rule.name even when label is provided via settings.ads.label - ...(send && ads?.conversionId && rule.name - ? pushAdsEvent( - event, - ads, - eventMapping.ads, - adsData, - rule.name, - env, - logger, - adsUserData, - ) - : []), - // Push to GTM if configured - ...(send && gtm?.containerId - ? [pushGTMEvent(event, gtm, eventMapping.gtm, gtmData, env, logger)] - : []), - ], + ] + : []), + // @TODO: Fix condition - should check for rule.settings?.ads?.label || rule.name + // Currently requires rule.name even when label is provided via settings.ads.label + ...(send && ads?.conversionId && rule.name + ? pushAdsEvent( + event, + ads, + eventMapping.ads, + adsData, + rule.name, + env, + logger, + adsUserData, + ) + : []), + // Push to GTM if configured + ...(send && gtm?.containerId + ? [pushGTMEvent(event, gtm, eventMapping.gtm, gtmData, env, logger)] + : []), + ]), 'transport', 'Google tag call failed', ); @@ -216,7 +230,6 @@ export const destinationGtag: Destination = { const defaultConsent = defaultConsentSet ? undefined : buildDefaultConsent(settings); - defaultConsentSet = true; // Build gtag consent object for update const gtagConsent: Record = {}; @@ -234,17 +247,21 @@ export const destinationGtag: Destination = { }); }); - await settle( - () => [ - ...(defaultConsent ? [gtag('consent', 'default', defaultConsent)] : []), - // Always use update after the initial default, when a parameter changes - ...(Object.keys(gtagConsent).length - ? [gtag('consent', 'update', gtagConsent)] - : []), - ], + await stepCall( + () => + Promise.all([ + ...(defaultConsent + ? [gtag('consent', 'default', defaultConsent)] + : []), + // Always use update after the initial default, when a parameter changes + ...(Object.keys(gtagConsent).length + ? [gtag('consent', 'update', gtagConsent)] + : []), + ]), 'consent-not-applied', 'Google tag did not take the consent update', ); + defaultConsentSet = true; }, }; diff --git a/packages/web/destinations/gtag/src/shared/gtag.ts b/packages/web/destinations/gtag/src/shared/gtag.ts index 6a25fb891..7e249e3e7 100644 --- a/packages/web/destinations/gtag/src/shared/gtag.ts +++ b/packages/web/destinations/gtag/src/shared/gtag.ts @@ -41,23 +41,6 @@ export function checkGtag(id: string, src?: string, timeout?: number): void { } } -/** - * Waits for what the gtag or dataLayer calls of one step returned (the stub - * returns nothing); a throw or a rejection becomes a StepError of `code`. - */ -export async function settle( - calls: () => unknown[], - code: string, - message: string, -): Promise { - try { - await Promise.all(calls()); - } catch (error) { - if (error instanceof StepError) throw error; - throw new StepError(message, { code, cause: error }); - } -} - export function initializeGtag( window: WindowWithDataLayer, ): Gtag.Gtag | undefined { diff --git a/packages/web/destinations/heap/src/index.ts b/packages/web/destinations/heap/src/index.ts index 852d2dc71..1d73cc651 100644 --- a/packages/web/destinations/heap/src/index.ts +++ b/packages/web/destinations/heap/src/index.ts @@ -4,6 +4,7 @@ import { isObject, isString, StepError, + stepCall, } from '@walkeros/core'; import { getEnv, loadVendor } from '@walkeros/web-core'; import type { @@ -86,21 +87,8 @@ function toHeapProperties(value: unknown): Record { return out; } -/** Runs one Heap call; a throw or a rejection becomes a StepError. */ -async function call( - fn: () => unknown, - code: string, - message: string, -): Promise { - try { - await fn(); - } catch (error) { - if (error instanceof StepError) throw error; - throw new StepError(message, { code, cause: error }); - } -} - -const send = (fn: () => unknown) => call(fn, 'transport', 'Heap call failed'); +const send = (fn: () => unknown) => + stepCall(fn, 'transport', 'Heap call failed'); export const destinationHeap: Destination = { type: 'heap', @@ -219,7 +207,7 @@ export const destinationHeap: Destination = { const toggle = granted ? heap.startTracking : heap.stopTracking; if (!toggle) return; - await call( + await stepCall( () => toggle.call(heap), 'consent-not-applied', 'Heap did not take the consent update', diff --git a/packages/web/destinations/hotjar/src/examples/step.ts b/packages/web/destinations/hotjar/src/examples/step.ts index 5485af0d7..55a2063de 100644 --- a/packages/web/destinations/hotjar/src/examples/step.ts +++ b/packages/web/destinations/hotjar/src/examples/step.ts @@ -158,19 +158,13 @@ export const pageViewSilent: HotjarStepExample = { /** * The silent stateChange rule of pageViewStateChange on a page view without * data: no path resolves and the rule suppresses the event, so nothing is - * sent. + * sent. A declared skip: an unmet mapping, not a failure. */ export const pageViewStateChangeWithoutData: HotjarStepExample = { public: false, description: - 'A silent rule whose stateChange mapping finds no path sends nothing, so the push fails with missing-input instead of counting as delivered.', + 'A silent rule whose stateChange mapping finds no path sends nothing: an unmet mapping, not a failure, so the push returns normally.', in: { name: 'page view', entity: 'page', action: 'view' }, - mapping: { - silent: true, - settings: { - stateChange: 'data.id', - }, - }, + mapping: pageViewStateChange.mapping, out: [], - outcome: { status: 'failed', code: 'missing-input' }, }; diff --git a/packages/web/destinations/hotjar/src/index.ts b/packages/web/destinations/hotjar/src/index.ts index a0ef410db..62c9bab88 100644 --- a/packages/web/destinations/hotjar/src/index.ts +++ b/packages/web/destinations/hotjar/src/index.ts @@ -1,4 +1,10 @@ -import { getMappingValue, isObject, isString, StepError } from '@walkeros/core'; +import { + getMappingValue, + isObject, + isString, + StepError, + stepCall, +} from '@walkeros/core'; import Hotjar from '@hotjar/browser'; import type { HotjarSDK, Destination, Env, Mapping, Settings } from './types'; @@ -15,21 +21,16 @@ function getHotjar(env: Env | undefined): HotjarSDK { /** * Runs one Hotjar call. The SDK reports failure by returning `false` (and - * logging to the console), so a `false` or a throw becomes a StepError. + * logging to the console), so a `false` fails like a throw. */ -async function call( - fn: () => unknown, - code: string, - message: string, -): Promise { - let result: unknown; - try { - result = await fn(); - } catch (error) { - throw new StepError(message, { code, cause: error }); - } - if (result === false) throw new StepError(message, { code }); -} +const call = (fn: () => unknown, code: string, message: string) => + stepCall( + async () => { + if ((await fn()) === false) throw new StepError(message, { code }); + }, + code, + message, + ); export const destinationHotjar: Destination = { type: 'hotjar', @@ -66,9 +67,6 @@ export const destinationHotjar: Destination = { const settings = (config.settings || {}) as Partial; const mappingSettings = (rule?.settings || {}) as Mapping; - // Whether any Hotjar call went out for this event - let sent = false; - // 1. Identify -- rule-level override wins over destination-level. // Fires first per Hotjar's guidance so subsequent event calls // are associated with the right user. @@ -96,7 +94,6 @@ export const destinationHotjar: Destination = { 'transport', 'Hotjar identify failed', ); - sent = true; } } } @@ -114,7 +111,6 @@ export const destinationHotjar: Destination = { 'transport', 'Hotjar stateChange failed', ); - sent = true; } } @@ -126,20 +122,10 @@ export const destinationHotjar: Destination = { 'transport', 'Hotjar event failed', ); - sent = true; } - // A silent rule that asked for identify or stateChange, which the event - // could not resolve, sent nothing at all - if ( - !sent && - (identifyMapping !== undefined || - mappingSettings.stateChange !== undefined) - ) - throw new StepError( - 'Hotjar identify or stateChange resolved to nothing for this event', - { code: 'missing-input' }, - ); + // A silent rule whose identify or stateChange resolves nothing sends + // nothing: a declared skip, not a failure. }, }; diff --git a/packages/web/destinations/linkedin/src/__tests__/stepExamples.test.ts b/packages/web/destinations/linkedin/src/__tests__/stepExamples.test.ts index 2fcaf5c79..68f22abd1 100644 --- a/packages/web/destinations/linkedin/src/__tests__/stepExamples.test.ts +++ b/packages/web/destinations/linkedin/src/__tests__/stepExamples.test.ts @@ -13,12 +13,32 @@ import type { WalkerOS, Mapping as WalkerOSMapping } from '@walkeros/core'; import { startFlow } from '@walkeros/collector'; -import { clone } from '@walkeros/core'; +import { clone, isObject, isString } from '@walkeros/core'; import { examples } from '../dev'; import type { Env, Lintrk, Settings } from '../types'; type CallRecord = [string, ...unknown[]]; +function isConsent(value: unknown): value is WalkerOS.Consent { + return ( + isObject(value) && + Object.values(value).every((granted) => typeof granted === 'boolean') + ); +} + +function isEvent(value: unknown): value is WalkerOS.Event { + return ( + isObject(value) && + isString(value.name) && + isString(value.entity) && + isString(value.action) + ); +} + +function isRule(value: unknown): value is WalkerOSMapping.Rule { + return isObject(value); +} + /** * Install a spy on env.window.lintrk. The spy records every call prefixed * with the dotted path `lintrk` so assertions can compare directly against @@ -40,16 +60,8 @@ function spyEnv(env: Env): { } describe('linkedin destination — step examples', () => { - it.each(Object.entries(examples.step))('%s', async (_name, rawExample) => { - const example = rawExample as { - in?: unknown; - mapping?: unknown; - out?: ReadonlyArray; - command?: 'consent' | 'user' | 'config' | 'run'; - settings?: Partial; - }; - - const env = clone(examples.env.push) as Env; + it.each(Object.entries(examples.step))('%s', async (name, example) => { + const env = clone(examples.env.push); const { env: spiedEnv, collected } = spyEnv(env); const dest = jest.requireActual('../').default; @@ -61,15 +73,18 @@ describe('linkedin destination — step examples', () => { }; if (example.command === 'consent') { - // Not currently used by LinkedIn fixtures. + if (!isConsent(example.in)) + throw new Error(`step example "${name}" has no consent input`); await elb('walker destination', { code: { ...dest, env: spiedEnv }, config: { settings: baseSettings, consent: { marketing: true } }, }); - await elb('walker consent', example.in as WalkerOS.Consent); + await elb('walker consent', example.in); } else { - const event = example.in as WalkerOS.Event; - const mapping = example.mapping as WalkerOSMapping.Rule | undefined; + if (!isEvent(example.in)) + throw new Error(`step example "${name}" has no event input`); + const event = example.in; + const mapping = isRule(example.mapping) ? example.mapping : undefined; const mappingConfig = mapping ? { [event.entity]: { [event.action]: mapping } } : undefined; @@ -84,9 +99,6 @@ describe('linkedin destination — step examples', () => { await elb(event); } - const expected = (example.out ?? []) as ReadonlyArray; - const actual = collected(); - - expect(actual).toEqual(expected); + expect(collected()).toEqual([...(example.out ?? [])]); }); }); diff --git a/packages/web/destinations/linkedin/src/examples/env.ts b/packages/web/destinations/linkedin/src/examples/env.ts index 0cb77bc56..ad083fac1 100644 --- a/packages/web/destinations/linkedin/src/examples/env.ts +++ b/packages/web/destinations/linkedin/src/examples/env.ts @@ -1,4 +1,4 @@ -import type { Env, Lintrk } from '../types'; +import type { Env } from '../types'; /** * Example environment configurations for LinkedIn Insight Tag destination. @@ -32,7 +32,7 @@ export const push: Env = { window: { _linkedin_partner_id: '123456', _linkedin_data_partner_ids: ['123456'], - lintrk: Object.assign(noop, { q: [] as unknown[] }) as unknown as Lintrk, + lintrk: Object.assign(noop, { q: [] }), }, }; diff --git a/packages/web/destinations/linkedin/src/index.ts b/packages/web/destinations/linkedin/src/index.ts index badc0843b..27164d609 100644 --- a/packages/web/destinations/linkedin/src/index.ts +++ b/packages/web/destinations/linkedin/src/index.ts @@ -1,19 +1,14 @@ -import type { WalkerOS } from '@walkeros/core'; import { getMappingValue, isNumber, isObject, isString, StepError, + stepCall, } from '@walkeros/core'; +import type { DestinationWeb } from '@walkeros/web-core'; import { getEnv, loadVendor } from '@walkeros/web-core'; -import type { - Destination, - Env, - Lintrk, - LintrkTrackData, - Mapping, -} from './types'; +import type { Destination, Lintrk, LintrkTrackData, Mapping } from './types'; // Types export export * as DestinationLinkedIn from './types'; @@ -39,25 +34,26 @@ const LINKEDIN_SCRIPT_SRC = * Idempotent - if `lintrk` is already installed (e.g. another tag installed * it), leave the existing function alone. */ -function installLintrkQueue(env: Env | undefined): void { +function installLintrkQueue(env?: DestinationWeb.Env): void { const { window } = getEnv(env); - const w = window as Env['window']; - if (w.lintrk) return; - const lintrk = function (this: unknown, ...args: unknown[]) { - (lintrk.q = lintrk.q || []).push(args); - } as unknown as Lintrk; - lintrk.q = []; - w.lintrk = lintrk; + if (window.lintrk) return; + const lintrk: Lintrk = Object.assign( + (...args: unknown[]) => { + (lintrk.q ||= []).push(args); + }, + { q: [] }, + ); + window.lintrk = lintrk; } -function setPartnerGlobals(env: Env | undefined, partnerId: string): void { +function setPartnerGlobals( + env: DestinationWeb.Env | undefined, + partnerId: string, +): void { const { window } = getEnv(env); - const w = window as Env['window']; - w._linkedin_partner_id = partnerId; - w._linkedin_data_partner_ids = w._linkedin_data_partner_ids || []; - if (!w._linkedin_data_partner_ids.includes(partnerId)) { - w._linkedin_data_partner_ids.push(partnerId); - } + window._linkedin_partner_id = partnerId; + const ids = (window._linkedin_data_partner_ids ||= []); + if (!ids.includes(partnerId)) ids.push(partnerId); } /** @@ -65,27 +61,17 @@ function setPartnerGlobals(env: Env | undefined, partnerId: string): void { * replays its own queue, so it is called synchronously and only a failed * script is reported. */ -function vendor(config: { loadScript?: boolean; timeout?: number }, env?: Env) { +function vendor( + config: { loadScript?: boolean; timeout?: number }, + env?: DestinationWeb.Env, +) { return loadVendor( config.loadScript ? LINKEDIN_SCRIPT_SRC : undefined, - () => (getEnv(env).window as Env['window']).lintrk, + () => getEnv(env).window.lintrk, { timeout: config.timeout }, ); } -/** Runs one lintrk call; a throw or a rejection becomes a StepError of `code`. */ -async function call( - fn: () => unknown, - code: string, - message: string, -): Promise { - try { - await fn(); - } catch (error) { - throw new StepError(message, { code, cause: error }); - } -} - /** * Resolve the short-key walkerOS mapping shape into a LintrkTrackData object. * @@ -102,12 +88,7 @@ async function call( function buildLintrkData(resolved: unknown): LintrkTrackData | undefined { if (!isObject(resolved)) return undefined; - const { id, value, currency, eventId } = resolved as { - id?: unknown; - value?: unknown; - currency?: unknown; - eventId?: unknown; - }; + const { id, value, currency, eventId } = resolved; // id must be a truthy number (Campaign Manager conversion_id). // Accept numeric strings by coercion - users may supply { value: "12345" }. @@ -151,36 +132,34 @@ export const destinationLinkedIn: Destination = { if (!settings?.apiKey) throw new StepError('LinkedIn needs settings.apiKey', { code: 'config' }); - setPartnerGlobals(env as Env | undefined, settings.apiKey); - installLintrkQueue(env as Env | undefined); + setPartnerGlobals(env, settings.apiKey); + installLintrkQueue(env); // Script injection is opt-in via config.loadScript (same default as // plausible / meta). Users who already embed the Insight Tag snippet in // their HTML should leave loadScript false - the destination only // installs the queue shim and mutates _linkedin_partner_id. - vendor(config, env as Env | undefined); + vendor(config, env); return config; }, async push(event, { config, rule, env, collector }) { - vendor(config, env as Env | undefined).check(); + vendor(config, env).check(); - const { window } = getEnv(env); - const w = window as Env['window']; - const lintrk = w.lintrk; + const { lintrk } = getEnv(env).window; if (!lintrk) throw new StepError('LinkedIn Insight Tag not ready: lintrk is missing', { code: 'vendor-not-ready', }); - const mappingSettings = (rule?.settings || {}) as Mapping; + const mappingSettings: Mapping = rule?.settings || {}; // Opt-in: a rule without a conversion makes no call. Honor rule-level // silent (core flag at packages/core/src/types/mapping.ts) the same way. if (rule?.silent !== true && mappingSettings.conversion !== undefined) { const resolved = await getMappingValue( - event as WalkerOS.Event, + event, mappingSettings.conversion, { collector }, ); @@ -192,7 +171,7 @@ export const destinationLinkedIn: Destination = { { code: 'missing-input' }, ); - await call( + await stepCall( () => lintrk('track', data), 'transport', 'LinkedIn lintrk call failed', @@ -200,20 +179,18 @@ export const destinationLinkedIn: Destination = { } }, - on(type, context) { - if (type !== 'consent' || !context.data) return; + on(type, { config, data, env }) { + if (type !== 'consent' || !isObject(data)) return; // LinkedIn has no vendor consent mode. The collector's config.consent gate // is the sole mechanism for blocking events. This handler exists for a // narrow case: config.loadScript was true BUT marketing consent was // initially denied, so init() did not run. When consent later grants // marketing, the script is injected; loadVendor injects it once per page. - const consent = context.data as WalkerOS.Consent; - const config = context.config; - if (!config?.loadScript) return; - if (consent.marketing !== true) return; + if (!config.loadScript) return; + if (data.marketing !== true) return; - vendor(config, context.env as Env | undefined); + vendor(config, env); }, }; diff --git a/packages/web/destinations/matomo/src/index.ts b/packages/web/destinations/matomo/src/index.ts index 07b537777..9d8214977 100644 --- a/packages/web/destinations/matomo/src/index.ts +++ b/packages/web/destinations/matomo/src/index.ts @@ -7,6 +7,7 @@ import { reportCause, stepId, StepError, + stepCall, } from '@walkeros/core'; import { getEnv, loadVendor } from '@walkeros/web-core'; import { resolveDimensionMap, sequence } from './dimensions'; @@ -18,17 +19,17 @@ export * as DestinationMatomo from './types'; * Pushes commands onto the queue in one synchronous block, so another * event's commands never interleave, then waits for what each push returns. */ -async function pushAll( +function pushAll( queue: unknown[], commands: unknown[][], code: string, message: string, -): Promise { - try { - await Promise.all(commands.map((command) => queue.push(command))); - } catch (error) { - throw new StepError(message, { code, cause: error }); - } +): Promise { + return stepCall( + () => Promise.all(commands.map((command) => queue.push(command))), + code, + message, + ); } /** diff --git a/packages/web/destinations/meta/src/index.ts b/packages/web/destinations/meta/src/index.ts index b45872d20..e7e56185d 100644 --- a/packages/web/destinations/meta/src/index.ts +++ b/packages/web/destinations/meta/src/index.ts @@ -1,6 +1,11 @@ import type { Config, Destination, Env } from './types'; import { SCRIPT_SRC, setup } from './setup'; -import { getGrantedConsent, isObject, StepError } from '@walkeros/core'; +import { + getGrantedConsent, + isObject, + StepError, + stepCall, +} from '@walkeros/core'; import { getEnv, loadVendor } from '@walkeros/web-core'; // Types @@ -28,19 +33,6 @@ function pixel(env?: Env): facebook.Pixel.Event { return fbq; } -/** Runs one fbq call; a throw or a rejection becomes a StepError of `code`. */ -async function call( - fn: () => unknown, - code: string, - message: string, -): Promise { - try { - await fn(); - } catch (error) { - throw new StepError(message, { code, cause: error }); - } -} - export const destinationMeta: Destination = { type: 'meta-pixel', @@ -58,7 +50,7 @@ export const destinationMeta: Destination = { vendor(config, env); const fbq = pixel(env); - await call( + await stepCall( () => fbq('init', pixelId), 'vendor-not-ready', 'Meta Pixel init call failed', @@ -79,7 +71,7 @@ export const destinationMeta: Destination = { const eventName = track || trackCustom || event.name; if (!rule.silent) - await call( + await stepCall( () => fbq( trackCustom ? 'trackCustom' : 'track', @@ -103,7 +95,7 @@ export const destinationMeta: Destination = { const { fbq } = getEnv(env).window; if (typeof fbq !== 'function') return; - await call( + await stepCall( () => fbq('consent', granted ? 'grant' : 'revoke'), 'consent-not-applied', 'Meta Pixel did not take the consent update', diff --git a/packages/web/destinations/mixpanel/src/__tests__/consent.test.ts b/packages/web/destinations/mixpanel/src/__tests__/consent.test.ts new file mode 100644 index 000000000..b210eda57 --- /dev/null +++ b/packages/web/destinations/mixpanel/src/__tests__/consent.test.ts @@ -0,0 +1,33 @@ +import { startFlow } from '@walkeros/collector'; +import { clone } from '@walkeros/core'; +import { destinationMixpanel } from '..'; +import { examples } from '../dev'; + +describe('mixpanel destination consent', () => { + it('follows the whole consent state: an unrelated update keeps tracking on', async () => { + const calls: string[] = []; + const env = clone(examples.env.push); + const mixpanel = env.mixpanel; + if (!mixpanel) throw new Error('env.mixpanel missing'); + mixpanel.opt_in_tracking = () => { + calls.push('opt_in'); + }; + mixpanel.opt_out_tracking = () => { + calls.push('opt_out'); + }; + const { elb } = await startFlow({ + destinations: { + mixpanel: { + code: { ...destinationMixpanel, env }, + config: { consent: { analytics: true }, settings: { apiKey: 'k' } }, + }, + }, + }); + + await elb('walker consent', { analytics: true }); + await elb('walker consent', { marketing: true }); + await elb('walker consent', { analytics: false }); + + expect(calls).toEqual(['opt_in', 'opt_out']); + }); +}); diff --git a/packages/web/destinations/mixpanel/src/examples/step.ts b/packages/web/destinations/mixpanel/src/examples/step.ts index ab5c6f25b..a169c019f 100644 --- a/packages/web/destinations/mixpanel/src/examples/step.ts +++ b/packages/web/destinations/mixpanel/src/examples/step.ts @@ -454,24 +454,14 @@ export const pageViewSilent: MixpanelStepExample = { /** * The silent group profile rule of companyUpdateGroupProfile on an update * without data: no group key and id resolve and the rule suppresses the - * track, so nothing is sent. + * track, so nothing is sent. A declared skip: an unmet mapping, not a + * failure. */ export const companyUpdateWithoutData: MixpanelStepExample = { public: false, description: - 'A silent rule whose group profile mapping finds no group sends nothing, so the push fails with missing-input instead of counting as delivered.', + 'A silent rule whose group profile mapping finds no group sends nothing: an unmet mapping, not a failure, so the push returns normally.', in: { name: 'company update', entity: 'company', action: 'update' }, - mapping: { - silent: true, - settings: { - groupProfile: { - map: { - key: { value: 'company_id' }, - id: 'data.company_id', - }, - }, - }, - }, + mapping: companyUpdateGroupProfile.mapping, out: [], - outcome: { status: 'failed', code: 'missing-input' }, }; diff --git a/packages/web/destinations/mixpanel/src/index.ts b/packages/web/destinations/mixpanel/src/index.ts index c2d6ee513..4358a38c3 100644 --- a/packages/web/destinations/mixpanel/src/index.ts +++ b/packages/web/destinations/mixpanel/src/index.ts @@ -1,11 +1,12 @@ -import type { WalkerOS } from '@walkeros/core'; import { + getGrantedConsent, getMappingValue, isArray, isBoolean, isObject, isString, StepError, + stepCall, } from '@walkeros/core'; import mixpanel from 'mixpanel-browser'; import type { @@ -29,19 +30,6 @@ function getMixpanel(env: Env | undefined): MixpanelSDK { return env?.mixpanel ?? (mixpanel as unknown as MixpanelSDK); } -/** Runs one Mixpanel call; a throw or a rejection becomes a StepError of `code`. */ -async function call( - fn: () => unknown, - code: string, - message: string, -): Promise { - try { - await fn(); - } catch (error) { - throw new StepError(message, { code, cause: error }); - } -} - /** * Resolve the distinctId from an identify mapping value and call * mixpanel.identify() only if the value changed since the last call. @@ -56,7 +44,7 @@ async function applyIdentify( const distinctId = resolved.distinctId; if (!isString(distinctId) || distinctId === '') return updated; if (distinctId === lastIdentity.distinctId) return updated; - await call( + await stepCall( () => mp.identify(distinctId), 'transport', 'Mixpanel identify failed', @@ -96,7 +84,7 @@ async function applyPeople( if (!isObject(bag)) continue; if (Object.keys(bag).length === 0) continue; // Typed dispatch - each op's first arg is a Record. - await call( + await stepCall( () => (mp.people[op] as (props: Record) => void)( bag as Record, @@ -112,7 +100,7 @@ async function applyPeople( isString(v), ); if (names.length > 0) - await call( + await stepCall( () => mp.people.unset(names), 'transport', 'Mixpanel people.unset failed', @@ -121,7 +109,7 @@ async function applyPeople( } if (resolved.delete_user === true) { - await call( + await stepCall( () => mp.people.delete_user(), 'transport', 'Mixpanel people.delete_user failed', @@ -155,7 +143,7 @@ async function applyGroupProfile( const bag = resolved[op]; if (!isObject(bag)) continue; if (Object.keys(bag).length === 0) continue; - await call( + await stepCall( () => (group[op] as (props: Record) => void)( bag as Record, @@ -171,7 +159,7 @@ async function applyGroupProfile( isString(v), ); if (names.length > 0) - await call( + await stepCall( () => group.unset(names), 'transport', 'Mixpanel group.unset failed', @@ -180,7 +168,7 @@ async function applyGroupProfile( } if (resolved.delete === true) { - await call( + await stepCall( () => group.delete(), 'transport', 'Mixpanel group.delete failed', @@ -223,7 +211,7 @@ export const destinationMixpanel: Destination = { }; // Mixpanel's init() is synchronous (returns void). - await call( + await stepCall( () => mp.init(apiKey, initConfig), 'vendor-not-ready', 'Mixpanel init failed', @@ -240,8 +228,6 @@ export const destinationMixpanel: Destination = { const settings = (config.settings || {}) as Settings; const mappingSettings = rule?.settings || {}; const state: RuntimeState = settings._state || {}; - // Whether any Mixpanel call or state change came out of this event - let sent = false; // 1. Reset - fires first so subsequent identity calls start clean. if (mappingSettings.reset !== undefined) { @@ -249,9 +235,8 @@ export const destinationMixpanel: Destination = { ? mappingSettings.reset : await getMappingValue(event, mappingSettings.reset, { collector }); if (resolved) { - await call(() => mp.reset(), 'transport', 'Mixpanel reset failed'); + await stepCall(() => mp.reset(), 'transport', 'Mixpanel reset failed'); state.lastIdentity = {}; - sent = true; } } @@ -267,7 +252,6 @@ export const destinationMixpanel: Destination = { resolved as Record, state.lastIdentity, ); - sent = sent || isString(resolved.distinctId); } } @@ -276,10 +260,8 @@ export const destinationMixpanel: Destination = { const resolved = await getMappingValue(event, mappingSettings.people, { collector, }); - if (isObject(resolved)) { - sent = - (await applyPeople(mp, resolved as Record)) || sent; - } + if (isObject(resolved)) + await applyPeople(mp, resolved as Record); } // 4. Group assignment - resolves to { key, id } and calls set_group. @@ -296,12 +278,11 @@ export const destinationMixpanel: Destination = { (isString(id) || isArray(id)) && id !== '' ) { - await call( + await stepCall( () => mp.set_group(key, id as string | string[]), 'transport', 'Mixpanel set_group failed', ); - sent = true; } } } @@ -313,11 +294,8 @@ export const destinationMixpanel: Destination = { mappingSettings.groupProfile, { collector }, ); - if (isObject(resolved)) { - sent = - (await applyGroupProfile(mp, resolved as Record)) || - sent; - } + if (isObject(resolved)) + await applyGroupProfile(mp, resolved as Record); } // 6. Default track - unless the rule opts out via silent. @@ -326,43 +304,30 @@ export const destinationMixpanel: Destination = { const properties = isObject(data) ? (data as Record) : {}; - await call( + await stepCall( () => mp.track(eventName, properties), 'transport', 'Mixpanel track failed', ); - sent = true; } - // A silent rule none of whose mappings resolved sent nothing at all - if (!sent && rule?.silent === true && Object.keys(mappingSettings).length) - throw new StepError( - 'Mixpanel mappings resolved to nothing for this event', - { - code: 'missing-input', - }, - ); - - // Persist state mutations back onto config.settings. + // A silent rule whose mappings resolve nothing sends nothing: a declared + // skip, not a failure. Persist state mutations back onto config.settings. settings._state = state; }, - async on(type, context) { - if (type !== 'consent' || !context.data) return; - const mp = getMixpanel(context.env as Env | undefined); + // The destination's required consent decides on the whole consent state, + // as it gates the pushes, and only a change that names a required key + // reaches Mixpanel: a revocation opts out, a grant opts back in. + async on(type, { config, data, env, collector }) { + const required = config.consent; + if (type !== 'consent' || !required || !isObject(data)) return; + if (!Object.keys(required).some((key) => key in data)) return; - const consent = context.data as WalkerOS.Consent; - // Derive the consent keys from config.consent - iterate every key the - // destination declared as required. If ALL required keys are granted, - // opt IN; otherwise opt OUT. - const required = context.config?.consent; - if (!required || Object.keys(required).length === 0) return; - - const allGranted = Object.keys(required).every( - (key) => consent[key] === true, - ); - await call( - () => (allGranted ? mp.opt_in_tracking() : mp.opt_out_tracking()), + const mp = getMixpanel(env as Env | undefined); + const granted = getGrantedConsent(required, collector.consent) !== false; + await stepCall( + () => (granted ? mp.opt_in_tracking() : mp.opt_out_tracking()), 'consent-not-applied', 'Mixpanel did not take the consent update', ); diff --git a/packages/web/destinations/mixpanel/src/schemas/settings.ts b/packages/web/destinations/mixpanel/src/schemas/settings.ts index 5abcb871f..59f36d5c1 100644 --- a/packages/web/destinations/mixpanel/src/schemas/settings.ts +++ b/packages/web/destinations/mixpanel/src/schemas/settings.ts @@ -101,6 +101,8 @@ export const SettingsSchema = z ) .optional(), }) - .strict(); + // Pass-through settings are loose: every other Mixpanel Config key is + // forwarded to init() as given. + .loose(); export type Settings = z.infer; diff --git a/packages/web/destinations/optimizely/src/__tests__/consent.test.ts b/packages/web/destinations/optimizely/src/__tests__/consent.test.ts new file mode 100644 index 000000000..384dde55c --- /dev/null +++ b/packages/web/destinations/optimizely/src/__tests__/consent.test.ts @@ -0,0 +1,47 @@ +import { startFlow } from '@walkeros/collector'; +import { getEvent } from '@walkeros/core'; +import { destinationOptimizely } from '..'; +import type { Env } from '../types'; + +describe('optimizely destination consent', () => { + it('tracks again after a revocation and a new grant', async () => { + const tracked: string[] = []; + const env: Env = { + optimizely: { + createInstance: () => ({ + onReady: () => Promise.resolve({ success: true }), + createUserContext: () => ({ + trackEvent: (eventKey: string) => { + tracked.push(eventKey); + }, + setAttribute: () => undefined, + }), + close: () => undefined, + }), + createPollingProjectConfigManager: () => ({}), + createBatchEventProcessor: () => ({}), + }, + }; + const { elb } = await startFlow({ + destinations: { + optimizely: { + code: { ...destinationOptimizely, env }, + config: { + consent: { analytics: true }, + settings: { sdkKey: 'key', userId: 'user.id' }, + }, + }, + }, + }); + + await elb('walker consent', { analytics: true }); + expect((await elb(getEvent('page view'))).ok).toBe(true); + await elb('walker consent', { analytics: false }); + await elb('walker consent', { analytics: true }); + const after = await elb(getEvent('product view')); + + expect(after.ok).toBe(true); + expect(after.done?.optimizely).toBeDefined(); + expect(tracked).toEqual(['page view', 'product view']); + }); +}); diff --git a/packages/web/destinations/optimizely/src/examples/step.ts b/packages/web/destinations/optimizely/src/examples/step.ts index ffd188550..077d41189 100644 --- a/packages/web/destinations/optimizely/src/examples/step.ts +++ b/packages/web/destinations/optimizely/src/examples/step.ts @@ -157,30 +157,29 @@ export const attributesOnlySilent: OptimizelyStepExample = { }; /** - * Consent revoked -- the destination closes the Optimizely client, - * flushing queued events and stopping datafile polling. + * Consent revoked: no vendor call. Optimizely sends only what walkerOS tracks, + * which the consent gate stops, so the client keeps running. */ export const consentRevoked: OptimizelyStepExample = { title: 'Consent revoked', description: - 'A walker consent command with analytics denied closes the Optimizely client and flushes queued events.', + 'A walker consent command with analytics denied makes no Optimizely call: walkerOS stops tracking for the consent-gated destination, and the client stays ready for the next grant.', command: 'consent', before: { analytics: true }, - in: { analytics: false } as WalkerOS.Consent, - settings: {} as Partial, - out: [['optimizely.close']], + in: { analytics: false }, + out: [], }; /** - * Consent granted -- no immediate SDK action needed. The destination - * re-initializes on the next push (walkerOS queues events until consent - * is granted, then re-inits). No calls expected. + * Consent granted: no SDK call. The client kept running, so the next push + * after a grant tracks again. */ export const consentGranted: OptimizelyStepExample = { public: false, + description: + 'A walker consent grant makes no Optimizely call: the client kept running, so the next push tracks again.', command: 'consent', - in: { analytics: true } as WalkerOS.Consent, - settings: {} as Partial, + in: { analytics: true }, out: [], }; diff --git a/packages/web/destinations/optimizely/src/index.ts b/packages/web/destinations/optimizely/src/index.ts index 8508865e1..2b1dd71d1 100644 --- a/packages/web/destinations/optimizely/src/index.ts +++ b/packages/web/destinations/optimizely/src/index.ts @@ -1,10 +1,10 @@ -import type { WalkerOS } from '@walkeros/core'; import { getMappingValue, isNumber, isObject, isString, StepError, + stepCall, } from '@walkeros/core'; import * as optimizelySDK from '@optimizely/optimizely-sdk'; import type { @@ -27,19 +27,6 @@ function getOptimizely(env: Env | undefined): OptimizelySDKType { return optimizelySDK as unknown as OptimizelySDKType; } -/** Runs one Optimizely call; a throw or a rejection becomes a StepError of `code`. */ -async function call( - fn: () => T | Promise, - code: string, - message: string, -): Promise { - try { - return await fn(); - } catch (error) { - throw new StepError(message, { code, cause: error }); - } -} - export const destinationOptimizely: Destination = { type: 'optimizely', @@ -55,7 +42,7 @@ export const destinationOptimizely: Destination = { const sdk = getOptimizely(env as Env | undefined); const { sdkKey } = settings; - const configManager = await call( + const configManager = await stepCall( () => sdk.createPollingProjectConfigManager({ sdkKey, @@ -66,7 +53,7 @@ export const destinationOptimizely: Destination = { 'Optimizely config manager failed to start', ); - const eventProcessor = await call( + const eventProcessor = await stepCall( () => sdk.createBatchEventProcessor({ batchSize: settings.batchSize ?? 10, @@ -81,12 +68,12 @@ export const destinationOptimizely: Destination = { eventProcessor: eventProcessor, }; - const client = await call( + const client = await stepCall( () => sdk.createInstance(instanceConfig), 'vendor-not-ready', 'Optimizely client failed to start', ); - const ready = await call( + const ready = await stepCall( () => client.onReady(), 'vendor-not-ready', 'Optimizely client did not become ready', @@ -144,7 +131,7 @@ export const destinationOptimizely: Destination = { // 3. Create or reuse user context if (userId !== state.lastUserId || !state.userContext) { - const userContext = await call( + const userContext = await stepCall( () => client.createUserContext(userId, baseAttributes), 'transport', 'Optimizely createUserContext failed', @@ -174,7 +161,7 @@ export const destinationOptimizely: Destination = { for (const [key, value] of Object.entries( resolved as Record, )) { - await call( + await stepCall( () => userContext.setAttribute(key, value), 'transport', 'Optimizely setAttribute failed', @@ -222,7 +209,7 @@ export const destinationOptimizely: Destination = { } } - await call( + await stepCall( () => userContext.trackEvent(eventKey, eventTags), 'transport', 'Optimizely trackEvent failed', @@ -233,29 +220,9 @@ export const destinationOptimizely: Destination = { settings._state = state; }, - async on(type, context) { - if (type !== 'consent' || !context.data) return; - - const required = context.config?.consent; - if (!required || Object.keys(required).length === 0) return; - - const consent = context.data as WalkerOS.Consent; - const allGranted = Object.keys(required).every( - (key) => consent[key] === true, - ); - - const state = (context.config?.settings as Settings)?._state; - if (!allGranted && state?.client) { - const { client } = state; - state.client = undefined; - state.userContext = undefined; - await call( - () => client.close(), - 'consent-not-applied', - 'Optimizely did not stop on the consent revocation', - ); - } - }, + // No on('consent'): Optimizely sends only the decisions and events walkerOS + // tracks, which the consent gate stops, so a revocation needs no vendor + // call and the client keeps running for the next grant. destroy({ config }) { const settings = (config?.settings || {}) as Settings; diff --git a/packages/web/destinations/piano/src/index.ts b/packages/web/destinations/piano/src/index.ts index 8c3e06de0..0f947231a 100644 --- a/packages/web/destinations/piano/src/index.ts +++ b/packages/web/destinations/piano/src/index.ts @@ -1,5 +1,5 @@ import type { Config, Env, Destination, PianoAnalytics } from './types'; -import { isObject, StepError } from '@walkeros/core'; +import { isObject, StepError, stepCall } from '@walkeros/core'; import { loadVendor } from '@walkeros/web-core'; // Types @@ -46,24 +46,23 @@ export const destinationPiano: Destination = { const pa = await vendor(config, env).ready(); const settings = config.settings; - try { - if (settings && !configured.has(pa)) { - await pa.setConfigurations({ - site: settings.site, - collectDomain: settings.collectDomain, - ...(isObject(settings.options) ? settings.options : {}), - }); - configured.add(pa); - } + await stepCall( + async () => { + if (settings && !configured.has(pa)) { + await pa.setConfigurations({ + site: settings.site, + collectDomain: settings.collectDomain, + ...(isObject(settings.options) ? settings.options : {}), + }); + configured.add(pa); + } - if (!rule?.silent) - await pa.sendEvent(event.name, isObject(data) ? data : {}); - } catch (error) { - throw new StepError('Piano Analytics call failed', { - code: 'transport', - cause: error, - }); - } + if (!rule?.silent) + await pa.sendEvent(event.name, isObject(data) ? data : {}); + }, + 'transport', + 'Piano Analytics call failed', + ); }, }; diff --git a/packages/web/destinations/pinterest/src/__tests__/consent.test.ts b/packages/web/destinations/pinterest/src/__tests__/consent.test.ts new file mode 100644 index 000000000..23b4271a6 --- /dev/null +++ b/packages/web/destinations/pinterest/src/__tests__/consent.test.ts @@ -0,0 +1,40 @@ +import { startFlow } from '@walkeros/collector'; +import { clone, getEvent } from '@walkeros/core'; +import { destinationPinterest } from '..'; +import { examples } from '../dev'; +import type { Pintrk } from '../types'; + +describe('pinterest destination consent', () => { + it('follows the whole consent state: an unrelated update keeps tracking, a revocation stops it', async () => { + const tracked: unknown[] = []; + const env = clone(examples.env.push); + const pintrk: Pintrk = Object.assign( + (...args: unknown[]) => { + if (args[0] === 'track') tracked.push(args[1]); + }, + { queue: [], version: '3.0' }, + ); + env.window.pintrk = pintrk; + const { elb } = await startFlow({ + destinations: { + pinterest: { + code: { ...destinationPinterest, env }, + config: { + consent: { marketing: true }, + loadScript: false, + settings: { apiKey: '2612345678901' }, + }, + }, + }, + }); + + await elb('walker consent', { marketing: true }); + await elb('walker consent', { analytics: false }); + await elb(getEvent('product view')); + await elb('walker consent', { marketing: false }); + await elb('walker consent', { marketing: true }); + await elb(getEvent('product add')); + + expect(tracked).toEqual(['product view', 'product add']); + }); +}); diff --git a/packages/web/destinations/pinterest/src/__tests__/init.test.ts b/packages/web/destinations/pinterest/src/__tests__/init.test.ts index b0110e27b..babd26419 100644 --- a/packages/web/destinations/pinterest/src/__tests__/init.test.ts +++ b/packages/web/destinations/pinterest/src/__tests__/init.test.ts @@ -13,18 +13,19 @@ import type { Env, Pintrk } from '../types'; function spy(env: Env): unknown[][] { const calls: unknown[][] = []; - const fn = ((...args: unknown[]) => { - calls.push(args); - }) as unknown as Pintrk; - fn.queue = []; - fn.version = '3.0'; + const fn: Pintrk = Object.assign( + (...args: unknown[]) => { + calls.push(args); + }, + { queue: [], version: '3.0' }, + ); env.window.pintrk = fn; return calls; } describe('pinterest destination — init', () => { it('fires pintrk("load", tagId) then pintrk("page") when pageview is true (default)', async () => { - const env = clone(examples.env.push) as Env; + const env = clone(examples.env.push); const calls = spy(env); const dest = jest.requireActual('../').default; @@ -43,7 +44,7 @@ describe('pinterest destination — init', () => { }); it('skips pintrk("page") when pageview is false', async () => { - const env = clone(examples.env.push) as Env; + const env = clone(examples.env.push); const calls = spy(env); const dest = jest.requireActual('../').default; @@ -65,7 +66,7 @@ describe('pinterest destination — init', () => { it('throws a config error when apiKey is missing', async () => { const dest = jest.requireActual('../').default; - const env = clone(examples.env.push) as Env; + const env = clone(examples.env.push); await expect( dest.init({ config: { settings: {} }, diff --git a/packages/web/destinations/pinterest/src/__tests__/stepExamples.test.ts b/packages/web/destinations/pinterest/src/__tests__/stepExamples.test.ts index d1ecad5a1..972fc6184 100644 --- a/packages/web/destinations/pinterest/src/__tests__/stepExamples.test.ts +++ b/packages/web/destinations/pinterest/src/__tests__/stepExamples.test.ts @@ -9,12 +9,32 @@ import type { WalkerOS, Mapping as WalkerOSMapping } from '@walkeros/core'; import { startFlow } from '@walkeros/collector'; -import { clone } from '@walkeros/core'; +import { clone, isObject, isString } from '@walkeros/core'; import { examples } from '../dev'; import type { Env, Pintrk } from '../types'; type CallRecord = [string, ...unknown[]]; +function isConsent(value: unknown): value is WalkerOS.Consent { + return ( + isObject(value) && + Object.values(value).every((granted) => typeof granted === 'boolean') + ); +} + +function isEvent(value: unknown): value is WalkerOS.Event { + return ( + isObject(value) && + isString(value.name) && + isString(value.entity) && + isString(value.action) + ); +} + +function isRule(value: unknown): value is WalkerOSMapping.Rule { + return isObject(value); +} + /** * Install a spy on env.window.pintrk that records every call as a * CallRecord tuple (["window.pintrk", ...args]). @@ -32,21 +52,16 @@ function spyEnv(env: Env): { env: Env; collected: () => CallRecord[] } { } describe('pinterest destination — step examples', () => { - it.each(Object.entries(examples.step))('%s', async (_name, rawExample) => { - const example = rawExample as { - in?: unknown; - mapping?: unknown; - out?: ReadonlyArray; - command?: 'consent' | 'user' | 'config' | 'run'; - }; - - const env = clone(examples.env.push) as Env; + it.each(Object.entries(examples.step))('%s', async (name, example) => { + const env = clone(examples.env.push); const { env: spiedEnv, collected } = spyEnv(env); const dest = jest.requireActual('../').default; const { elb } = await startFlow(); if (example.command === 'consent') { + if (!isConsent(example.in)) + throw new Error(`step example "${name}" has no consent input`); await elb('walker destination', { code: { ...dest, env: spiedEnv }, config: { @@ -54,10 +69,12 @@ describe('pinterest destination — step examples', () => { settings: { apiKey: '2612345678901' }, }, }); - await elb('walker consent', example.in as WalkerOS.Consent); + await elb('walker consent', example.in); } else { - const event = example.in as WalkerOS.Event; - const mapping = example.mapping as WalkerOSMapping.Rule | undefined; + if (!isEvent(example.in)) + throw new Error(`step example "${name}" has no event input`); + const event = example.in; + const mapping = isRule(example.mapping) ? example.mapping : undefined; const mappingConfig = mapping ? { [event.entity]: { [event.action]: mapping } } : undefined; @@ -72,13 +89,12 @@ describe('pinterest destination — step examples', () => { await elb(event); } - // Drop init-time calls (load + page) — covered by init.test.ts. - const expected = (example.out ?? []) as ReadonlyArray; + // Drop init-time calls (load + page): covered by init.test.ts. const actual = collected().filter((call) => { const cmd = call[1]; return cmd !== 'load' && cmd !== 'page'; }); - expect(actual).toEqual(expected); + expect(actual).toEqual([...(example.out ?? [])]); }); }); diff --git a/packages/web/destinations/pinterest/src/examples/env.ts b/packages/web/destinations/pinterest/src/examples/env.ts index 866af5bf3..a73f61514 100644 --- a/packages/web/destinations/pinterest/src/examples/env.ts +++ b/packages/web/destinations/pinterest/src/examples/env.ts @@ -3,7 +3,7 @@ import type { Env, Pintrk } from '../types'; /** * Example environment configurations for the Pinterest destination. * - * These mock window.pintrk and document, allowing tests to run in a pure + * These mock window.pintrk, allowing tests to run in a pure * Node environment without jsdom and without touching the real Pinterest * CDN. Tests replace env.window.pintrk with a jest.fn() spy. */ @@ -11,60 +11,28 @@ import type { Env, Pintrk } from '../types'; const noop = () => {}; const noopPintrk: Pintrk = Object.assign(noop, { - queue: [] as unknown[][], + queue: [], version: '3.0', -}) as unknown as Pintrk; +}); /** - * Pre-init environment — pintrk is not yet on the window. + * Pre-init environment: pintrk is not yet on the window. * Used to validate the init path that creates the queue function. */ export const init: Env = { window: { pintrk: undefined, }, - document: { - createElement: () => ({ - src: '', - async: false, - dataset: {}, - setAttribute: () => {}, - removeAttribute: () => {}, - }), - head: { appendChild: () => {} }, - getElementsByTagName: () => [ - { - parentNode: { insertBefore: () => {} }, - }, - ], - querySelector: () => null, - }, }; /** - * Post-init environment — pintrk is available as a spy-able function. + * Post-init environment: pintrk is available as a spy-able function. * Tests clone this and replace `pintrk` with a jest.fn() to record calls. */ export const push: Env = { window: { pintrk: noopPintrk, }, - document: { - createElement: () => ({ - src: '', - async: false, - dataset: {}, - setAttribute: () => {}, - removeAttribute: () => {}, - }), - head: { appendChild: () => {} }, - getElementsByTagName: () => [ - { - parentNode: { insertBefore: () => {} }, - }, - ], - querySelector: () => null, - }, }; /** Simulation tracking paths for CLI --simulate. */ diff --git a/packages/web/destinations/pinterest/src/examples/step.ts b/packages/web/destinations/pinterest/src/examples/step.ts index 9ac62fbca..0af2aeb23 100644 --- a/packages/web/destinations/pinterest/src/examples/step.ts +++ b/packages/web/destinations/pinterest/src/examples/step.ts @@ -1,5 +1,5 @@ -import type { Flow, WalkerOS } from '@walkeros/core'; -import { createEvent, getEvent } from '@walkeros/core'; +import type { Flow } from '@walkeros/core'; +import { createEvent, getEvent, isObject } from '@walkeros/core'; /** * Output convention: @@ -198,10 +198,10 @@ export const orderCompleteCheckout: Flow.StepExample = { loop: [ 'nested', { - condition: (value: unknown) => { - const v = value as { data?: { price?: unknown } }; - return typeof v?.data?.price === 'number'; - }, + condition: (value: unknown) => + isObject(value) && + isObject(value.data) && + typeof value.data.price === 'number', map: { product_id: 'data.id', product_name: 'data.color', @@ -329,7 +329,7 @@ export const consentRevoke: Flow.StepExample = { description: 'For a destination with config.consent { marketing: true }, revoking marketing makes no vendor call: the destination stops sending track calls itself.', command: 'consent', - in: { marketing: false } as WalkerOS.Consent, + in: { marketing: false }, out: [], }; @@ -338,8 +338,10 @@ export const consentRevoke: Flow.StepExample = { */ export const consentGrant: Flow.StepExample = { public: false, + description: + 'A grant of the required consent lets the destination send track calls again; Pinterest has no consent call to make.', command: 'consent', - in: { marketing: true } as WalkerOS.Consent, + in: { marketing: true }, out: [], }; @@ -355,24 +357,13 @@ export const silentRule: Flow.StepExample = { /** * The silent identify rule of identifyOnlySilent on an update without data: * no identity resolves and the rule suppresses the track, so nothing is - * sent. + * sent. A declared skip: an unmet mapping, not a failure. */ export const identifyOnlySilentWithoutData: Flow.StepExample = { public: false, description: - 'A silent rule whose identify mapping finds no email or id sends nothing, so the push fails with missing-input instead of counting as delivered.', + 'A silent rule whose identify mapping finds no email or id sends nothing: an unmet mapping, not a failure, so the push returns normally.', in: { name: 'user update', entity: 'user', action: 'update' }, - mapping: { - silent: true, - settings: { - identify: { - map: { - em: 'data.email', - external_id: 'data.id', - }, - }, - }, - }, + mapping: identifyOnlySilent.mapping, out: [], - outcome: { status: 'failed', code: 'missing-input' }, }; diff --git a/packages/web/destinations/pinterest/src/index.ts b/packages/web/destinations/pinterest/src/index.ts index cf63dbe2c..c2c8b9b58 100644 --- a/packages/web/destinations/pinterest/src/index.ts +++ b/packages/web/destinations/pinterest/src/index.ts @@ -1,7 +1,14 @@ import type { DestinationWeb } from '@walkeros/web-core'; import { getEnv, loadVendor } from '@walkeros/web-core'; -import { getMappingValue, isObject, isString, StepError } from '@walkeros/core'; -import type { Collector, WalkerOS } from '@walkeros/core'; +import { + getGrantedConsent, + getMappingValue, + isObject, + isString, + StepError, + stepCall, +} from '@walkeros/core'; +import type { Collector, Mapping, WalkerOS } from '@walkeros/core'; import type { Destination, EventData, @@ -22,15 +29,24 @@ const PINTEREST_SCRIPT_SRC = 'https://s.pinimg.com/ct/core.js'; */ function installPintrkQueue(env?: DestinationWeb.Env): void { const { window } = getEnv(env); - const w = window as Window & { pintrk?: Pintrk }; - if (w.pintrk) return; - const queue: unknown[][] = []; - const fn = ((...args: unknown[]) => { - (w.pintrk!.queue as unknown[][]).push(args); - }) as unknown as Pintrk; - fn.queue = queue; - fn.version = '3.0'; - w.pintrk = fn; + if (window.pintrk) return; + const pintrk: Pintrk = Object.assign( + (...args: unknown[]) => { + pintrk.queue?.push(args); + }, + { queue: [], version: '3.0' }, + ); + window.pintrk = pintrk; +} + +/** The pintrk queue or tag, or a StepError when it is not there. */ +function getPintrk(env?: DestinationWeb.Env): Pintrk { + const { pintrk } = getEnv(env).window; + if (!pintrk) + throw new StepError('Pinterest Tag not ready: pintrk is missing', { + code: 'vendor-not-ready', + }); + return pintrk; } /** @@ -44,43 +60,25 @@ function vendor( ) { return loadVendor( config.loadScript !== false ? PINTEREST_SCRIPT_SRC : undefined, - () => (getEnv(env).window as Window).pintrk, + () => getEnv(env).window.pintrk, { timeout: config.timeout }, ); } -/** Runs one pintrk call; a throw or a rejection becomes a StepError of `code`. */ -async function call( - fn: () => unknown, - code: string, - message: string, -): Promise { - try { - await fn(); - } catch (error) { - throw new StepError(message, { code, cause: error }); - } -} - /** * Resolve an identify mapping value into IdentifyFields. Strict allow-list: * only `em` and `external_id` pass through. */ async function resolveIdentify( event: WalkerOS.Event, - mappingValue: unknown, + mappingValue: Mapping.Value | undefined, collector: Collector.Instance, ): Promise { if (mappingValue === undefined || mappingValue === null) return undefined; - const resolved = await getMappingValue( - event, - mappingValue as Parameters[1], - { collector }, - ); + const resolved = await getMappingValue(event, mappingValue, { collector }); if (!isObject(resolved)) return undefined; const fields: IdentifyFields = {}; - const em = (resolved as Record).em; - const external_id = (resolved as Record).external_id; + const { em, external_id } = resolved; if (isString(em) && em) fields.em = em; if (isString(external_id) && external_id) fields.external_id = external_id; return Object.keys(fields).length > 0 ? fields : undefined; @@ -115,16 +113,15 @@ export const destinationPinterest: Destination = { installPintrkQueue(env); vendor(config, env); - const { window } = getEnv(env); - const pintrk = (window as Window).pintrk as Pintrk; + const pintrk = getPintrk(env); - await call( + await stepCall( () => pintrk('load', apiKey), 'vendor-not-ready', 'Pinterest load call failed', ); if (pageview) - await call( + await stepCall( () => pintrk('page'), 'vendor-not-ready', 'Pinterest page call failed', @@ -143,30 +140,20 @@ export const destinationPinterest: Destination = { async push(event, { config, rule, data, env, collector }) { vendor(config, env).check(); - const settings = (config.settings || {}) as Settings; + const settings: Partial = config.settings || {}; const state = (settings._state ||= { consentGranted: true }); - const { window } = getEnv(env); - const pintrk = (window as Window).pintrk as Pintrk; - if (!pintrk) - throw new StepError('Pinterest Tag not ready: pintrk is missing', { - code: 'vendor-not-ready', - }); + const pintrk = getPintrk(env); // Consent revoked: suppress. This is the consent state, not a failure. if (state.consentGranted !== false) { // 1. Identity - per-event override falls back to destination-level. - const identifyMapping = - (rule?.settings?.identify as unknown) ?? settings.identify; + // A silent rule whose identify resolves nothing (or an identity that + // is already current) sends nothing: a declared skip, not a failure. + const identifyMapping = rule?.settings?.identify ?? settings.identify; const identify = await resolveIdentify(event, identifyMapping, collector); - // A silent rule that resolves no identity sends nothing at all - if (!identify && rule?.silent === true && identifyMapping !== undefined) - throw new StepError( - 'Pinterest identify resolved to nothing for this event', - { code: 'missing-input' }, - ); if (identify && !identifyEqual(identify, state.lastIdentify)) { - await call( + await stepCall( () => pintrk('set', identify), 'transport', 'Pinterest set call failed', @@ -177,22 +164,18 @@ export const destinationPinterest: Destination = { // 2. Silent - process identity (above) but suppress the default track. if (rule?.silent !== true) { // 3. Track - const eventName = isString(rule?.name) - ? (rule!.name as string) - : event.name; + const eventName = isString(rule?.name) ? rule.name : event.name; // context.data contains both include-flattened and mapping.data-resolved // properties. Include is the bottom layer, mapping.data wins on conflict. - const contextData: Record = isObject(data) - ? (data as Record) - : {}; + const contextData: Record = isObject(data) ? data : {}; const eventData: EventData = { ...(event.id ? { event_id: event.id } : {}), ...contextData, }; - await call( + await stepCall( () => pintrk('track', eventName, eventData), 'transport', 'Pinterest track call failed', @@ -201,39 +184,21 @@ export const destinationPinterest: Destination = { } }, - on(type, context) { - if (type !== 'consent' || !context.data) return; - const settings = context.config?.settings as Settings | undefined; - if (!settings) return; - - const consent = context.data as WalkerOS.Consent; - const required = context.config?.consent; - if (!required) return; - - const keys = Object.keys(required); - if (keys.length === 0) return; - - // Deny as soon as any required key is explicitly false. Grant only if - // every required key is explicitly true. - let anyDenied = false; - let allGranted = true; - for (const key of keys) { - if (consent[key] === true) continue; - if (consent[key] === false) { - anyDenied = true; - allGranted = false; - } else { - // undefined / unknown - don't flip - allGranted = false; - } - } - - const state = (settings._state ||= { consentGranted: true }); - if (anyDenied) { - state.consentGranted = false; - } else if (allGranted) { - state.consentGranted = true; - } + // Pinterest has no consent API: the destination stops its own sends. The + // destination's required consent decides on the whole consent state, as + // it gates the pushes, and only a change that names a required key counts. + on(type, { config, data, collector }) { + const required = config.consent; + if (type !== 'consent' || !required || !isObject(data)) return; + if (!Object.keys(required).some((key) => key in data)) return; + const settings = config.settings; + if (!isObject(settings)) return; + + const state = isObject(settings._state) + ? settings._state + : (settings._state = { consentGranted: true }); + state.consentGranted = + getGrantedConsent(required, collector.consent) !== false; }, }; diff --git a/packages/web/destinations/piwikpro/src/index.ts b/packages/web/destinations/piwikpro/src/index.ts index ea8571a82..d33a14012 100644 --- a/packages/web/destinations/piwikpro/src/index.ts +++ b/packages/web/destinations/piwikpro/src/index.ts @@ -5,6 +5,7 @@ import { isArray, isObject, StepError, + stepCall, } from '@walkeros/core'; import { getEnv, loadVendor } from '@walkeros/web-core'; import type { DimensionValues } from './dimensions'; @@ -31,17 +32,17 @@ const linkTracked = new WeakSet(); * Pushes commands onto the queue in one synchronous block, so another * event's commands never interleave, then waits for what each push returns. */ -async function pushAll( +function pushAll( queue: unknown[], commands: unknown[][], code: string, message: string, -): Promise { - try { - await Promise.all(commands.map((command) => queue.push(command))); - } catch (error) { - throw new StepError(message, { code, cause: error }); - } +): Promise { + return stepCall( + () => Promise.all(commands.map((command) => queue.push(command))), + code, + message, + ); } /** diff --git a/packages/web/destinations/plausible/src/index.ts b/packages/web/destinations/plausible/src/index.ts index c799a2c0f..0bb3fe859 100644 --- a/packages/web/destinations/plausible/src/index.ts +++ b/packages/web/destinations/plausible/src/index.ts @@ -1,5 +1,5 @@ import type { Config, Destination, Env } from './types'; -import { isObject, StepError } from '@walkeros/core'; +import { isObject, StepError, stepCall } from '@walkeros/core'; import { getEnv, loadVendor } from '@walkeros/web-core'; // Types @@ -40,14 +40,11 @@ export const destinationPlausible: Destination = { // A silent rule sends no Plausible event if (!rule?.silent) { const params = isObject(data) ? data : {}; - try { - await plausible(`${event.name}`, params); - } catch (error) { - throw new StepError('Plausible call failed', { - code: 'transport', - cause: error, - }); - } + await stepCall( + () => plausible(`${event.name}`, params), + 'transport', + 'Plausible call failed', + ); } }, }; diff --git a/packages/web/destinations/posthog/src/__tests__/consent.test.ts b/packages/web/destinations/posthog/src/__tests__/consent.test.ts new file mode 100644 index 000000000..fe21812eb --- /dev/null +++ b/packages/web/destinations/posthog/src/__tests__/consent.test.ts @@ -0,0 +1,33 @@ +import { startFlow } from '@walkeros/collector'; +import { clone } from '@walkeros/core'; +import { destinationPostHog } from '..'; +import { examples } from '../dev'; + +describe('posthog destination consent', () => { + it('follows the whole consent state: an unrelated update keeps capturing on', async () => { + const calls: string[] = []; + const env = clone(examples.env.push); + const posthog = env.posthog; + if (!posthog) throw new Error('env.posthog missing'); + posthog.opt_in_capturing = () => { + calls.push('opt_in'); + }; + posthog.opt_out_capturing = () => { + calls.push('opt_out'); + }; + const { elb } = await startFlow({ + destinations: { + posthog: { + code: { ...destinationPostHog, env }, + config: { consent: { analytics: true }, settings: { apiKey: 'k' } }, + }, + }, + }); + + await elb('walker consent', { analytics: true }); + await elb('walker consent', { marketing: true }); + await elb('walker consent', { analytics: false }); + + expect(calls).toEqual(['opt_in', 'opt_out']); + }); +}); diff --git a/packages/web/destinations/posthog/src/examples/step.ts b/packages/web/destinations/posthog/src/examples/step.ts index d89ae2eba..a0985eae6 100644 --- a/packages/web/destinations/posthog/src/examples/step.ts +++ b/packages/web/destinations/posthog/src/examples/step.ts @@ -377,24 +377,14 @@ export const pageViewSilent: PostHogStepExample = { /** * The silent group rule of groupAssignmentWithProperties on an update * without data: no group type and key resolve and the rule suppresses the - * capture, so nothing is sent. + * capture, so nothing is sent. A declared skip: an unmet mapping, not a + * failure. */ export const companyUpdateWithoutData: PostHogStepExample = { public: false, description: - 'A silent rule whose group mapping finds no group sends nothing, so the push fails with missing-input instead of counting as delivered.', + 'A silent rule whose group mapping finds no group sends nothing: an unmet mapping, not a failure, so the push returns normally.', in: { name: 'company update', entity: 'company', action: 'update' }, - mapping: { - silent: true, - settings: { - group: { - map: { - type: { value: 'company' }, - key: 'data.company_id', - }, - }, - }, - }, + mapping: groupAssignmentWithProperties.mapping, out: [], - outcome: { status: 'failed', code: 'missing-input' }, }; diff --git a/packages/web/destinations/posthog/src/index.ts b/packages/web/destinations/posthog/src/index.ts index 8f59d7d45..dee3e387c 100644 --- a/packages/web/destinations/posthog/src/index.ts +++ b/packages/web/destinations/posthog/src/index.ts @@ -1,5 +1,11 @@ -import type { WalkerOS } from '@walkeros/core'; -import { getMappingValue, isObject, isString, StepError } from '@walkeros/core'; +import { + getGrantedConsent, + getMappingValue, + isObject, + isString, + StepError, + stepCall, +} from '@walkeros/core'; import posthogSDK from 'posthog-js'; import type { Destination, @@ -22,19 +28,6 @@ function getPostHog(env: Env | undefined): PostHogSDK { return env?.posthog ?? (posthogSDK as unknown as PostHogSDK); } -/** Runs one PostHog call; a throw or a rejection becomes a StepError of `code`. */ -async function call( - fn: () => unknown, - code: string, - message: string, -): Promise { - try { - await fn(); - } catch (error) { - throw new StepError(message, { code, cause: error }); - } -} - /** * Resolve the identify mapping value, call the right SDK method. * @@ -67,7 +60,7 @@ async function applyIdentity( if (isString(distinctId) && distinctId !== '') { if (distinctId !== lastIdentity.distinctId || $set || $setOnce) { - await call( + await stepCall( () => posthog.identify(distinctId, $set, $setOnce), 'transport', 'PostHog identify failed', @@ -76,7 +69,7 @@ async function applyIdentity( } applied = true; } else if ($set || $setOnce) { - await call( + await stepCall( () => posthog.setPersonProperties($set, $setOnce), 'transport', 'PostHog setPersonProperties failed', @@ -110,7 +103,7 @@ async function applyGroup( if (isString(type) && isString(key)) { if (type !== lastGroup.type || key !== lastGroup.key || properties) { - await call( + await stepCall( () => posthog.group(type, key, properties), 'transport', 'PostHog group failed', @@ -160,7 +153,7 @@ export const destinationPostHog: Destination = { // destination is truly ready before returning; otherwise we resolve // immediately so tests with no-op mocks don't hang. let loadedFired = false; - await call( + await stepCall( () => posthog.init(apiKey, { ...options, @@ -184,8 +177,6 @@ export const destinationPostHog: Destination = { const settings = (config.settings || {}) as Settings; const mappingSettings = rule?.settings || {}; const state: RuntimeState = settings._state || {}; - // Whether any PostHog call or state change came out of this event - let sent = false; // 1. Reset - fires first so subsequent identity calls start clean. if (mappingSettings.reset !== undefined) { @@ -196,10 +187,13 @@ export const destinationPostHog: Destination = { collector, }); if (resolved) { - await call(() => posthog.reset(), 'transport', 'PostHog reset failed'); + await stepCall( + () => posthog.reset(), + 'transport', + 'PostHog reset failed', + ); state.lastIdentity = {}; state.lastGroup = {}; - sent = true; } } @@ -216,7 +210,6 @@ export const destinationPostHog: Destination = { state.lastIdentity, ); state.lastIdentity = identity.lastIdentity; - sent = sent || identity.applied; } } @@ -233,7 +226,6 @@ export const destinationPostHog: Destination = { state.lastGroup, ); state.lastGroup = group.lastGroup; - sent = sent || group.applied; } } @@ -243,46 +235,32 @@ export const destinationPostHog: Destination = { const properties = isObject(data) ? (data as Record) : {}; - await call( + await stepCall( () => posthog.capture(eventName, properties), 'transport', 'PostHog capture failed', ); - sent = true; } - // A silent rule none of whose mappings resolved sent nothing at all - if (!sent && rule?.silent === true && Object.keys(mappingSettings).length) - throw new StepError( - 'PostHog mappings resolved to nothing for this event', - { - code: 'missing-input', - }, - ); - - // Persist state. walkerOS destinations are long-lived singletons so + // A silent rule whose mappings resolve nothing sends nothing: a declared + // skip, not a failure. Persist state. walkerOS destinations are long-lived singletons so // mutating settings in place is safe (Snowplow precedent). settings._state = state; }, - async on(type, context) { - if (type !== 'consent' || !context?.data) return; - const posthog = getPostHog(context.env as Env | undefined); - - const consent = context.data as WalkerOS.Consent; - // Derive required consent keys from config.consent. If all declared - // keys are granted, opt IN; otherwise opt OUT. Missing config.consent - // means nothing to check and we take no action. - const required = (context.config as { consent?: WalkerOS.Consent }) - ?.consent; - if (!required || Object.keys(required).length === 0) return; + // The destination's required consent decides on the whole consent state, + // as it gates the pushes, and only a change that names a required key + // reaches PostHog: a revocation opts out, a grant opts back in. + async on(type, { config, data, env, collector }) { + const required = config.consent; + if (type !== 'consent' || !required || !isObject(data)) return; + if (!Object.keys(required).some((key) => key in data)) return; - const allGranted = Object.keys(required).every( - (key) => consent[key] === true, - ); - await call( + const posthog = getPostHog(env as Env | undefined); + const granted = getGrantedConsent(required, collector.consent) !== false; + await stepCall( () => - allGranted ? posthog.opt_in_capturing() : posthog.opt_out_capturing(), + granted ? posthog.opt_in_capturing() : posthog.opt_out_capturing(), 'consent-not-applied', 'PostHog did not take the consent update', ); diff --git a/packages/web/destinations/posthog/src/schemas/settings.ts b/packages/web/destinations/posthog/src/schemas/settings.ts index 0807ece14..79e540597 100644 --- a/packages/web/destinations/posthog/src/schemas/settings.ts +++ b/packages/web/destinations/posthog/src/schemas/settings.ts @@ -115,6 +115,8 @@ export const SettingsSchema = z ) .optional(), }) - .strict(); + // Pass-through settings are loose: every other posthog-js config key + // (respect_dnt, ...) is forwarded to init() as given. + .loose(); export type Settings = z.infer; diff --git a/packages/web/destinations/segment/src/examples/step.ts b/packages/web/destinations/segment/src/examples/step.ts index 91597beb6..55501e0a3 100644 --- a/packages/web/destinations/segment/src/examples/step.ts +++ b/packages/web/destinations/segment/src/examples/step.ts @@ -502,6 +502,8 @@ export const consentContextOmitted: SegmentStepExample = { */ export const consentRevokeNoOp: SegmentStepExample = { public: false, + description: + 'Segment has no opt-out and Analytics.js sends nothing on its own (initialPageview false), so a revocation makes no call: walkerOS stops the pushes of a consent-gated destination.', command: 'consent', in: { analytics: false } as WalkerOS.Consent, out: [], @@ -518,17 +520,14 @@ export const silentRule: SegmentStepExample = { /** * The silent group rule of companyUpdateGroup on an update without data: no - * groupId resolves and the rule suppresses the track, so nothing is sent. + * groupId resolves, so nothing is sent. A declared skip: an unmet mapping, + * not a failure. */ export const companyUpdateWithoutData: SegmentStepExample = { public: false, description: - 'A silent rule whose group mapping finds no groupId sends nothing, so the push fails with missing-input instead of counting as delivered.', + 'A silent rule whose group mapping finds no groupId sends nothing: an unmet mapping, not a failure, so the push returns normally.', in: { name: 'company update', entity: 'company', action: 'update' }, - mapping: { - silent: true, - settings: { group: { map: { groupId: 'data.company_id' } } }, - }, + mapping: companyUpdateGroup.mapping, out: [], - outcome: { status: 'failed', code: 'missing-input' }, }; diff --git a/packages/web/destinations/segment/src/index.ts b/packages/web/destinations/segment/src/index.ts index f4fd4b386..85c527010 100644 --- a/packages/web/destinations/segment/src/index.ts +++ b/packages/web/destinations/segment/src/index.ts @@ -6,6 +6,7 @@ import { isString, isThenable, StepError, + stepCall, } from '@walkeros/core'; import { AnalyticsBrowser } from '@segment/analytics-next'; import type { @@ -76,17 +77,8 @@ function isSegmentAnalytics(value: unknown): value is SegmentAnalytics { ); } -/** Waits for one Analytics.js call; a throw or a rejection becomes a StepError. */ -async function send(fn: () => unknown): Promise { - try { - await fn(); - } catch (error) { - throw new StepError('Segment call failed', { - code: 'transport', - cause: error, - }); - } -} +const send = (fn: () => unknown) => + stepCall(fn, 'transport', 'Segment call failed'); /** * Stable hash of a traits object for state-diffing. Uses a deterministic @@ -141,15 +133,14 @@ function buildEventOptions( /** * Resolve and apply identity mapping. Calls analytics.identify() only - * when the resolved value differs from the last-fired identity. Returns - * whether it called the SDK. + * when the resolved value differs from the last-fired identity. */ async function applyIdentify( analytics: SegmentAnalytics, resolved: Record, state: RuntimeState, options?: SegmentEventOptions, -): Promise { +): Promise { const last = state.lastIdentity || {}; const userId = isString(resolved.userId) ? resolved.userId : undefined; const anonymousId = isString(resolved.anonymousId) @@ -160,18 +151,15 @@ async function applyIdentify( ? resolved.traits : undefined; - let called = false; - if (anonymousId && anonymousId !== last.anonymousId) { + if (anonymousId && anonymousId !== last.anonymousId) await send(() => analytics.setAnonymousId(anonymousId)); - called = true; - } const traitsHash = hashTraits(traits); const userIdChanged = userId !== last.userId; const traitsChanged = traitsHash !== (last.traitsHash ?? ''); const hasChange = userIdChanged || (traits !== undefined && traitsChanged); - if (!hasChange && !anonymousId) return called; + if (!hasChange && !anonymousId) return; await send(() => options @@ -184,18 +172,17 @@ async function applyIdentify( anonymousId: anonymousId ?? last.anonymousId, traitsHash, }; - return true; } -/** Resolve and apply the group mapping; returns whether it called the SDK. */ +/** Resolve and apply the group mapping, once per change. */ async function applyGroup( analytics: SegmentAnalytics, resolved: Record, state: RuntimeState, options?: SegmentEventOptions, -): Promise { +): Promise { const groupId = isString(resolved.groupId) ? resolved.groupId : undefined; - if (!groupId) return false; + if (!groupId) return; const traits = isObject(resolved.traits) && !isArray(resolved.traits) @@ -206,7 +193,7 @@ async function applyGroup( const traitsHash = hashTraits(traits); const groupIdChanged = groupId !== last.groupId; const traitsChanged = traitsHash !== (last.traitsHash ?? ''); - if (!groupIdChanged && !traitsChanged) return false; + if (!groupIdChanged && !traitsChanged) return; await send(() => options @@ -215,17 +202,16 @@ async function applyGroup( ); state.lastGroup = { groupId, traitsHash }; - return true; } -/** Resolve and apply the page mapping; returns whether it called the SDK. */ +/** Resolve and apply the page mapping. */ async function applyPage( analytics: SegmentAnalytics, pageMapping: unknown, event: WalkerOS.Event, collector: unknown, options?: SegmentEventOptions, -): Promise { +): Promise { // Boolean true → call page() with no args (SDK auto-collect). if (pageMapping === true) { await send(() => @@ -233,7 +219,7 @@ async function applyPage( ? analytics.page(undefined, undefined, undefined, options) : analytics.page(), ); - return true; + return; } const resolved = await getMappingValue( @@ -241,7 +227,7 @@ async function applyPage( pageMapping as Parameters[1], { collector } as Parameters[2], ); - if (!isObject(resolved)) return false; + if (!isObject(resolved)) return; const category = isString(resolved.category) ? resolved.category : undefined; const name = isString(resolved.name) ? resolved.name : undefined; @@ -264,7 +250,6 @@ async function applyPage( return analytics.page(category ?? null, name ?? null); return analytics.page(); }); - return true; } export const destinationSegment: Destination = { @@ -281,19 +266,21 @@ export const destinationSegment: Destination = { const settings: Settings = { ...config.settings, apiKey }; const _state: RuntimeState = { lastIdentity: {}, lastGroup: {} }; - let loaded: unknown; - try { - loaded = loadAnalytics(getSegment(env), settings); - // load() answers the instance at once; a wrapped SDK may answer it - // through a promise. - if (!isSegmentAnalytics(loaded) && isThenable(loaded)) - loaded = await loaded; - } catch (error) { - throw new StepError('Segment load() failed', { - code: 'vendor-not-ready', - cause: error, - }); - } + // load() answers the instance at once, itself a thenable that settles + // once Analytics.js is ready, so it is not awaited; a wrapped SDK may + // answer the instance through a promise. The tuple keeps it unawaited. + const [loaded] = await stepCall( + async (): Promise<[unknown]> => { + const result: unknown = loadAnalytics(getSegment(env), settings); + return [ + isSegmentAnalytics(result) || !isThenable(result) + ? result + : await result, + ]; + }, + 'vendor-not-ready', + 'Segment load() failed', + ); if (!isSegmentAnalytics(loaded)) throw new StepError('Segment load() returned no Analytics instance', { code: 'vendor-not-ready', @@ -328,7 +315,6 @@ export const destinationSegment: Destination = { const mappingSettings = rule?.settings || {}; const options = buildEventOptions(event, settings); - let calls = 0; // 1. Reset fires first so subsequent identity calls start clean. if (mappingSettings.reset !== undefined) { @@ -338,7 +324,6 @@ export const destinationSegment: Destination = { : await getMappingValue(event, mappingSettings.reset, { collector }); if (resolved) { await send(() => analytics.reset()); - calls++; state.lastIdentity = {}; state.lastGroup = {}; } @@ -350,11 +335,8 @@ export const destinationSegment: Destination = { const resolved = await getMappingValue(event, identifyMapping, { collector, }); - if ( - isObject(resolved) && - (await applyIdentify(analytics, resolved, state, options)) - ) - calls++; + if (isObject(resolved)) + await applyIdentify(analytics, resolved, state, options); } // 3. Group - rule-level override wins over destination-level. @@ -363,25 +345,19 @@ export const destinationSegment: Destination = { const resolved = await getMappingValue(event, groupMapping, { collector, }); - if ( - isObject(resolved) && - (await applyGroup(analytics, resolved, state, options)) - ) - calls++; + if (isObject(resolved)) + await applyGroup(analytics, resolved, state, options); } // 4. Page (explicit first-class Segment pattern) - if ( - mappingSettings.page !== undefined && - (await applyPage( + if (mappingSettings.page !== undefined) + await applyPage( analytics, mappingSettings.page, event, collector, options, - )) - ) - calls++; + ); // 5. Default track call unless rule opts out via silent. if (rule?.silent !== true) { @@ -393,22 +369,11 @@ export const destinationSegment: Destination = { ? analytics.track(eventName, properties, options) : analytics.track(eventName, properties), ); - calls++; } + // A silent rule whose settings resolve nothing (or an identity or group + // that is already current) sends nothing: a declared skip. settings._state = state; - - // A silent rule whose own settings resolved nothing sent nothing. - const ruleSettings = - mappingSettings.reset !== undefined || - mappingSettings.identify !== undefined || - mappingSettings.group !== undefined || - mappingSettings.page !== undefined; - if (!calls && ruleSettings) - throw new StepError( - `Segment: the silent rule of "${event.name}" resolved no identity, group or page`, - { code: 'missing-input' }, - ); }, }; diff --git a/packages/web/destinations/segment/src/schemas/settings.ts b/packages/web/destinations/segment/src/schemas/settings.ts index fb9ed2614..c53968f84 100644 --- a/packages/web/destinations/segment/src/schemas/settings.ts +++ b/packages/web/destinations/segment/src/schemas/settings.ts @@ -1,66 +1,72 @@ import { z } from '@walkeros/core/dev'; -export const SettingsSchema = z.object({ - apiKey: z - .string() - .min(1) - .describe( - 'Your Segment source write key. Find it in your Segment workspace under Connections, Sources, Settings, API Keys. Maps to `writeKey` in the Analytics.js `load()` call (like a1B2c3D4e5F6a1B2c3D4e5F6a1B2c3D4).', - ), - cdnURL: z - .string() - .describe( - 'Not applied: the destination passes it in the load() options, where Analytics.js does not read it, so settings are always fetched from https://cdn.segment.com.', - ) - .optional(), - initialPageview: z - .boolean() - .describe( - 'When true, the SDK fires an automatic initial page() call on load. Default: false (walkerOS sources handle page tracking, so this is disabled to avoid duplicate page views).', - ) - .optional(), - disableClientPersistence: z - .boolean() - .describe( - 'When true, prevents any cookie or localStorage writes. Useful for privacy-conscious setups. Default: false.', - ) - .optional(), - disableAutoISOConversion: z - .boolean() - .describe('Disable automatic ISO string → Date conversion. Default: false.') - .optional(), - retryQueue: z - .boolean() - .describe('Retry failed events. Default: true.') - .optional(), - obfuscate: z - .boolean() - .describe('Obfuscate event payloads. Default: false.') - .optional(), - integrations: z - .record(z.string(), z.unknown()) - .describe( - 'Enable/disable downstream Segment destinations. Example: { "All": true, "Mixpanel": false }.', - ) - .optional(), - identify: z - .unknown() - .describe( - 'Destination-level identity mapping. Resolves to an object with any of: userId, traits, anonymousId. Resolved on every push: analytics.identify fires on the first push and when userId or traits change, and on every push while anonymousId is set.', - ) - .optional(), - group: z - .unknown() - .describe( - 'Destination-level group mapping. Resolves to an object with: groupId, traits. Fires on the first push and re-fires on change.', - ) - .optional(), - consent: z - .record(z.string(), z.string()) - .describe( - 'Mapping from walkerOS consent keys → Segment `categoryPreferences` keys. Example: { "marketing": "Advertising", "analytics": "Analytics" }. Only mapped keys are forwarded; when omitted, no consent context is sent.', - ) - .optional(), -}); +export const SettingsSchema = z + .object({ + apiKey: z + .string() + .min(1) + .describe( + 'Your Segment source write key. Find it in your Segment workspace under Connections, Sources, Settings, API Keys. Maps to `writeKey` in the Analytics.js `load()` call (like a1B2c3D4e5F6a1B2c3D4e5F6a1B2c3D4).', + ), + cdnURL: z + .string() + .describe( + 'Not applied: the destination passes it in the load() options, where Analytics.js does not read it, so settings are always fetched from https://cdn.segment.com.', + ) + .optional(), + initialPageview: z + .boolean() + .describe( + 'When true, the SDK fires an automatic initial page() call on load. Default: false (walkerOS sources handle page tracking, so this is disabled to avoid duplicate page views).', + ) + .optional(), + disableClientPersistence: z + .boolean() + .describe( + 'When true, prevents any cookie or localStorage writes. Useful for privacy-conscious setups. Default: false.', + ) + .optional(), + disableAutoISOConversion: z + .boolean() + .describe( + 'Disable automatic ISO string → Date conversion. Default: false.', + ) + .optional(), + retryQueue: z + .boolean() + .describe('Retry failed events. Default: true.') + .optional(), + obfuscate: z + .boolean() + .describe('Obfuscate event payloads. Default: false.') + .optional(), + integrations: z + .record(z.string(), z.unknown()) + .describe( + 'Enable/disable downstream Segment destinations. Example: { "All": true, "Mixpanel": false }.', + ) + .optional(), + identify: z + .unknown() + .describe( + 'Destination-level identity mapping. Resolves to an object with any of: userId, traits, anonymousId. Resolved on every push: analytics.identify fires on the first push and when userId or traits change, and on every push while anonymousId is set.', + ) + .optional(), + group: z + .unknown() + .describe( + 'Destination-level group mapping. Resolves to an object with: groupId, traits. Fires on the first push and re-fires on change.', + ) + .optional(), + consent: z + .record(z.string(), z.string()) + .describe( + 'Mapping from walkerOS consent keys → Segment `categoryPreferences` keys. Example: { "marketing": "Advertising", "analytics": "Analytics" }. Only mapped keys are forwarded; when omitted, no consent context is sent.', + ) + .optional(), + }) + // Pass-through settings are loose: every other Analytics.js InitOptions + // key is forwarded to load() as given. + .loose(); export type Settings = z.infer; diff --git a/packages/web/destinations/snowplow/src/__tests__/conformance.test.ts b/packages/web/destinations/snowplow/src/__tests__/conformance.test.ts index 92b6349a8..47b0725fa 100644 --- a/packages/web/destinations/snowplow/src/__tests__/conformance.test.ts +++ b/packages/web/destinations/snowplow/src/__tests__/conformance.test.ts @@ -16,6 +16,7 @@ describe('snowplow conformance', () => { settings: { collectorUrl: 'https://collector.example.com', pageViewEvent: 'page view', + activityTracking: { minimumVisitLength: 30, heartbeatDelay: 10 }, }, env: { push: examples.env.push, simulation: examples.env.simulation }, root: path.join(__dirname, '..', '..'), diff --git a/packages/web/destinations/snowplow/src/__tests__/consent.test.ts b/packages/web/destinations/snowplow/src/__tests__/consent.test.ts new file mode 100644 index 000000000..4cbe296b1 --- /dev/null +++ b/packages/web/destinations/snowplow/src/__tests__/consent.test.ts @@ -0,0 +1,60 @@ +import { startFlow } from '@walkeros/collector'; +import { getEvent } from '@walkeros/core'; +import { destinationSnowplow } from '..'; +import type { Env, SnowplowFunction } from '../types'; + +describe('snowplow destination consent', () => { + it('turns the tracker senders off on a revocation and on again on a grant', async () => { + const calls: unknown[][] = []; + const snowplow: SnowplowFunction = Object.assign( + (...args: unknown[]) => { + calls.push(args); + }, + { q: [] }, + ); + const env: Env = { window: { snowplow } }; + const activityTracking = { minimumVisitLength: 30, heartbeatDelay: 10 }; + const linkClicks = { trackContent: true }; + const { elb } = await startFlow({ + destinations: { + snowplow: { + code: { ...destinationSnowplow, env }, + config: { + consent: { marketing: true }, + settings: { + collectorUrl: 'https://collector.example.com', + activityTracking, + plugins: [ + { + url: 'https://cdn.example.com/link-click-tracking.js', + name: [ + 'snowplowLinkClickTracking', + 'LinkClickTrackingPlugin', + ], + options: linkClicks, + }, + ], + }, + }, + }, + }, + }); + + await elb('walker consent', { marketing: true }); + await elb(getEvent('page view')); + calls.length = 0; + + await elb('walker consent', { marketing: false }); + expect(calls).toEqual([ + ['disableActivityTracking'], + ['disableLinkClickTracking'], + ]); + + calls.length = 0; + await elb('walker consent', { marketing: true }); + expect(calls).toEqual([ + ['enableActivityTracking', activityTracking], + ['enableLinkClickTracking', linkClicks], + ]); + }); +}); diff --git a/packages/web/destinations/snowplow/src/__tests__/stepExamples.test.ts b/packages/web/destinations/snowplow/src/__tests__/stepExamples.test.ts index 61af958b0..9e4001a50 100644 --- a/packages/web/destinations/snowplow/src/__tests__/stepExamples.test.ts +++ b/packages/web/destinations/snowplow/src/__tests__/stepExamples.test.ts @@ -35,8 +35,10 @@ function spySnowplow(env: Env): { } describe('snowplow destination — step examples', () => { + // The consent revoke runs in the conformance test (gated, granted first) + // and in consent.test.ts. const stepEntries = Object.entries(examples.step).filter( - ([name]) => name !== 'init', + ([name, example]) => name !== 'init' && example.command !== 'consent', ); it('init', async () => { diff --git a/packages/web/destinations/snowplow/src/adapter.ts b/packages/web/destinations/snowplow/src/adapter.ts index 062a9010f..04664812b 100644 --- a/packages/web/destinations/snowplow/src/adapter.ts +++ b/packages/web/destinations/snowplow/src/adapter.ts @@ -8,6 +8,7 @@ import type { ActivityTrackingConfiguration, BrowserPlugin, } from '@snowplow/browser-tracker-core'; +import { StepError } from '@walkeros/core'; /** * Create adapter from sp.js command queue (window.snowplow) @@ -37,6 +38,9 @@ export function createQueueAdapter( enableActivityTracking(config: ActivityTrackingConfiguration) { snowplow('enableActivityTracking', config); }, + disableActivityTracking() { + snowplow('disableActivityTracking'); + }, addPlugin(config: { plugin: BrowserPlugin } | [string, [string, string]]) { if (Array.isArray(config)) { // URL-based plugin: [url, [namespace, constructor]] @@ -121,6 +125,15 @@ export function createBrowserTrackerAdapter( functions.enableActivityTracking(config); } }, + disableActivityTracking() { + // Page pings keep going without it: the revocation cannot be applied. + if (!functions.disableActivityTracking) + throw new StepError( + 'Snowplow tracker.disableActivityTracking is missing, so activity pings continue after a revocation', + { code: 'consent-not-applied' }, + ); + functions.disableActivityTracking(); + }, addPlugin(config: { plugin: BrowserPlugin } | [string, [string, string]]) { if (functions.addPlugin && !Array.isArray(config)) { functions.addPlugin(config); diff --git a/packages/web/destinations/snowplow/src/examples/step.ts b/packages/web/destinations/snowplow/src/examples/step.ts index ceed2a6b2..099de19b3 100644 --- a/packages/web/destinations/snowplow/src/examples/step.ts +++ b/packages/web/destinations/snowplow/src/examples/step.ts @@ -471,15 +471,15 @@ export const silentRule: Flow.StepExample = { }; /** - * A revocation of the consent the destination requires. Without - * settings.consent there is no Enhanced Consent call, so no vendor call is - * made and the collector's consent gate stops the sending. + * A revocation of the consent the destination requires turns off the + * tracker's own senders: with activityTracking set, the page pings stop. + * Without settings.consent there is no Enhanced Consent event. */ export const consentRevoke: Flow.StepExample = { public: false, description: - 'For a destination with config.consent { marketing: true } and no settings.consent, revoking marketing makes no vendor call: the collector stops sending.', + 'For a destination with config.consent { marketing: true } and activityTracking, revoking marketing calls disableActivityTracking, so sp.js stops its page pings; a grant enables them again.', command: 'consent', in: { marketing: false }, - out: [], + out: [['window.snowplow', 'disableActivityTracking']], }; diff --git a/packages/web/destinations/snowplow/src/index.ts b/packages/web/destinations/snowplow/src/index.ts index 056418eb1..e67a6c1d5 100644 --- a/packages/web/destinations/snowplow/src/index.ts +++ b/packages/web/destinations/snowplow/src/index.ts @@ -16,7 +16,12 @@ import { import { DEFAULT_SCRIPT_URL, setup } from './setup'; import { pushSnowplowEvent } from './push'; import { createQueueAdapter, createBrowserTrackerAdapter } from './adapter'; -import { isObject, StepError } from '@walkeros/core'; +import { + getGrantedConsent, + isObject, + StepError, + stepCall, +} from '@walkeros/core'; import { getEnv, loadVendor } from '@walkeros/web-core'; // Types @@ -51,20 +56,52 @@ function vendor( } /** - * Runs one Snowplow call or block; a throw or a rejection that is not - * already a StepError becomes a StepError of `code`. + * Turns the tracker's own senders off on a revocation of the destination's + * required consent and on again on the next grant: activity pings, and link + * click tracking where a URL plugin enabled it. The required consent decides + * on the whole consent state, as it gates the pushes, and only a change that + * names a required key counts. */ -async function guard( - fn: () => T | Promise, - code: string, - message: string, -): Promise { - try { - return await fn(); - } catch (error) { - if (error instanceof StepError) throw error; - throw new StepError(message, { code, cause: error }); - } +async function followConsent( + settings: Partial, + required: WalkerOS.Consent | undefined, + change: WalkerOS.AnyObject, + consent: WalkerOS.Consent, +): Promise { + const state = settings._state; + const adapter = state?.adapter; + if (!state || !adapter || !required) return; + if (!Object.keys(required).some((key) => key in change)) return; + + const granted = getGrantedConsent(required, consent) !== false; + if (granted !== Boolean(state.paused)) return; + + const linkClicks = (settings.plugins ?? []) + .filter(isUrlBasedPlugin) + .filter( + (plugin) => + (plugin.enableMethod ?? deriveEnableMethod(plugin.name[1])) === + 'enableLinkClickTracking', + ); + const { activityTracking } = settings; + + await stepCall( + () => { + if (activityTracking) { + if (granted) adapter.enableActivityTracking(activityTracking); + else adapter.disableActivityTracking(); + } + for (const plugin of linkClicks) { + if (!granted) adapter.call('disableLinkClickTracking'); + else if (plugin.options) + adapter.call('enableLinkClickTracking', plugin.options); + else adapter.call('enableLinkClickTracking'); + } + }, + 'consent-not-applied', + 'Snowplow did not take the consent update', + ); + state.paused = !granted; } /** @@ -244,7 +281,7 @@ export const destinationSnowplow: Destination = { ); // Initialize tracker - await guard( + await stepCall( () => trackerFunctions.newTracker( settings.trackerName || 'sp', @@ -277,7 +314,7 @@ export const destinationSnowplow: Destination = { vendor({ ...config, settings }, env); // Initialize tracker via queue - await guard( + await stepCall( () => snowplow('newTracker', settings.trackerName || 'sp', collectorUrl, { appId: settings.appId || 'walkerOS', @@ -304,14 +341,14 @@ export const destinationSnowplow: Destination = { // Enable activity tracking if configured if (settings.activityTracking) { const activityTracking = settings.activityTracking; - await guard( + await stepCall( () => tracker.enableActivityTracking(activityTracking), 'vendor-not-ready', 'Snowplow enableActivityTracking failed', ); } - await guard( + await stepCall( () => { // Load plugins if (settings.plugins) { @@ -378,7 +415,7 @@ export const destinationSnowplow: Destination = { vendor(config, env as Env | undefined).check(); const eventMapping = rule.settings || {}; - await guard( + await stepCall( () => pushSnowplowEvent( event, @@ -415,6 +452,15 @@ export const destinationSnowplow: Destination = { // own Settings/Env needs a core-types change to make `on` generic over // Types; tracked as a follow-up. These two `as` casts are that boundary. const settings = (context.config?.settings || {}) as Partial; + + // The tracker's own senders follow the destination's required consent + await followConsent( + settings, + context.config.consent, + consent, + context.collector.consent, + ); + const consentConfig = settings.consent; // Skip if consent tracking is not configured @@ -448,7 +494,7 @@ export const destinationSnowplow: Destination = { baseParams.gdprApplies = consentConfig.gdprApplies; // Call the appropriate Enhanced Consent plugin method - await guard( + await stepCall( () => { if (allDenied) { // For deny, use required scopes (the ones being denied) diff --git a/packages/web/destinations/snowplow/src/types/index.ts b/packages/web/destinations/snowplow/src/types/index.ts index 744ec7096..c05116910 100644 --- a/packages/web/destinations/snowplow/src/types/index.ts +++ b/packages/web/destinations/snowplow/src/types/index.ts @@ -93,6 +93,8 @@ export interface TrackerFunctions { config: ActivityTrackingConfiguration, trackers?: string[], ) => void; + /** Disable activity tracking (page pings), on a consent revocation */ + disableActivityTracking?: (trackers?: string[]) => void; /** Add plugin */ addPlugin?: (config: { plugin: BrowserPlugin }, trackers?: string[]) => void; /** Add global contexts */ @@ -144,6 +146,7 @@ export interface SnowplowAdapter { trackStructEvent(event: Record): void; setUserId(userId?: string | null): void; enableActivityTracking(config: ActivityTrackingConfiguration): void; + disableActivityTracking(): void; addPlugin( config: { plugin: BrowserPlugin } | [string, [string, string]], ): void; @@ -385,6 +388,8 @@ export interface RuntimeState { userIdSet?: boolean; /** The initialized adapter instance */ adapter?: SnowplowAdapter; + /** The tracker's own senders are off after a consent revocation */ + paused?: boolean; } /** diff --git a/packages/web/destinations/tiktok/src/__tests__/consent.test.ts b/packages/web/destinations/tiktok/src/__tests__/consent.test.ts new file mode 100644 index 000000000..5e38351bd --- /dev/null +++ b/packages/web/destinations/tiktok/src/__tests__/consent.test.ts @@ -0,0 +1,35 @@ +import { startFlow } from '@walkeros/collector'; +import { clone } from '@walkeros/core'; +import { destinationTikTok } from '..'; +import { examples } from '../dev'; + +describe('tiktok destination consent', () => { + it('follows the whole consent state: an unrelated update keeps cookies on', async () => { + const calls: string[] = []; + const env = clone(examples.env.push); + env.window.ttq.enableCookie = () => { + calls.push('enableCookie'); + }; + env.window.ttq.disableCookie = () => { + calls.push('disableCookie'); + }; + const { elb } = await startFlow({ + destinations: { + tiktok: { + code: { ...destinationTikTok, env }, + config: { + consent: { marketing: true }, + loadScript: false, + settings: { apiKey: 'k' }, + }, + }, + }, + }); + + await elb('walker consent', { marketing: true }); + await elb('walker consent', { analytics: true }); + await elb('walker consent', { marketing: false }); + + expect(calls).toEqual(['enableCookie', 'disableCookie']); + }); +}); diff --git a/packages/web/destinations/tiktok/src/__tests__/stepExamples.test.ts b/packages/web/destinations/tiktok/src/__tests__/stepExamples.test.ts index 83d00479e..124e6e139 100644 --- a/packages/web/destinations/tiktok/src/__tests__/stepExamples.test.ts +++ b/packages/web/destinations/tiktok/src/__tests__/stepExamples.test.ts @@ -37,7 +37,7 @@ describe('tiktok destination — step examples', () => { '%s', async (name, example) => { // Fresh clone per test so mutations don't bleed across cases. - const env = clone(examples.env.push) as Env; + const env = clone(examples.env.push); const { env: spiedEnv, collected, reset } = spyTtq(env); const dest = jest.requireActual('../').default; diff --git a/packages/web/destinations/tiktok/src/examples/env.ts b/packages/web/destinations/tiktok/src/examples/env.ts index 4f7da311c..18ebc11e8 100644 --- a/packages/web/destinations/tiktok/src/examples/env.ts +++ b/packages/web/destinations/tiktok/src/examples/env.ts @@ -2,53 +2,41 @@ import type { Env, TTQ } from '../types'; const noop = () => {}; -/** Build a fresh mock ttq — each call returns a new object so tests can mutate independently. */ +/** Build a fresh mock ttq: each call returns a new object so tests can mutate independently. */ function makeMockTTQ(): TTQ { - const fn = (() => {}) as unknown as TTQ; - fn.load = noop; - fn.page = noop; - fn.track = noop; - fn.identify = noop; - fn.enableCookie = noop; - fn.disableCookie = noop; - fn.methods = [ - 'page', - 'track', - 'identify', - 'instances', - 'debug', - 'on', - 'off', - 'once', - 'ready', - 'alias', - 'group', - 'enableCookie', - 'disableCookie', - ]; - fn._i = {}; - fn.loaded = true; - return fn; + return Object.assign(() => {}, { + load: noop, + page: noop, + track: noop, + identify: noop, + enableCookie: noop, + disableCookie: noop, + methods: [ + 'page', + 'track', + 'identify', + 'instances', + 'debug', + 'on', + 'off', + 'once', + 'ready', + 'alias', + 'group', + 'enableCookie', + 'disableCookie', + ], + _i: {}, + loaded: true, + }); } -/** - * Minimal web env — window has a mock ttq. document has the bare minimum - * for addScript() (createElement + head.appendChild), with no-op - * implementations so init() can run without a real DOM. - */ +/** Minimal web env: window has a mock ttq. */ function makeEnv(): Env { return { window: { ttq: makeMockTTQ(), }, - document: { - createElement: () => - ({ - src: '', - async: false, - }) as unknown as Element, - head: { appendChild: noop }, - }, }; } diff --git a/packages/web/destinations/tiktok/src/examples/step.ts b/packages/web/destinations/tiktok/src/examples/step.ts index a0f453348..9776cfb2e 100644 --- a/packages/web/destinations/tiktok/src/examples/step.ts +++ b/packages/web/destinations/tiktok/src/examples/step.ts @@ -404,12 +404,12 @@ export const silentRule: TikTokStepExample = { /** * A silent rule that only sets an identity, on an event with no email, phone * or id: nothing resolves and the rule suppresses the track, so nothing is - * sent. + * sent. A declared skip: an unmet mapping, not a failure. */ export const silentIdentifyWithoutData: TikTokStepExample = { public: false, description: - 'A silent rule whose identify mapping finds no email, phone or id sends nothing, so the push fails with missing-input instead of counting as delivered.', + 'A silent rule whose identify mapping finds no email, phone or id sends nothing: an unmet mapping, not a failure, so the push returns normally.', in: { name: 'user update', entity: 'user', action: 'update' }, mapping: { silent: true, @@ -424,5 +424,4 @@ export const silentIdentifyWithoutData: TikTokStepExample = { }, }, out: [], - outcome: { status: 'failed', code: 'missing-input' }, }; diff --git a/packages/web/destinations/tiktok/src/index.ts b/packages/web/destinations/tiktok/src/index.ts index 4f7a4c688..ce1a90654 100644 --- a/packages/web/destinations/tiktok/src/index.ts +++ b/packages/web/destinations/tiktok/src/index.ts @@ -1,5 +1,12 @@ -import type { WalkerOS } from '@walkeros/core'; -import { getMappingValue, isObject, isString, StepError } from '@walkeros/core'; +import { + getGrantedConsent, + getMappingValue, + isObject, + isString, + StepError, + stepCall, +} from '@walkeros/core'; +import type { DestinationWeb } from '@walkeros/web-core'; import { getEnv, loadVendor } from '@walkeros/web-core'; import { SCRIPT_SRC, setup } from './setup'; import type { @@ -11,6 +18,22 @@ import type { TTQ, } from './types'; +const TTQ_METHODS = [ + 'load', + 'track', + 'identify', + 'enableCookie', + 'disableCookie', +]; + +/** A loaded Pixel (or a test mock), not the queue `setup()` installs. */ +function isTTQ(value: unknown): value is TTQ { + return ( + typeof value === 'function' && + TTQ_METHODS.every((key) => typeof Reflect.get(value, key) === 'function') + ); +} + // Types export export * as DestinationTikTok from './types'; @@ -19,10 +42,9 @@ export * as DestinationTikTok from './types'; * falls through to the global window when env is undefined (production), * or uses env.window when the caller provides a mock (tests). */ -function getTTQ(env: unknown): TTQ { - const { window } = getEnv(env as Parameters[0]); - const { ttq } = window as unknown as { ttq?: TTQ }; - if (!ttq) +function getTTQ(env?: DestinationWeb.Env): TTQ { + const { ttq } = getEnv(env).window; + if (!isTTQ(ttq)) throw new StepError('TikTok Pixel not ready: window.ttq is missing', { code: 'vendor-not-ready', }); @@ -36,31 +58,15 @@ function getTTQ(env: unknown): TTQ { */ function vendor( config: { loadScript?: boolean; timeout?: number }, - env: unknown, + env?: DestinationWeb.Env, ) { return loadVendor( config.loadScript !== false ? SCRIPT_SRC : undefined, - () => { - const { window } = getEnv(env as Parameters[0]); - return (window as unknown as { ttq?: TTQ }).ttq; - }, + () => getEnv(env).window.ttq, { timeout: config.timeout }, ); } -/** Runs one ttq call; a throw or a rejection becomes a StepError of `code`. */ -async function call( - fn: () => unknown, - code: string, - message: string, -): Promise { - try { - await fn(); - } catch (error) { - throw new StepError(message, { code, cause: error }); - } -} - /** * Resolve an identify mapping value and, if any Advanced Matching * parameter changed since last push, call ttq.identify() with the full @@ -73,10 +79,7 @@ async function applyIdentify( ttq: TTQ, resolved: Record, lastIdentity: NonNullable = {}, -): Promise<{ - lastIdentity: NonNullable; - resolved: boolean; -}> { +): Promise> { const next: NonNullable = { ...lastIdentity }; const params: IdentifyParams = {}; let changed = false; @@ -93,11 +96,14 @@ async function applyIdentify( // Skip ttq.identify() entirely if the resolved object has no non-empty // values (anonymous visitor) or if nothing changed. - if (!changed || Object.keys(params).length === 0) - return { lastIdentity: next, resolved: Object.keys(params).length > 0 }; + if (!changed || Object.keys(params).length === 0) return next; - await call(() => ttq.identify(params), 'transport', 'TikTok identify failed'); - return { lastIdentity: next, resolved: true }; + await stepCall( + () => ttq.identify(params), + 'transport', + 'TikTok identify failed', + ); + return next; } export const destinationTikTok: Destination = { @@ -132,7 +138,7 @@ export const destinationTikTok: Destination = { // TikTok's SDK auto-fires ttq.page() on load; we let it fire (no knob). const loadOptions: Record = { ...passthrough }; - await call( + await stepCall( () => ttq.load(apiKey, loadOptions), 'vendor-not-ready', 'TikTok load call failed', @@ -146,12 +152,10 @@ export const destinationTikTok: Destination = { async push(event, { config, rule, data, env, collector }) { vendor(config, env).check(); const ttq = getTTQ(env); - const settings = (config.settings || {}) as Settings; - const mappingSettings = (rule?.settings || {}) as Mapping; + const settings: Partial = config.settings || {}; + const mappingSettings: Mapping = rule?.settings || {}; const state: RuntimeState = settings._state || {}; - // Whether any ttq call or identity came out of this event - let sent = false; // Identity - rule-level override wins over destination-level. // Resolved and diffed against runtime state; only changes fire @@ -163,13 +167,11 @@ export const destinationTikTok: Destination = { collector, }); if (isObject(resolved)) { - const identity = await applyIdentify( + state.lastIdentity = await applyIdentify( ttq, - resolved as Record, + resolved, state.lastIdentity, ); - state.lastIdentity = identity.lastIdentity; - sent = identity.resolved; } } @@ -177,52 +179,35 @@ export const destinationTikTok: Destination = { // rules that are identify-only). if (rule?.silent !== true) { const eventName = isString(rule?.name) ? rule.name : event.name; - const params: Record = isObject(data) - ? (data as Record) - : {}; + const params: Record = isObject(data) ? data : {}; - await call( + await stepCall( () => ttq.track(eventName, params, { event_id: event.id }), 'transport', 'TikTok track failed', ); - sent = true; } - // A silent rule that resolved no identity sent nothing at all - if (!sent && rule?.silent === true && identifyMapping !== undefined) - throw new StepError( - 'TikTok identify resolved to nothing for this event', - { - code: 'missing-input', - }, - ); - + // A silent rule whose identify resolves nothing (or an identity that is + // already current) sends nothing: a declared skip, not a failure. // Persist runtime state mutations. walkerOS destinations are // long-lived singletons so mutating settings in place is safe and // matches the Snowplow precedent. settings._state = state; }, - async on(type, context) { - if (type !== 'consent' || !context.data) return; - const ttq = getTTQ(context.env); - - const consent = context.data as WalkerOS.Consent; - // Derive the consent key(s) to check from config.consent. TikTok is - // an advertising tool, so the typical key is "marketing". If ALL - // required keys are granted, enable cookies; otherwise disable. A - // missing config.consent means nothing to check and we take no - // action. - const required = context.config?.consent; - if (!required || Object.keys(required).length === 0) return; - - const allGranted = Object.keys(required).every( - (key) => consent[key] === true, - ); + // The destination's required consent decides on the whole consent state, + // as it gates the pushes, and only a change that names a required key + // reaches TikTok: a revocation disables its cookies, a grant enables them. + async on(type, { config, data, env, collector }) { + const required = config.consent; + if (type !== 'consent' || !required || !isObject(data)) return; + if (!Object.keys(required).some((key) => key in data)) return; - await call( - () => (allGranted ? ttq.enableCookie() : ttq.disableCookie()), + const ttq = getTTQ(env); + const granted = getGrantedConsent(required, collector.consent) !== false; + await stepCall( + () => (granted ? ttq.enableCookie() : ttq.disableCookie()), 'consent-not-applied', 'TikTok did not take the consent update', ); diff --git a/packages/web/destinations/tiktok/src/schemas/settings.ts b/packages/web/destinations/tiktok/src/schemas/settings.ts index a5f6a711a..135c7fb7b 100644 --- a/packages/web/destinations/tiktok/src/schemas/settings.ts +++ b/packages/web/destinations/tiktok/src/schemas/settings.ts @@ -29,6 +29,8 @@ export const SettingsSchema = z ) .optional(), }) - .strict(); + // Pass-through settings are loose: every other TikTok option is forwarded + // to ttq.load() as given. + .loose(); export type Settings = z.infer; diff --git a/packages/web/destinations/tiktok/src/setup.ts b/packages/web/destinations/tiktok/src/setup.ts index aca545568..9d6cfa0f4 100644 --- a/packages/web/destinations/tiktok/src/setup.ts +++ b/packages/web/destinations/tiktok/src/setup.ts @@ -1,66 +1,53 @@ import type { DestinationWeb } from '@walkeros/web-core'; +import type { TTQStub } from './types'; import { getEnv } from '@walkeros/web-core'; /** The TikTok Pixel library, loaded through `loadVendor`. */ export const SCRIPT_SRC = 'https://analytics.tiktok.com/i18n/pixel/events.js'; -interface TTQFunction { - (...args: unknown[]): void; - methods?: string[]; - _i?: Record; - loaded?: boolean; - page?: (...args: unknown[]) => void; - track?: (...args: unknown[]) => void; - identify?: (...args: unknown[]) => void; - enableCookie?: (...args: unknown[]) => void; - disableCookie?: (...args: unknown[]) => void; - load?: (...args: unknown[]) => void; -} +const METHODS = [ + 'page', + 'track', + 'identify', + 'instances', + 'debug', + 'on', + 'off', + 'once', + 'ready', + 'alias', + 'group', + 'enableCookie', + 'disableCookie', +]; /** * Create the `window.ttq` queue object (TikTok's snippet pattern). * Any method call before the real SDK loads is buffered; once the CDN - * script runs, the queue is replayed in order. Idempotent — does - * nothing if ttq already exists on the window. + * script runs, the queue is replayed in order. Idempotent: does nothing if + * ttq already exists on the window (a loaded Pixel, or a mock in tests). */ export function setup(env?: DestinationWeb.Env): void { const { window } = getEnv(env); - const w = window as unknown as { ttq?: TTQFunction }; - if (w.ttq && (w.ttq as TTQFunction).loaded) return; - - // Preserve any existing mock ttq (from env.window.ttq in tests). Only - // construct a new queue if nothing is there. - if (!w.ttq) { - const n = function (...args: unknown[]): void { - (n._i as Record) ||= {}; - ((n._i as { all?: unknown[] }).all ||= []).push(args); - } as TTQFunction; - - n.methods = [ - 'page', - 'track', - 'identify', - 'instances', - 'debug', - 'on', - 'off', - 'once', - 'ready', - 'alias', - 'group', - 'enableCookie', - 'disableCookie', - ]; + if (window.ttq) return; - // Stub out each method to enqueue through the root `n` function. - for (const method of n.methods) { - (n as unknown as Record)[method] = ( - ...args: unknown[] - ) => n(method, ...args); - } + const record = (...args: unknown[]): void => { + (stub._i.all ||= []).push(args); + }; + const stub: TTQStub = Object.assign(record, { + methods: [...METHODS], + _i: {}, + loaded: false, + }); - n._i = {}; - n.loaded = false; - w.ttq = n; - } + // Stub out each method to enqueue through the root function. + window.ttq = Object.assign( + stub, + Object.fromEntries( + METHODS.map((method) => [ + method, + (...args: unknown[]) => stub(method, ...args), + ]), + ), + ); } diff --git a/packages/web/destinations/tiktok/src/types/index.ts b/packages/web/destinations/tiktok/src/types/index.ts index cce4b50ae..d010d6e96 100644 --- a/packages/web/destinations/tiktok/src/types/index.ts +++ b/packages/web/destinations/tiktok/src/types/index.ts @@ -133,10 +133,22 @@ export interface Env extends DestinationWeb.Env { window: { ttq: TTQ; }; - document: { - createElement: (tagName: string) => Element; - head: { appendChild: (node: unknown) => void }; - }; +} + +/** + * The queue `setup()` installs until the Pixel script replaces it: each + * call is kept in `_i.all`. + */ +export type TTQStub = ((...args: unknown[]) => void) & { + methods: string[]; + _i: Record; + loaded: boolean; +}; + +declare global { + interface Window { + ttq?: TTQ | TTQStub; + } } export type Types = CoreDestination.Types; From 3d7f175f29bd0c65ad6969a31219294b02e9a246 Mon Sep 17 00:00:00 2001 From: alexanderkirtzel Date: Wed, 21 Oct 2026 08:50:46 +0000 Subject: [PATCH 30/41] server seat round --- .../core/src/__tests__/batchOutcome.test.ts | 61 +++++++++++++++ .../core/src/__tests__/retryBudget.test.ts | 29 +++++++ packages/server/core/src/batchOutcome.ts | 28 +++++++ packages/server/core/src/index.ts | 2 + packages/server/core/src/retryBudget.ts | 43 +++++++++++ .../src/firehose/__tests__/pushBatch.test.ts | 6 +- .../aws/src/firehose/lib/deliver.ts | 53 +------------ .../aws/src/firehose/pushBatch.ts | 5 +- .../@google-cloud/bigquery-storage.ts | 24 ++++++ .../__tests__/bigquery-storage-mock.d.ts | 4 + .../gcp/src/bigquery/__tests__/index.test.ts | 75 +++++++++++++++++++ .../destinations/gcp/src/bigquery/config.ts | 2 +- .../gcp/src/bigquery/examples/step.ts | 21 ++++++ .../destinations/gcp/src/bigquery/push.ts | 6 +- .../gcp/src/bigquery/pushBatch.ts | 7 +- .../destinations/gcp/src/bigquery/writer.ts | 26 ++++++- .../gcp/src/pubsub/__tests__/init.test.ts | 37 ++++++++- .../destinations/gcp/src/pubsub/push.ts | 25 ++++++- .../hubspot/src/__tests__/batch.test.ts | 49 ++++++++++++ .../server/destinations/hubspot/src/push.ts | 36 ++++++--- .../src/__tests__/push-missing-topic.test.ts | 33 ++++++++ .../server/destinations/kafka/src/config.ts | 5 +- .../server/destinations/kafka/src/index.ts | 14 ++-- .../server/destinations/kafka/src/push.ts | 16 ++++ .../server/destinations/pinterest/src/push.ts | 6 +- .../server/destinations/redis/src/config.ts | 13 ++-- .../server/destinations/redis/src/index.ts | 13 +--- .../sqlite/src/__tests__/init.test.ts | 59 ++++++++++++++- .../server/destinations/sqlite/src/config.ts | 10 +-- .../server/destinations/sqlite/src/index.ts | 17 ++--- .../server/destinations/sqlite/src/push.ts | 65 +++++++++------- .../server/destinations/sqlite/src/setup.ts | 2 +- .../destinations/sqlite/src/types/index.ts | 8 +- .../server/destinations/tiktok/src/push.ts | 7 +- .../stores/fs/src/__tests__/store.test.ts | 12 ++- packages/server/stores/fs/src/store.ts | 7 +- .../ga4/src/__tests__/transformer.test.ts | 5 +- packages/transformers/ga4/src/transformer.ts | 12 ++- 38 files changed, 675 insertions(+), 168 deletions(-) create mode 100644 packages/server/core/src/__tests__/batchOutcome.test.ts create mode 100644 packages/server/core/src/__tests__/retryBudget.test.ts create mode 100644 packages/server/core/src/batchOutcome.ts create mode 100644 packages/server/core/src/retryBudget.ts diff --git a/packages/server/core/src/__tests__/batchOutcome.test.ts b/packages/server/core/src/__tests__/batchOutcome.test.ts new file mode 100644 index 000000000..839fdbf62 --- /dev/null +++ b/packages/server/core/src/__tests__/batchOutcome.test.ts @@ -0,0 +1,61 @@ +import { StepError } from '@walkeros/core'; +import { batchOutcome } from '../batchOutcome'; + +const missing = new StepError('no identity', { code: 'missing-input' }); +const down = new StepError('vendor down', { code: 'transport', status: 503 }); +const rejected = new StepError('bad request', { + code: 'transport', + status: 400, +}); + +describe('batchOutcome', () => { + it('resolves when nothing failed', () => { + expect(batchOutcome(3, [])).toBeUndefined(); + }); + + it('lists the failed entries when some landed', () => { + const failed = [{ index: 1, error: down }]; + expect(batchOutcome(3, failed, new Error('send'))).toEqual({ failed }); + }); + + it.each([ + [ + 'the send error', + [ + { index: 0, error: missing }, + { index: 1, error: down }, + ], + new Error('send failed'), + 'send failed', + ], + [ + 'the first retryable entry error', + [ + { index: 0, error: missing }, + { index: 1, error: rejected }, + { index: 2, error: down }, + ], + undefined, + 'vendor down', + ], + [ + 'the first entry error when none is retryable', + [ + { index: 0, error: missing }, + { index: 1, error: rejected }, + ], + undefined, + 'no identity', + ], + ])('throws %s when nothing landed', (_name, failed, sendError, message) => { + expect(() => batchOutcome(failed.length, failed, sendError)).toThrow( + message, + ); + }); + + it('throws a transport failure for an entry that failed without an error', () => { + expect(() => batchOutcome(1, [{ index: 0 }])).toThrow( + expect.objectContaining({ code: 'transport', retryable: true }), + ); + }); +}); diff --git a/packages/server/core/src/__tests__/retryBudget.test.ts b/packages/server/core/src/__tests__/retryBudget.test.ts new file mode 100644 index 000000000..c109f4587 --- /dev/null +++ b/packages/server/core/src/__tests__/retryBudget.test.ts @@ -0,0 +1,29 @@ +import { retryBudget } from '../retryBudget'; + +describe('retryBudget', () => { + it('gives a quarter per attempt and two retries with its own wait', () => { + expect(retryBudget()).toEqual({ attempt: 2500, retries: 2, wait: 1000 }); + expect(retryBudget(4000)).toEqual({ attempt: 1000, retries: 2, wait: 400 }); + }); + + it.each([ + // An SDK that waits 1 s, then 2 s: three attempts need 12 s. + [10_000, 1], + [12_000, 2], + [1_500, 0], + ])('keeps the retries of fixed waits within %i ms', (timeout, retries) => { + const waits = (count: number) => (1000 * count * (count + 1)) / 2; + expect(retryBudget(timeout, waits).retries).toBe(retries); + }); + + it('hands the configurable wait to the waits it is given', () => { + // The first retry waits the configurable wait, the next one 1 s more. + const waits = (count: number, wait: number) => + count && wait + 500 * count * (count - 1); + expect(retryBudget(10_000, waits)).toEqual({ + attempt: 2500, + retries: 2, + wait: 1000, + }); + }); +}); diff --git a/packages/server/core/src/batchOutcome.ts b/packages/server/core/src/batchOutcome.ts new file mode 100644 index 000000000..da6a90596 --- /dev/null +++ b/packages/server/core/src/batchOutcome.ts @@ -0,0 +1,28 @@ +import type { Destination } from '@walkeros/core'; +import { StepError, toFailureReason } from '@walkeros/core'; + +/** + * The outcome of a batch from its failed entries: nothing failed resolves, + * some failed is a partial outcome, and when nothing landed the batch throws + * one representative error, so the collector counts a whole-batch failure. + * That error is the send error when the send itself failed, else the first + * retryable entry error, else the first (an entry failed without an error + * is an unknown transport failure). + */ +export function batchOutcome( + total: number, + failed: Destination.BatchFailure[], + sendError?: unknown, +): void | Destination.BatchOutcome { + if (!failed.length) return; + if (failed.length < total) return { failed }; + if (sendError !== undefined) throw sendError; + const retryable = failed.find( + ({ error }) => toFailureReason(error).retryable, + ); + const { error } = retryable ?? failed[0]; + throw ( + error ?? + new StepError('No entry of the batch landed', { code: 'transport' }) + ); +} diff --git a/packages/server/core/src/index.ts b/packages/server/core/src/index.ts index c88641eed..ee1847bbe 100644 --- a/packages/server/core/src/index.ts +++ b/packages/server/core/src/index.ts @@ -1,6 +1,8 @@ // Export server-specific utilities export * from './sendServer'; export * from './getHashServer'; +export * from './batchOutcome'; +export * from './retryBudget'; // Export server-specific types export * from './types'; diff --git a/packages/server/core/src/retryBudget.ts b/packages/server/core/src/retryBudget.ts new file mode 100644 index 000000000..8ce7bf41e --- /dev/null +++ b/packages/server/core/src/retryBudget.ts @@ -0,0 +1,43 @@ +import { resolveDestinationTimeout } from '@walkeros/core'; + +/** Share of the destination's timeout one SDK attempt may take. */ +const ATTEMPT_SHARE = 0.25; + +/** Share of the timeout an SDK with a configurable wait waits before a retry. */ +const WAIT_SHARE = 0.1; + +/** At most three attempts: the first one and two retries. */ +const MAX_RETRIES = 2; + +/** How an SDK's own retry fits the destination's timeout. */ +export interface RetryBudget { + /** Time one attempt (one request) may take, in ms. */ + attempt: number; + /** Retries after the first attempt. */ + retries: number; + /** Wait before a retry, for an SDK whose wait is configurable, in ms. */ + wait: number; +} + +/** + * The retry an SDK destination configures, so a send's worst case ends + * before the collector stops waiting (`config.timeout`, 10 s by default): a + * quarter of the timeout per attempt, and at most two retries, as many as fit + * together with the SDK's waits between them. `waited(retries, wait)` is the + * SDK's longest total wait before that many retries, given the configurable + * wait; without it the SDK is set to wait `wait`, a tenth of the timeout, + * before each retry. + */ +export function retryBudget( + timeout?: number, + waited: (retries: number, wait: number) => number = (retries, wait) => + retries * wait, +): RetryBudget { + const total = resolveDestinationTimeout(timeout); + const attempt = Math.floor(total * ATTEMPT_SHARE); + const wait = Math.floor(total * WAIT_SHARE); + let retries = MAX_RETRIES; + while (retries > 0 && (retries + 1) * attempt + waited(retries, wait) > total) + retries--; + return { attempt, retries, wait }; +} diff --git a/packages/server/destinations/aws/src/firehose/__tests__/pushBatch.test.ts b/packages/server/destinations/aws/src/firehose/__tests__/pushBatch.test.ts index 6608021e4..f5fa2d1ad 100644 --- a/packages/server/destinations/aws/src/firehose/__tests__/pushBatch.test.ts +++ b/packages/server/destinations/aws/src/firehose/__tests__/pushBatch.test.ts @@ -74,7 +74,7 @@ describe('Firehose pushBatch outcomes', () => { }); }); - test('mixed causes throw the dominant code, retryable only if all are', async () => { + test('mixed causes throw the first retryable error', async () => { const { result } = await flush(1200, async (_input, call) => { if (call === 2) throw new ResourceNotFoundException({ @@ -90,10 +90,8 @@ describe('Firehose pushBatch outcomes', () => { expect(error).toBeInstanceOf(StepError); expect(error).toMatchObject({ code: 'ServiceUnavailableException', - retryable: false, + retryable: true, }); - expect(String(error)).toContain('1000 ServiceUnavailableException'); - expect(String(error)).toContain('200 ResourceNotFoundException'); }); test('an empty batch makes no call', async () => { diff --git a/packages/server/destinations/aws/src/firehose/lib/deliver.ts b/packages/server/destinations/aws/src/firehose/lib/deliver.ts index e8e2707cc..b1dbd3514 100644 --- a/packages/server/destinations/aws/src/firehose/lib/deliver.ts +++ b/packages/server/destinations/aws/src/firehose/lib/deliver.ts @@ -1,5 +1,4 @@ -import type { Destination, Logger, WalkerOS } from '@walkeros/core'; -import { StepError } from '@walkeros/core'; +import type { Logger, StepError, WalkerOS } from '@walkeros/core'; import { PutRecordBatchCommand } from '@aws-sdk/client-firehose'; import type { Env, Settings } from '../types'; import type { DeliveryTarget } from '../../lib/errors'; @@ -190,53 +189,3 @@ export async function deliver( return failures.sort((a, b) => a.index - b.index); } - -/** - * The error thrown when nothing landed. One shared cause is thrown as is; - * mixed causes give the most frequent code, retryable only if every record - * was, with the counts per code in the message. - */ -export function nothingLanded(failures: readonly DeliveryFailure[]): StepError { - const first = failures[0].error; - if (failures.every(({ error }) => error === first)) return first; - - const byCode = new Map(); - for (const { error } of failures) { - const entry = byCode.get(error.code); - if (entry) entry.count += 1; - else byCode.set(error.code, { count: 1, sample: error }); - } - - let dominant = { count: 0, sample: first }; - for (const entry of byCode.values()) - if (entry.count > dominant.count) dominant = entry; - - const counts = [...byCode.entries()] - .map(([code, { count }]) => `${count} ${code}`) - .join(', '); - - return new StepError( - `${dominant.sample.message} (all ${failures.length} records failed: ${counts})`, - { - code: dominant.sample.code, - status: dominant.sample.status, - retryable: failures.every(({ error }) => error.retryable), - cause: dominant.sample, - }, - ); -} - -/** - * The collector's reading of a delivery: resolve when everything landed, - * the failed indices when some did, a throw when none did. - */ -export function toOutcome( - total: number, - failures: readonly DeliveryFailure[], -): void | Destination.BatchOutcome { - if (failures.length === 0) return; - if (failures.length >= total) throw nothingLanded(failures); - return { - failed: failures.map(({ index, error }) => ({ index, error })), - }; -} diff --git a/packages/server/destinations/aws/src/firehose/pushBatch.ts b/packages/server/destinations/aws/src/firehose/pushBatch.ts index 4db7b5c8f..d7c3c0c71 100644 --- a/packages/server/destinations/aws/src/firehose/pushBatch.ts +++ b/packages/server/destinations/aws/src/firehose/pushBatch.ts @@ -1,6 +1,7 @@ import type { PushBatchFn } from './types'; import { resolveSettings } from './config'; -import { deliver, toOutcome } from './lib/deliver'; +import { batchOutcome } from '@walkeros/server-core'; +import { deliver } from './lib/deliver'; /** * Sends a flushed batch. Resolves when every record landed, returns the @@ -24,7 +25,7 @@ export const pushBatch: PushBatchFn = async (batch, context) => { ) : []; - return toOutcome( + return batchOutcome( batch.entries.length, failures.map(({ index, error }) => ({ index: sending[index].index, diff --git a/packages/server/destinations/gcp/src/bigquery/__mocks__/@google-cloud/bigquery-storage.ts b/packages/server/destinations/gcp/src/bigquery/__mocks__/@google-cloud/bigquery-storage.ts index 92a726013..b182af921 100644 --- a/packages/server/destinations/gcp/src/bigquery/__mocks__/@google-cloud/bigquery-storage.ts +++ b/packages/server/destinations/gcp/src/bigquery/__mocks__/@google-cloud/bigquery-storage.ts @@ -11,6 +11,7 @@ interface MockRowError { let nextAppendRowErrors: MockRowError[] | null = null; let nextAppendThrow: unknown = null; let nextGetResultReject: unknown = null; +let nextAppendStatus: { code: number; message: string } | null = null; let nextCreateStreamConnectionError: unknown = null; /** @@ -72,11 +73,20 @@ class MockJSONWriter { nextAppendRowErrors = null; const queuedReject = nextGetResultReject; nextGetResultReject = null; + const queuedStatus = nextAppendStatus; + nextAppendStatus = null; return { getResult: async () => { // Models a gRPC deadline-exceeded (or any stream error): the append is // accepted but getResult() rejects when the stream's deadline fires. if (queuedReject !== null) throw queuedReject; + // An error status the SDK does not retry resolves with `error` set + // and no rowErrors. + if (queuedStatus !== null) + return { + error: queuedStatus, + writeStream: 'projects/p/datasets/d/tables/t/streams/_default', + }; return { appendResult: { offset: { value: '0' } }, rowErrors: queuedErrors ?? [], @@ -190,6 +200,7 @@ function __resetMockCalls() { nextAppendRowErrors = null; nextAppendThrow = null; nextGetResultReject = null; + nextAppendStatus = null; nextCreateStreamConnectionError = null; lastConnection = null; allConnections.length = 0; @@ -239,6 +250,18 @@ function __setNextGetResultReject(err: unknown): void { nextGetResultReject = err; } +/** + * Test-only: make the next appendRows().getResult() resolve with an error + * status (a quota, a dropped table), as the SDK does for a status it does not + * retry. Auto-resets after one use. + */ +function __setNextAppendStatus(status: { + code: number; + message: string; +}): void { + nextAppendStatus = status; +} + /** * Test-only: queue an error for the next createStreamConnection() call so * tests can simulate openWriter failures (NotFound, INVALID_ARGUMENT, etc.). @@ -260,5 +283,6 @@ export { __setNextAppendRowErrors, __setNextAppendThrow, __setNextGetResultReject, + __setNextAppendStatus, __setNextOpenWriterError, }; diff --git a/packages/server/destinations/gcp/src/bigquery/__tests__/bigquery-storage-mock.d.ts b/packages/server/destinations/gcp/src/bigquery/__tests__/bigquery-storage-mock.d.ts index 9d3f43769..85ffa3694 100644 --- a/packages/server/destinations/gcp/src/bigquery/__tests__/bigquery-storage-mock.d.ts +++ b/packages/server/destinations/gcp/src/bigquery/__tests__/bigquery-storage-mock.d.ts @@ -15,6 +15,10 @@ declare module '@google-cloud/bigquery-storage' { ): void; export function __setNextAppendThrow(err: unknown): void; export function __setNextGetResultReject(err: unknown): void; + export function __setNextAppendStatus(status: { + code: number; + message: string; + }): void; export function __setNextOpenWriterError(err: unknown): void; } diff --git a/packages/server/destinations/gcp/src/bigquery/__tests__/index.test.ts b/packages/server/destinations/gcp/src/bigquery/__tests__/index.test.ts index eb038a5d5..a43d45ad7 100644 --- a/packages/server/destinations/gcp/src/bigquery/__tests__/index.test.ts +++ b/packages/server/destinations/gcp/src/bigquery/__tests__/index.test.ts @@ -13,6 +13,7 @@ import { __setNextAppendRowErrors, __setNextAppendThrow, __setNextGetResultReject, + __setNextAppendStatus, __setNextOpenWriterError, __getLastConnection, __getAllConnections, @@ -701,6 +702,80 @@ describe('Server Destination BigQuery', () => { expect(logger.error).not.toHaveBeenCalled(); }); + test.each([ + ['RESOURCE_EXHAUSTED, a quota', 8, false], + ['NOT_FOUND, a dropped table', 5, false], + ['UNAVAILABLE with write retries off', 14, true], + ])( + 'an append answered with %s fails push, retryable as the SDK retries it', + async (_name, code, retryable) => { + const { writer, writeClient } = await openWriter( + { projectId, datasetId, tableId }, + createMockLogger(), + ); + const config: Config = { + settings: { + client: new BigQuery({ projectId }), + projectId, + datasetId, + tableId, + location: 'EU', + writer, + writeClient, + }, + }; + __setNextAppendStatus({ code, message: 'row 0 quoted here' }); + + const pushing = destination.push( + event, + createMockContext({ config, env: testEnv, id: 'test-bq' }), + ); + await expect(pushing).rejects.toMatchObject({ + name: 'StepError', + code: 'transport', + retryable, + message: `BigQuery append failed (code ${code})`, + }); + }, + ); + + test('an append answered with an error status fails the whole batch', async () => { + if (!destination.pushBatch) throw new Error('pushBatch missing'); + const { writer, writeClient } = await openWriter( + { projectId, datasetId, tableId }, + createMockLogger(), + ); + const config: Config = { + settings: { + client: new BigQuery({ projectId }), + projectId, + datasetId, + tableId, + location: 'EU', + writer, + writeClient, + }, + }; + __setNextAppendStatus({ code: 7, message: 'permission denied' }); + const events = [createEvent(), createEvent()]; + + await expect( + destination.pushBatch( + { + key: 'k', + events, + data: events.map(() => undefined), + entries: events.map((event) => ({ event })), + }, + createMockContext({ config, env: testEnv, id: 'test-bq' }), + ), + ).rejects.toMatchObject({ + code: 'transport', + retryable: false, + message: 'BigQuery append failed (code 7)', + }); + }); + test('a deadline-exceeded getResult rejects pushBatch (whole batch DLQ)', async () => { if (!destination.pushBatch) throw new Error('pushBatch missing'); diff --git a/packages/server/destinations/gcp/src/bigquery/config.ts b/packages/server/destinations/gcp/src/bigquery/config.ts index b962653da..4c69ba57c 100644 --- a/packages/server/destinations/gcp/src/bigquery/config.ts +++ b/packages/server/destinations/gcp/src/bigquery/config.ts @@ -10,7 +10,7 @@ export function getConfig( env: Env | undefined, logger: Logger.Instance, ): Config { - const settings = partialConfig.settings || ({} as Settings); + const settings: Partial = partialConfig.settings || {}; const { projectId, bigquery } = settings; let { client, location, datasetId, tableId } = settings; diff --git a/packages/server/destinations/gcp/src/bigquery/examples/step.ts b/packages/server/destinations/gcp/src/bigquery/examples/step.ts index 33843fe6a..1de49f960 100644 --- a/packages/server/destinations/gcp/src/bigquery/examples/step.ts +++ b/packages/server/destinations/gcp/src/bigquery/examples/step.ts @@ -62,3 +62,24 @@ export const silentRule: Flow.StepExample = { mapping: { silent: true }, out: [], }; + +/** + * A failed append: the Storage Write call fails, so the row is not written + * and the push fails, retryable, instead of counting as delivered. An append + * the server answers with an error status (a quota, a dropped table) fails + * the same way, retryable only for the statuses the SDK retries itself. + */ +export const appendFailed: Flow.StepExample = { + public: false, + description: + 'A failed Storage Write append fails the push, retryable, instead of counting as delivered.', + in: getEvent('page view', { timestamp: 1700001102 }), + respond: { error: 'network' }, + out: [ + [ + 'appendRows', + [expectedRow(getEvent('page view', { timestamp: 1700001102 }))], + ], + ], + outcome: { status: 'failed', code: 'transport', retryable: true }, +}; diff --git a/packages/server/destinations/gcp/src/bigquery/push.ts b/packages/server/destinations/gcp/src/bigquery/push.ts index fe2b419e6..09803d9ca 100644 --- a/packages/server/destinations/gcp/src/bigquery/push.ts +++ b/packages/server/destinations/gcp/src/bigquery/push.ts @@ -1,7 +1,7 @@ import type { PushFn } from './types'; import { isObject, StepError } from '@walkeros/core'; import { eventToRow } from './eventToRow'; -import { ensureWriter } from './writer'; +import { appendError, ensureWriter } from './writer'; // A silent rule has no row of its own: nothing to append. export const push: PushFn = async function (event, context) { @@ -52,6 +52,10 @@ const append: PushFn = async function (event, { config, data, logger }) { }); } + // The server answered the append with an error status: nothing landed. + const failed = appendError(result); + if (failed) throw failed; + if (result.rowErrors && result.rowErrors.length > 0) // The server rejected the row itself: a retry sends the same row. Its // message can quote the row, so only the code is named. diff --git a/packages/server/destinations/gcp/src/bigquery/pushBatch.ts b/packages/server/destinations/gcp/src/bigquery/pushBatch.ts index aa176e613..4d262e28e 100644 --- a/packages/server/destinations/gcp/src/bigquery/pushBatch.ts +++ b/packages/server/destinations/gcp/src/bigquery/pushBatch.ts @@ -1,7 +1,7 @@ import type { PushBatchFn } from './types'; import { isObject, StepError } from '@walkeros/core'; import { eventToRow } from './eventToRow'; -import { ensureWriter } from './writer'; +import { appendError, ensureWriter } from './writer'; /** * Batched push using a single appendRows call. @@ -63,6 +63,11 @@ export const pushBatch: PushBatchFn = async (batch, { config, logger }) => { }); } + // The server answered the append with an error status: no row landed, the + // whole batch fails. + const failed = appendError(result); + if (failed) throw failed; + if (result.rowErrors && result.rowErrors.length > 0) { // Report per-row failures so the collector DLQs only the failed rows and // counts the succeeded rows as delivered. `re.index` is the row's position diff --git a/packages/server/destinations/gcp/src/bigquery/writer.ts b/packages/server/destinations/gcp/src/bigquery/writer.ts index 9526f6819..68e1cd01c 100644 --- a/packages/server/destinations/gcp/src/bigquery/writer.ts +++ b/packages/server/destinations/gcp/src/bigquery/writer.ts @@ -48,7 +48,31 @@ export interface WriteConnection { } /** The fields of an append response that push and pushBatch read. */ -export type AppendOutcome = Pick; +export type AppendOutcome = Pick< + AppendResponse, + 'rowErrors' | 'appendResult' | 'error' +>; + +/** + * The gRPC statuses the Storage Write SDK retries itself (its stream + * connection's `isRetryableError`): ABORTED, UNAVAILABLE, CANCELLED, + * INTERNAL, DEADLINE_EXCEEDED. + */ +const RETRYABLE_STATUS = [10, 14, 1, 13, 4]; + +/** + * The failure of an append the server answered with an error status: a + * quota, a dropped table, a lost permission. Nothing of it was written. The + * status message can quote the rows, so only its code is named. + */ +export function appendError(outcome: AppendOutcome): StepError | undefined { + if (!outcome.error) return undefined; + const status = outcome.error.code ?? undefined; + return new StepError(`BigQuery append failed (code ${status ?? 'unknown'})`, { + code: 'transport', + retryable: status !== undefined && RETRYABLE_STATUS.includes(status), + }); +} /** The row writer surface push and pushBatch use. */ export interface RowWriter { diff --git a/packages/server/destinations/gcp/src/pubsub/__tests__/init.test.ts b/packages/server/destinations/gcp/src/pubsub/__tests__/init.test.ts index 6e8e419ee..b7e8df628 100644 --- a/packages/server/destinations/gcp/src/pubsub/__tests__/init.test.ts +++ b/packages/server/destinations/gcp/src/pubsub/__tests__/init.test.ts @@ -1,9 +1,17 @@ jest.mock('@google-cloud/pubsub'); import { PubSub } from '@google-cloud/pubsub'; -import { __getMockCalls, __resetMockCalls } from '@google-cloud/pubsub'; +import { + __getMockCalls, + __resetMockCalls, + __setTopicHarness, +} from '@google-cloud/pubsub'; import type { Destination, Env, InitSettings, Settings } from '../types'; -import { createMockContext, createMockLogger } from '@walkeros/core'; +import { + createEvent, + createMockContext, + createMockLogger, +} from '@walkeros/core'; import type { MockLogger } from '@walkeros/core'; interface ResolvedConfigShape { @@ -159,4 +167,29 @@ describe('Server Destination Pub/Sub init', () => { }), ).resolves.toBeUndefined(); }); + + test.each([ + ['INVALID_ARGUMENT, an oversize message', 3, { retryable: false }], + ['UNAVAILABLE', 14, { retryable: true }], + ['a REST 400', 400, { status: 400, retryable: false }], + ])( + 'a publish failing with %s is retryable as the SDK retries it', + async (_name, code, expected) => { + const config = await callInit(destination, { + projectId: 'p', + topic: 'events', + }); + if (!isResolvedConfig(config) || !destination.push) + throw new Error('init returned no config'); + __setTopicHarness({ + publishError: Object.assign(new Error('rejected'), { code }), + }); + await expect( + destination.push( + createEvent(), + createMockContext({ config, id: 'test-pubsub' }), + ), + ).rejects.toMatchObject({ code: 'transport', ...expected }); + }, + ); }); diff --git a/packages/server/destinations/gcp/src/pubsub/push.ts b/packages/server/destinations/gcp/src/pubsub/push.ts index 729e9955e..ca3ad2694 100644 --- a/packages/server/destinations/gcp/src/pubsub/push.ts +++ b/packages/server/destinations/gcp/src/pubsub/push.ts @@ -9,6 +9,17 @@ const UNAUTHENTICATED_GRPC = 16; const PERMISSION_DENIED_HTTP = 403; const UNAUTHENTICATED_HTTP = 401; +/** + * The gRPC codes the Pub/Sub SDK itself retries a publish on (CANCELLED, + * UNKNOWN, DEADLINE_EXCEEDED, RESOURCE_EXHAUSTED, ABORTED, INTERNAL, + * UNAVAILABLE, its publisher client config). Any other gRPC code, such as + * INVALID_ARGUMENT for an oversize message, fails the same way again. + */ +const RETRYABLE_GRPC = [1, 2, 4, 8, 10, 13, 14]; + +/** gRPC codes run from 0 to 16; a larger one is the HTTP status of a REST call. */ +const MAX_GRPC_CODE = 16; + interface ErrorWithCode { code: number; } @@ -193,7 +204,7 @@ const publish: PushFn = async function ( topicHandle.resumePublishing(orderingKey); } catch (resumeErr) { logger.debug('Pub/Sub resumePublishing failed', { - orderingKey, + topic: topicName, error: resumeErr instanceof Error ? resumeErr.message : String(resumeErr), }); @@ -215,8 +226,18 @@ const publish: PushFn = async function ( `service account roles/pubsub.publisher on this topic.`, { code: 'config', cause: err }, ); - throw new StepError('Pub/Sub publish failed', { + // Retryable as the SDK's own retry decides: by the gRPC code, or by the + // HTTP status of a REST call; without a code, a transport failure. + if (!hasNumericCode(err)) + throw new StepError('Pub/Sub publish failed', { + code: 'transport', + cause: err, + }); + throw new StepError(`Pub/Sub publish failed (code ${err.code})`, { code: 'transport', + ...(err.code > MAX_GRPC_CODE + ? { status: err.code } + : { retryable: RETRYABLE_GRPC.includes(err.code) }), cause: err, }); } diff --git a/packages/server/destinations/hubspot/src/__tests__/batch.test.ts b/packages/server/destinations/hubspot/src/__tests__/batch.test.ts index c88389c8e..b514e804a 100644 --- a/packages/server/destinations/hubspot/src/__tests__/batch.test.ts +++ b/packages/server/destinations/hubspot/src/__tests__/batch.test.ts @@ -54,3 +54,52 @@ describe('hubspot config.batch', () => { await collector.command('shutdown'); }); }); + +describe('hubspot declared skips', () => { + it('says at DEBUG which mapping a declared skip found nothing for', async () => { + const lines: Array<[string, unknown]> = []; + const { collector } = await startFlow({ + logger: { + level: 'DEBUG', + handler: (_level, message, context) => lines.push([message, context]), + }, + destinations: { + hubspot: { + code: destinationHubspot, + config: { + settings: { accessToken: 'pat', eventNamePrefix: 'pe1_' }, + batch: undefined, + mapping: { + user: { + login: { + silent: true, + settings: { identify: { map: { email: 'data.email' } } }, + }, + }, + }, + }, + env: { client: clientRecording([]) }, + }, + }, + }); + + await collector.push( + createEvent({ + name: 'user login', + user: { email: 'a@x.com' }, + data: {}, + }), + ); + + expect( + lines.filter(([message]) => message.includes('resolved to nothing')), + ).toEqual([ + [ + 'HubSpot skipped identify: identify resolved to nothing', + expect.objectContaining({ event: 'user login' }), + ], + ]); + + await collector.command('shutdown'); + }); +}); diff --git a/packages/server/destinations/hubspot/src/push.ts b/packages/server/destinations/hubspot/src/push.ts index a4ad2f501..d729ea7d4 100644 --- a/packages/server/destinations/hubspot/src/push.ts +++ b/packages/server/destinations/hubspot/src/push.ts @@ -10,6 +10,7 @@ import type { } from './types'; import type { Collector, Destination, Logger, WalkerOS } from '@walkeros/core'; import { getMappingValue, isObject, isString, StepError } from '@walkeros/core'; +import { batchOutcome } from '@walkeros/server-core'; /** The HubSpot batch API takes at most this many events per request. */ const MAX_BATCH = 500; @@ -90,7 +91,10 @@ async function prepare( const resolved = await getMappingValue(event, identifyMapping, { collector, }); - if (isObject(resolved)) await applyIdentify(sdk, resolved, state, logger); + const email = isObject(resolved) ? resolved.email : undefined; + if (isObject(resolved) && isString(email) && email) + await applyIdentify(sdk, resolved, email, state); + else skipped(logger, event, 'identify', 'identify'); } // 3. The event (unless silent: true) @@ -193,10 +197,7 @@ export const pushBatch: PushBatchFn = async function ( } } - // Nothing landed: a whole-batch failure, which the breaker counts. - if (failed.length === batch.entries.length && failed.length) - throw sendError ?? failed[0].error; - if (failed.length) return { failed }; + return batchOutcome(batch.entries.length, failed, sendError); }; /** Settings as init resolved them, the required ones present. */ @@ -246,18 +247,29 @@ function hashProperties( } } +/** + * A configured mapping that resolved to nothing skips its call: a declared + * skip, said once at DEBUG with the event and the mapping key, never the + * value, so a wrong mapping path does not look like an event without data. + * The contact upsert needs the identify mapping's email. + */ +function skipped( + logger: Logger.Instance, + event: WalkerOS.Event, + call: string, + key: string, +): void { + logger.debug(`HubSpot skipped ${call}: ${key} resolved to nothing`, { + event: event.name, + }); +} + async function applyIdentify( sdk: HubSpotClientMock, resolved: WalkerOS.AnyObject, + identifyEmail: string, state: RuntimeState, - logger: Logger.Instance, ): Promise { - const identifyEmail = isString(resolved.email) ? resolved.email : undefined; - if (!identifyEmail) { - logger.warn('HubSpot identify requires email; skipping contact upsert'); - return; - } - const rawProperties = isObject(resolved.properties) ? toStringProperties(resolved.properties) : undefined; diff --git a/packages/server/destinations/kafka/src/__tests__/push-missing-topic.test.ts b/packages/server/destinations/kafka/src/__tests__/push-missing-topic.test.ts index e87ae87a1..920ab10c1 100644 --- a/packages/server/destinations/kafka/src/__tests__/push-missing-topic.test.ts +++ b/packages/server/destinations/kafka/src/__tests__/push-missing-topic.test.ts @@ -90,6 +90,39 @@ describe('push: actionable error when topic missing', () => { expect(logger.error).not.toHaveBeenCalled(); }); + it.each([ + [ + 'an error no retry fixes (MESSAGE_TOO_LARGE)', + Object.assign(new Error('message too large'), { + name: 'KafkaJSNonRetriableError', + retriable: false, + }), + false, + ], + [ + 'retries run out on a retried error', + Object.assign(new Error('broker not available'), { + name: 'KafkaJSNumberOfRetriesExceeded', + retriable: false, + retryCount: 5, + }), + true, + ], + ])('takes kafkajs retriable for %s', async (_name, error, retryable) => { + const producer = makeProducer(() => Promise.reject(error)); + const context = createMockContext({ + config: makeConfig(producer), + logger: createMockLogger(), + id: 'kafka', + }); + + await expect(push(getEvent(), context)).rejects.toMatchObject({ + code: 'transport', + retryable, + cause: error, + }); + }); + it('connects once, at the first push', async () => { const connect = jest.fn().mockResolvedValue(undefined); const producer = { ...makeProducer(() => Promise.resolve([])), connect }; diff --git a/packages/server/destinations/kafka/src/config.ts b/packages/server/destinations/kafka/src/config.ts index a7ddb4300..40e085ecf 100644 --- a/packages/server/destinations/kafka/src/config.ts +++ b/packages/server/destinations/kafka/src/config.ts @@ -19,9 +19,8 @@ const COMPRESSION_FALLBACK: Record = { }; export function getConfig(partialConfig: PartialConfig = {}): Config { - const raw = (partialConfig.settings ?? {}) as Partial; - const kafka: Partial = - raw.kafka ?? ({} as Partial); + const raw: Partial = partialConfig.settings ?? {}; + const kafka: Partial = raw.kafka ?? {}; const { brokers, topic } = kafka; if (!brokers || brokers.length === 0 || !topic) { diff --git a/packages/server/destinations/kafka/src/index.ts b/packages/server/destinations/kafka/src/index.ts index 7bb0876ff..26d730a37 100644 --- a/packages/server/destinations/kafka/src/index.ts +++ b/packages/server/destinations/kafka/src/index.ts @@ -1,6 +1,5 @@ import type { Destination, - Settings, Env, KafkaClientMock, KafkaClientConfig, @@ -24,8 +23,7 @@ export const destinationKafka: Destination = { async init({ config: partialConfig, env }) { const config = getConfig(partialConfig); - const settings = config.settings as Settings; - const kafka = settings.kafka; + const kafka = config.settings.kafka; // Skip creation if a producer has already been wired in (testing). if (kafka._producer) return config; @@ -82,14 +80,14 @@ export const destinationKafka: Destination = { }, async destroy({ config }) { - const settings = config?.settings as Settings | undefined; - const producer = settings?.kafka?._producer; - if (producer) { + const kafka = config?.settings?.kafka; + const producer = kafka?._producer; + if (kafka && producer) { try { await producer.disconnect(); } finally { - settings.kafka._producer = undefined; - settings.kafka._connected = undefined; + kafka._producer = undefined; + kafka._connected = undefined; } } }, diff --git a/packages/server/destinations/kafka/src/push.ts b/packages/server/destinations/kafka/src/push.ts index 3d8e87cdf..7fcb836f8 100644 --- a/packages/server/destinations/kafka/src/push.ts +++ b/packages/server/destinations/kafka/src/push.ts @@ -26,6 +26,21 @@ function isUnknownTopicError(err: unknown): boolean { return err.code === UNKNOWN_TOPIC_CODE || err.type === UNKNOWN_TOPIC_TYPE; } +/** + * kafkajs's own classification of a failed send. Its errors carry + * `retriable`; its retry wraps an error it gave up on: one no retry fixes + * (MESSAGE_TOO_LARGE) stays not retriable, while running out of retries on + * one it does retry (KafkaJSNumberOfRetriesExceeded, which carries + * `retryCount`) can succeed at a later send. Undefined for any other error. + */ +function kafkaRetriable(error: unknown): boolean | undefined { + if (typeof error !== 'object' || error === null) return undefined; + const retriable: unknown = Reflect.get(error, 'retriable'); + if (typeof retriable !== 'boolean') return undefined; + if (typeof Reflect.get(error, 'retryCount') === 'number') return true; + return retriable; +} + /** * The producer's connection, opened once at the first push; a failed * connect fails that push and is tried again at the next one. @@ -124,6 +139,7 @@ const send: PushFn = async function ( ); throw new StepError('Kafka push failed', { code: 'transport', + retryable: kafkaRetriable(error), cause: error, }); } diff --git a/packages/server/destinations/pinterest/src/push.ts b/packages/server/destinations/pinterest/src/push.ts index d35566828..6d5e28274 100644 --- a/packages/server/destinations/pinterest/src/push.ts +++ b/packages/server/destinations/pinterest/src/push.ts @@ -18,6 +18,10 @@ const send: PushFn = async function ( event, { config, data, ingest, env, logger, collector }, ) { + if (!config.settings) + throw new StepError('Pinterest settings missing, init() not run', { + code: 'not-ready', + }); const { accessToken, adAccountId, @@ -29,7 +33,7 @@ const send: PushFn = async function ( partner_name, ip, userAgent, - } = config.settings!; + } = config.settings; const eventData = isObject(data) ? data : {}; const configData = config.data diff --git a/packages/server/destinations/redis/src/config.ts b/packages/server/destinations/redis/src/config.ts index 6a40f6f34..95e6a6788 100644 --- a/packages/server/destinations/redis/src/config.ts +++ b/packages/server/destinations/redis/src/config.ts @@ -8,9 +8,8 @@ import type { import { isObject, StepError } from '@walkeros/core'; export function getConfig(partialConfig: PartialConfig = {}): Config { - const raw = (partialConfig.settings ?? {}) as Partial; - const redis: Partial = - raw.redis ?? ({} as Partial); + const raw: Partial = partialConfig.settings ?? {}; + const redis: Partial = raw.redis ?? {}; if (!redis.streamKey) throw new StepError('Config settings redis.streamKey missing', { @@ -29,7 +28,9 @@ export function getConfig(partialConfig: PartialConfig = {}): Config { } export function isRedisEnv(env: unknown): env is Env { - if (!isObject(env)) return false; - const maybe = env as { Redis?: { Client?: unknown } }; - return typeof maybe.Redis?.Client === 'function'; + return ( + isObject(env) && + isObject(env.Redis) && + typeof env.Redis.Client === 'function' + ); } diff --git a/packages/server/destinations/redis/src/index.ts b/packages/server/destinations/redis/src/index.ts index 54cf6d17c..5af1d70c7 100644 --- a/packages/server/destinations/redis/src/index.ts +++ b/packages/server/destinations/redis/src/index.ts @@ -1,7 +1,5 @@ import type { Destination, - Settings, - Env, RedisClientMock, RedisClientConstructor, RedisClientOptions, @@ -20,8 +18,7 @@ export const destinationRedis: Destination = { async init({ config: partialConfig, env, reportError }) { const config = getConfig(partialConfig); - const settings = config.settings as Settings; - const redis = settings.redis; + const redis = config.settings.redis; // Skip creation if a client has already been wired in (testing). if (redis._client) return config; @@ -29,10 +26,7 @@ export const destinationRedis: Destination = { let Constructor: RedisClientConstructor | undefined; // Prefer env-injected constructor (testing, dependency injection). - if (isRedisEnv(env)) { - const envTyped = env as Env; - Constructor = envTyped.Redis?.Client; - } + if (isRedisEnv(env)) Constructor = env.Redis?.Client; // Production path: load real ioredis SDK. if (!Constructor) { @@ -79,8 +73,7 @@ export const destinationRedis: Destination = { }, async destroy({ config }) { - const settings = config?.settings as Settings | undefined; - const redis = settings?.redis; + const redis = config?.settings?.redis; if (!redis) return; const client = redis._client; diff --git a/packages/server/destinations/sqlite/src/__tests__/init.test.ts b/packages/server/destinations/sqlite/src/__tests__/init.test.ts index 17b8d2f76..048e5aac3 100644 --- a/packages/server/destinations/sqlite/src/__tests__/init.test.ts +++ b/packages/server/destinations/sqlite/src/__tests__/init.test.ts @@ -1,4 +1,8 @@ -import { createEvent, createMockContext } from '@walkeros/core'; +import { + createEvent, + createMockContext, + createMockLogger, +} from '@walkeros/core'; import { destinationSQLite } from '..'; import type { Settings, SqliteClient, SqliteClientFactory } from '../types'; @@ -330,4 +334,57 @@ describe('push error handling', () => { expect(logger.error).not.toHaveBeenCalled(); }); + + /** A client whose database holds only the events table. */ + function eventsOnly(run: () => Promise): SqliteClient { + return { + execute: () => Promise.resolve(), + prepare: () => run, + query: (_sql, args) => + Promise.resolve(args?.[0] === 'events' ? [{ name: 'events' }] : []), + close: () => Promise.resolve(), + }; + } + + it('fails a constraint violation as not retryable', async () => { + const violation = Object.assign(new Error('UNIQUE constraint failed'), { + code: 'SQLITE_CONSTRAINT_UNIQUE', + }); + const config = await initOf({ + config: { settings: { sqlite: { url: ':memory:' } } }, + env: { client: eventsOnly(() => Promise.reject(violation)) }, + logger: createMockLogger(), + id: 'sqlite', + collector: createMockContext().collector, + }); + await expect(firstPush(config)).rejects.toMatchObject({ + code: 'transport', + retryable: false, + cause: violation, + }); + }); + + it('fails a missing rule table with config', async () => { + const config = await initOf({ + config: { settings: { sqlite: { url: ':memory:' } } }, + env: { client: eventsOnly(() => Promise.resolve()) }, + logger: createMockLogger(), + id: 'sqlite', + collector: createMockContext().collector, + }); + if (!config) throw new Error('init returned no config'); + await expect( + destinationSQLite.push( + createEvent({ name: 'page view' }), + createMockContext({ + config, + id: 'sqlite', + rule: { settings: { table: 'orders' } }, + }), + ), + ).rejects.toMatchObject({ + code: 'config', + message: expect.stringContaining('"orders"'), + }); + }); }); diff --git a/packages/server/destinations/sqlite/src/config.ts b/packages/server/destinations/sqlite/src/config.ts index 91a9ca676..85b5e3a2f 100644 --- a/packages/server/destinations/sqlite/src/config.ts +++ b/packages/server/destinations/sqlite/src/config.ts @@ -12,9 +12,8 @@ export function getConfig( partialConfig: PartialConfig = {}, logger: Logger.Instance, ): Config { - const raw = (partialConfig.settings ?? {}) as Partial; - const sqlite: Partial = - raw.sqlite ?? ({} as Partial); + const raw: Partial = partialConfig.settings ?? {}; + const sqlite: Partial = raw.sqlite ?? {}; if (!sqlite.url) throw new StepError('Config settings sqlite.url missing', { @@ -65,8 +64,5 @@ export function getConfig( export function isSqliteEnv(env: unknown): env is Env { if (!isObject(env)) return false; - const maybe = env as { SqliteDriver?: unknown; client?: unknown }; - return ( - typeof maybe.SqliteDriver === 'function' || isObject(maybe.client ?? null) - ); + return typeof env.SqliteDriver === 'function' || isObject(env.client ?? null); } diff --git a/packages/server/destinations/sqlite/src/index.ts b/packages/server/destinations/sqlite/src/index.ts index 05067b3a1..9468db084 100644 --- a/packages/server/destinations/sqlite/src/index.ts +++ b/packages/server/destinations/sqlite/src/index.ts @@ -1,4 +1,4 @@ -import type { Destination, Env, Settings, SqliteClient } from './types'; +import type { Destination, SqliteClient } from './types'; import { getConfig, isSqliteEnv } from './config'; import { createClientFromSettings } from './drivers'; import { StepError } from '@walkeros/core'; @@ -40,11 +40,10 @@ export const destinationSQLite: Destination = { // Prefer env-injected client / factory (tests, dependency injection). if (isSqliteEnv(env)) { - const envTyped = env as Env; - if (envTyped.client) { - client = envTyped.client; - } else if (envTyped.SqliteDriver) { - client = await envTyped.SqliteDriver(sqlite.url, sqlite.authToken); + if (env.client) { + client = env.client; + } else if (env.SqliteDriver) { + client = await env.SqliteDriver(sqlite.url, sqlite.authToken); } } @@ -63,7 +62,7 @@ export const destinationSQLite: Destination = { // The table check and the prepared insert run at the first push, so // init does no query that can fail. sqlite._client = client; - sqlite._ownedClient = !(isSqliteEnv(env) && (env as Env).client); + sqlite._ownedClient = !(isSqliteEnv(env) && env.client); return config; }, @@ -73,8 +72,7 @@ export const destinationSQLite: Destination = { }, async destroy({ config }) { - const settings = config?.settings as Settings | undefined; - const sqlite = settings?.sqlite; + const sqlite = config?.settings?.sqlite; if (!sqlite) return; const client = sqlite._client; @@ -84,7 +82,6 @@ export const destinationSQLite: Destination = { await client.close(); } finally { sqlite._client = undefined; - sqlite._runInsert = undefined; sqlite._ready = undefined; sqlite._ownedClient = undefined; } diff --git a/packages/server/destinations/sqlite/src/push.ts b/packages/server/destinations/sqlite/src/push.ts index 8ad70b986..5bfc2cbf6 100644 --- a/packages/server/destinations/sqlite/src/push.ts +++ b/packages/server/destinations/sqlite/src/push.ts @@ -6,17 +6,18 @@ import { tableExists } from './setup'; type RunInsert = (args: ReadonlyArray) => Promise; /** - * Check the table and prepare the insert, once per client: the legacy - * `schema: 'auto'` creates it, the default path probes it and fails with - * the setup hint when it is missing. A failed attempt is retried at the next - * push. + * Check a table and prepare its insert: the legacy `schema: 'auto'` creates + * the table, the default path probes it and fails with `config` when it is + * missing (with the setup hint for the destination's own table, which setup + * creates). */ async function prepareTable( sqlite: SqliteSettings, client: SqliteClient, id: string, + table: string, ): Promise { - const table = sqlite.table ?? 'events'; + const own = table === (sqlite.table ?? 'events'); if (sqlite._legacyAutoCreate) { try { await client.execute(buildCreateTableSql(table)); @@ -38,19 +39,13 @@ async function prepareTable( } if (!exists) throw new StepError( - `SQLite table "${table}" not found in ${sqlite.url}. ` + - `Run "walkeros setup destination.${id}" to create it.`, + own + ? `SQLite table "${table}" not found in ${sqlite.url}. ` + + `Run "walkeros setup destination.${id}" to create it.` + : `SQLite table "${table}" (rule settings.table) not found in ${sqlite.url}.`, { code: 'config' }, ); } - return prepare(client, table); -} - -/** A prepared insert; the driver call may answer synchronously or not. */ -async function prepare( - client: SqliteClient, - table: string, -): Promise { try { return await client.prepare(buildInsertSql(table)); } catch (err) { @@ -61,20 +56,37 @@ async function prepare( } } -/** The insert of the destination's table, prepared at the first push. */ +/** + * The insert of a table, checked and prepared at its first push, once per + * table. A failed attempt is retried at the next push. + */ function insertOf( sqlite: SqliteSettings, client: SqliteClient, id: string, + table: string, ): Promise { - if (!sqlite._ready) { - const ready = prepareTable(sqlite, client, id); - ready.catch(() => { - if (sqlite._ready === ready) sqlite._ready = undefined; + const tables = (sqlite._ready ??= new Map()); + let ready = tables.get(table); + if (!ready) { + const preparing = prepareTable(sqlite, client, id, table); + preparing.catch(() => { + if (tables.get(table) === preparing) tables.delete(table); }); - sqlite._ready = ready; + tables.set(table, preparing); + ready = preparing; } - return sqlite._ready; + return ready; +} + +/** + * A constraint violation (a duplicate id under the setup's unique index, a + * NOT NULL column) fails the same way on every attempt: not retryable. + */ +function isConstraintError(error: unknown): boolean { + if (!(error instanceof Error)) return false; + const code: unknown = Reflect.get(error, 'code'); + return typeof code === 'string' && code.startsWith('SQLITE_CONSTRAINT'); } // A silent rule has no INSERT of its own: nothing to write. @@ -93,13 +105,9 @@ const insert: PushFn = async function (event, { config, rule, logger, id }) { const ruleSettings = rule?.settings ?? {}; const ruleTable = isString(ruleSettings.table) ? ruleSettings.table : ''; - // If the rule overrides the table, we can't reuse the cached prepared statement - // (which was bound to the default table). Build an ad-hoc prepared run. + // A rule may write to its own table; each table is checked once. const effectiveTable = ruleTable || sqlite.table || 'events'; - const runInsert = - ruleTable && ruleTable !== sqlite.table - ? await prepare(client, effectiveTable) - : (sqlite._runInsert ?? (await insertOf(sqlite, client, id))); + const runInsert = await insertOf(sqlite, client, id, effectiveTable); const row = eventToRow(event); @@ -114,6 +122,7 @@ const insert: PushFn = async function (event, { config, rule, logger, id }) { // The collector reports it; the driver's own message stays the cause. throw new StepError('SQLite INSERT failed', { code: 'transport', + retryable: !isConstraintError(error), cause: error, }); } diff --git a/packages/server/destinations/sqlite/src/setup.ts b/packages/server/destinations/sqlite/src/setup.ts index 9fbc6efc0..f40a6ef4b 100644 --- a/packages/server/destinations/sqlite/src/setup.ts +++ b/packages/server/destinations/sqlite/src/setup.ts @@ -158,7 +158,7 @@ export const setup = async ( const table = sqlite.table ?? 'events'; // Open transient client (or reuse env-injected one). We do NOT touch - // sqlite._client / sqlite._runInsert here, setup is a separate lifecycle. + // sqlite._client / sqlite._ready here, setup is a separate lifecycle. let client: SqliteClient; let owned = true; if (isSqliteEnv(env) && env.client) { diff --git a/packages/server/destinations/sqlite/src/types/index.ts b/packages/server/destinations/sqlite/src/types/index.ts index 4f4cdbba9..9319e109b 100644 --- a/packages/server/destinations/sqlite/src/types/index.ts +++ b/packages/server/destinations/sqlite/src/types/index.ts @@ -62,9 +62,11 @@ export interface SqliteSettings { // Runtime, set during init/setup, not user-facing. _client?: SqliteClient; - _runInsert?: (args: ReadonlyArray) => Promise; - /** The table check and insert preparation, run once at the first push. */ - _ready?: Promise<(args: ReadonlyArray) => Promise>; + /** Each table's check and insert preparation, run once at its first push. */ + _ready?: Map< + string, + Promise<(args: ReadonlyArray) => Promise> + >; _ownedClient?: boolean; /** * Internal flag set by the migration shim. When true, the first push runs diff --git a/packages/server/destinations/tiktok/src/push.ts b/packages/server/destinations/tiktok/src/push.ts index d5f000822..0750ba0bc 100644 --- a/packages/server/destinations/tiktok/src/push.ts +++ b/packages/server/destinations/tiktok/src/push.ts @@ -3,7 +3,6 @@ import type { EventContext, PushFn, RequestBody, - Settings, UserData, Env, } from './types'; @@ -36,7 +35,11 @@ const send: PushFn = async function ( partner_name, ip, userAgent, - } = config.settings as Settings; + } = config.settings ?? {}; + if (!accessToken || !pixelCode) + throw new StepError('TikTok settings missing, init() not run', { + code: 'not-ready', + }); const eventData = isObject(data) ? data : {}; const configData = config.data diff --git a/packages/server/stores/fs/src/__tests__/store.test.ts b/packages/server/stores/fs/src/__tests__/store.test.ts index 542b567e9..065b98a7f 100644 --- a/packages/server/stores/fs/src/__tests__/store.test.ts +++ b/packages/server/stores/fs/src/__tests__/store.test.ts @@ -75,13 +75,19 @@ describe('FsStore', () => { expect(await store.get('missing')).toBeUndefined(); }); - it('throws a StepError for a read that fails, not a miss', async () => { + it('returns undefined for a key that names a directory', async () => { const store = await createStore(tmpDir); await fs.mkdir(path.join(tmpDir, 'folder')); - await expect(store.get('folder')).rejects.toMatchObject({ + expect(await store.get('folder')).toBeUndefined(); + }); + + it('throws a StepError for a read that fails, not a miss', async () => { + const store = await createStore(tmpDir); + await fs.symlink('loop', path.join(tmpDir, 'loop')); + await expect(store.get('loop')).rejects.toMatchObject({ name: 'StepError', code: 'transport', - message: 'fs store read failed (EISDIR)', + message: 'fs store read failed (ELOOP)', }); }); diff --git a/packages/server/stores/fs/src/store.ts b/packages/server/stores/fs/src/store.ts index 97b5cc8ec..9cf91ad86 100644 --- a/packages/server/stores/fs/src/store.ts +++ b/packages/server/stores/fs/src/store.ts @@ -20,10 +20,13 @@ function errorCode(err: unknown): string | undefined { return typeof err.code === 'string' ? err.code : undefined; } -/** The file (or a directory on its path) does not exist: a miss. */ +/** + * No file is stored at the key: it does not exist, a directory on its path + * is a file, or the key names a directory. A miss. + */ function isMissing(err: unknown): boolean { const code = errorCode(err); - return code === 'ENOENT' || code === 'ENOTDIR'; + return code === 'ENOENT' || code === 'ENOTDIR' || code === 'EISDIR'; } /** A failed file operation as a StepError, its message without the key. */ diff --git a/packages/transformers/ga4/src/__tests__/transformer.test.ts b/packages/transformers/ga4/src/__tests__/transformer.test.ts index becf273b7..3012cfeff 100644 --- a/packages/transformers/ga4/src/__tests__/transformer.test.ts +++ b/packages/transformers/ga4/src/__tests__/transformer.test.ts @@ -166,8 +166,9 @@ describe('transformerGa4', () => { }); const result = await (await transformerGa4(ctx)).push(emptyEvent, ctx); expect(result).toBe(false); - expect(logger.warn).toHaveBeenCalledTimes(1); - expect(String(logger.warn.mock.calls[0][0])).toContain( + expect(logger.error).toHaveBeenCalledTimes(1); + expect(logger.warn).not.toHaveBeenCalled(); + expect(String(logger.error.mock.calls[0][0])).toContain( `${count} events exceed maxEvents ${maxEvents ?? 100}`, ); }, diff --git a/packages/transformers/ga4/src/transformer.ts b/packages/transformers/ga4/src/transformer.ts index 10f4a4eaa..fe7db612b 100644 --- a/packages/transformers/ga4/src/transformer.ts +++ b/packages/transformers/ga4/src/transformer.ts @@ -1,5 +1,5 @@ import type { Ingest, Transformer } from '@walkeros/core'; -import { StepError } from '@walkeros/core'; +import { StepError, reportCause, stepId } from '@walkeros/core'; import { parseRequest, splitBodyLines } from './parse'; import { mapHitToEvents } from './map'; import { defaultMapping } from './defaults'; @@ -41,7 +41,7 @@ export const transformerGa4: Transformer.Init< type: 'ga4', config, async push(_event, ctx) { - const { logger } = ctx; + const { logger, collector, id } = ctx; // Without a request there is nothing to decode: a wiring mistake (the // source's config.ingest) that would drop all traffic, so it fails. @@ -54,11 +54,17 @@ export const transformerGa4: Transformer.Init< // Bound the fan-out before decoding: every event becomes a concurrent // push, and the source's batch cap does not apply to a raw text body. + // The dropped request is a loss, so it is reported (ERROR, once per + // cause and minute). if (raw.body) { const lineCount = splitBodyLines(raw.body).length; if (lineCount > maxEvents) { - logger.warn( + reportCause( + collector, + logger, + { step: stepId('transformer', id), code: 'invalid' }, `transformer-ga4: ${lineCount} events exceed maxEvents ${maxEvents}; request dropped`, + { events: lineCount, maxEvents }, ); return false; } From 4af2617ce25d14e7abe4680bf643d207b18d2c02 Mon Sep 17 00:00:00 2001 From: alexanderkirtzel Date: Wed, 21 Oct 2026 08:58:47 +0000 Subject: [PATCH 31/41] server sdk fixes --- .../amplitude/src/__tests__/failure.test.ts | 45 +++++ .../amplitude/src/__tests__/retry.test.ts | 75 ++++++++ .../amplitude/src/examples/step.ts | 39 +++- .../destinations/amplitude/src/index.ts | 31 ++- .../server/destinations/amplitude/src/push.ts | 152 +++++++++------ .../amplitude/src/schemas/settings.ts | 8 +- .../destinations/amplitude/src/transport.ts | 53 ++++++ .../posthog/src/__tests__/failure.test.ts | 50 +++++ .../posthog/src/__tests__/retry.test.ts | 97 ++++++++++ .../server/destinations/posthog/src/config.ts | 12 +- .../destinations/posthog/src/examples/step.ts | 19 +- .../server/destinations/posthog/src/index.ts | 2 +- .../server/destinations/posthog/src/push.ts | 96 ++++++++-- .../posthog/src/schemas/settings.ts | 8 +- .../rudderstack/src/__tests__/failure.test.ts | 71 +++++++ .../rudderstack/src/__tests__/retry.test.ts | 83 ++++++++ .../rudderstack/src/examples/step.ts | 19 ++ .../destinations/rudderstack/src/index.ts | 12 +- .../destinations/rudderstack/src/push.ts | 180 ++++++++++++------ .../rudderstack/src/schemas/settings.ts | 12 +- .../rudderstack/src/types/index.ts | 4 +- .../segment/src/__tests__/failure.test.ts | 72 +++++++ .../segment/src/__tests__/retry.test.ts | 93 +++++++++ .../destinations/segment/src/examples/step.ts | 19 ++ .../server/destinations/segment/src/index.ts | 23 ++- .../server/destinations/segment/src/push.ts | 164 +++++++++++----- .../segment/src/schemas/settings.ts | 8 +- .../destinations/segment/src/types/index.ts | 4 +- .../sqlite/src/__tests__/init.test.ts | 16 +- 29 files changed, 1238 insertions(+), 229 deletions(-) create mode 100644 packages/server/destinations/amplitude/src/__tests__/retry.test.ts create mode 100644 packages/server/destinations/amplitude/src/transport.ts create mode 100644 packages/server/destinations/posthog/src/__tests__/retry.test.ts create mode 100644 packages/server/destinations/rudderstack/src/__tests__/retry.test.ts create mode 100644 packages/server/destinations/segment/src/__tests__/retry.test.ts diff --git a/packages/server/destinations/amplitude/src/__tests__/failure.test.ts b/packages/server/destinations/amplitude/src/__tests__/failure.test.ts index 243ab4fcd..5c61d3750 100644 --- a/packages/server/destinations/amplitude/src/__tests__/failure.test.ts +++ b/packages/server/destinations/amplitude/src/__tests__/failure.test.ts @@ -79,4 +79,49 @@ describe('amplitude send failures', () => { await collector.command('shutdown'); }); + + it('says at DEBUG which mapping a declared skip found nothing for', async () => { + const lines: Array<[string, unknown]> = []; + const { collector } = await startFlow({ + logger: { + level: 'DEBUG', + handler: (_level, message, context) => lines.push([message, context]), + }, + destinations: { + amplitude: { + code: destinationAmplitude, + config: { + settings: { apiKey: 'test' }, + batch: undefined, + mapping: { + company: { + update: { + silent: true, + settings: { + group: { + map: { type: { value: 'company' }, name: 'data.company' }, + }, + }, + }, + }, + }, + }, + env: envFailing(() => false), + }, + }, + }); + + await collector.push(createEvent({ name: 'company update', data: {} })); + + expect( + lines.filter(([message]) => message.includes('resolved to nothing')), + ).toEqual([ + [ + 'Amplitude skipped group: group resolved to nothing', + expect.objectContaining({ event: 'company update' }), + ], + ]); + + await collector.command('shutdown'); + }); }); diff --git a/packages/server/destinations/amplitude/src/__tests__/retry.test.ts b/packages/server/destinations/amplitude/src/__tests__/retry.test.ts new file mode 100644 index 000000000..e415247ef --- /dev/null +++ b/packages/server/destinations/amplitude/src/__tests__/retry.test.ts @@ -0,0 +1,75 @@ +import { startFlow } from '@walkeros/collector'; +import { createEvent } from '@walkeros/core'; +import { destinationAmplitude } from '..'; + +/** Run fake time until the promise settles; answers how long it took. */ +async function settle( + promise: Promise, +): Promise<{ value: T; ms: number }> { + const started = Date.now(); + let done = false; + promise.then( + () => (done = true), + () => (done = true), + ); + for (let step = 0; !done && step < 1000; step++) + await jest.advanceTimersByTimeAsync(50); + return { value: await promise, ms: Date.now() - started }; +} + +describe('amplitude transport within config.timeout', () => { + beforeEach(() => jest.useFakeTimers()); + afterEach(() => { + jest.useRealTimers(); + jest.restoreAllMocks(); + }); + + it('fails a request Amplitude never answers within the timeout, and the next flush delivers', async () => { + // Amplitude's endpoint behind the real SDK: no answer until it is up. + let up = false; + let requests = 0; + jest.spyOn(globalThis, 'fetch').mockImplementation((_url, init) => { + requests++; + if (up) + return Promise.resolve( + new Response(JSON.stringify({ code: 200, events_ingested: 1 }), { + status: 200, + }), + ); + return new Promise((_resolve, reject) => + init?.signal?.addEventListener('abort', () => + reject(init.signal?.reason), + ), + ); + }); + + const { collector } = await startFlow({ + logger: { level: 'ERROR', handler: () => undefined }, + destinations: { + amplitude: { + code: destinationAmplitude, + config: { settings: { apiKey: 'k', logLevel: 0 }, batch: undefined }, + }, + }, + }); + + const hung = await settle( + collector.push(createEvent({ name: 'product view' })), + ); + // The SDK's own failure (three requests of 2.5 s and its waits), not the + // collector's race. + expect(hung.value.failed?.amplitude?.reason).toMatchObject({ + code: 'transport', + message: 'Amplitude track failed (code 500)', + }); + expect(requests).toBe(3); + expect(hung.ms).toBeLessThan(10_000); + + up = true; + const next = await settle( + collector.push(createEvent({ name: 'product view' })), + ); + expect(next.value.ok).toBe(true); + expect(requests).toBe(4); + }); +}); diff --git a/packages/server/destinations/amplitude/src/examples/step.ts b/packages/server/destinations/amplitude/src/examples/step.ts index b7f9468af..6f9e52fc7 100644 --- a/packages/server/destinations/amplitude/src/examples/step.ts +++ b/packages/server/destinations/amplitude/src/examples/step.ts @@ -23,8 +23,8 @@ export const defaultEventForwarding: AmplitudeStepExample = { title: 'Default event', description: 'A walkerOS event forwarded as an Amplitude track call with the event name and empty properties.', - in: getEvent('product view', { timestamp: 1700000100 }), - out: [['amplitude.track', 'product view', {}]], + in: getEvent('product view', { id: 'ev1700000100', timestamp: 1700000100 }), + out: [['amplitude.track', 'product view', {}, { insert_id: 'ev1700000100' }]], }; /** @@ -46,7 +46,7 @@ export const destinationLevelInclude: AmplitudeStepExample = { title: 'Include data section', description: 'Destination-level include flattens the event data section into prefixed event_properties on every track call.', - in: getEvent('product view', { timestamp: 1700000102 }), + in: getEvent('product view', { id: 'ev1700000102', timestamp: 1700000102 }), configInclude: ['data'], out: [ [ @@ -59,6 +59,7 @@ export const destinationLevelInclude: AmplitudeStepExample = { data_size: 'l', data_price: 420, }, + { insert_id: 'ev1700000102' }, ], ], }; @@ -76,7 +77,7 @@ export const destinationLevelIdentify: AmplitudeStepExample = { title: 'Identify per event', description: 'Destination-level identify resolves user_id, device_id, and session_id into the Amplitude EventOptions on every call.', - in: getEvent('page view', { timestamp: 1700000104 }), + in: getEvent('page view', { id: 'ev1700000104', timestamp: 1700000104 }), settings: { identify: { map: { @@ -92,6 +93,7 @@ export const destinationLevelIdentify: AmplitudeStepExample = { 'page view', {}, { + insert_id: 'ev1700000104', user_id: 'us3r', device_id: 'c00k13', session_id: 394324160, @@ -114,6 +116,7 @@ export const userLoginIdentify: AmplitudeStepExample = { 'A user login maps to amplitude.identify with set, setOnce, and add operations while the default track call is skipped.', in: createEvent({ name: 'user login', + id: 'ev1700000105', timestamp: 1700000105, data: { user_id: 'new-user-123', @@ -166,6 +169,7 @@ export const userLoginIdentify: AmplitudeStepExample = { }, }, { + insert_id: 'ev1700000105-identify', user_id: 'new-user-123', }, ], @@ -186,6 +190,7 @@ export const subscriptionRenewRevenue: AmplitudeStepExample = { 'A subscription renewal fires a single amplitude.revenue call with productId, price, and a currency fallback.', in: createEvent({ name: 'subscription renew', + id: 'ev1700000107', timestamp: 1700000107, data: { plan_id: 'plan-pro', @@ -214,6 +219,7 @@ export const subscriptionRenewRevenue: AmplitudeStepExample = { revenueType: 'renewal', currency: 'EUR', }, + { insert_id: 'ev1700000107-revenue-0' }, ], ], }; @@ -234,7 +240,7 @@ export const orderCompleteMultiProduct: AmplitudeStepExample = { title: 'Multi-product order', description: 'An order with multiple nested products fires one amplitude.revenue per product plus a single track for the order.', - in: getEvent('order complete', { timestamp: 1700000108 }), + in: getEvent('order complete', { id: 'ev1700000108', timestamp: 1700000108 }), mapping: { include: ['data', 'globals'], settings: { @@ -268,6 +274,7 @@ export const orderCompleteMultiProduct: AmplitudeStepExample = { revenueType: 'purchase', currency: 'EUR', }, + { insert_id: 'ev1700000108-revenue-0' }, ], [ 'amplitude.revenue', @@ -278,6 +285,7 @@ export const orderCompleteMultiProduct: AmplitudeStepExample = { revenueType: 'purchase', currency: 'EUR', }, + { insert_id: 'ev1700000108-revenue-1' }, ], [ 'amplitude.track', @@ -290,6 +298,7 @@ export const orderCompleteMultiProduct: AmplitudeStepExample = { data_value: 555, globals_pagegroup: 'shop', }, + { insert_id: 'ev1700000108' }, ], ], }; @@ -304,6 +313,7 @@ export const groupAssignmentWithProperties: AmplitudeStepExample = { 'A company update assigns the user to a group and sets group properties via setGroup plus groupIdentify.', in: createEvent({ name: 'company update', + id: 'ev1700000109', timestamp: 1700000109, data: { company: 'Acme', @@ -341,7 +351,12 @@ export const groupAssignmentWithProperties: AmplitudeStepExample = { }, }, out: [ - ['amplitude.setGroup', 'company', 'Acme'], + [ + 'amplitude.setGroup', + 'company', + 'Acme', + { insert_id: 'ev1700000109-group' }, + ], [ 'amplitude.groupIdentify', 'company', @@ -355,6 +370,7 @@ export const groupAssignmentWithProperties: AmplitudeStepExample = { founded: 2020, }, }, + { insert_id: 'ev1700000109-groupIdentify' }, ], ], }; @@ -393,6 +409,9 @@ export const eventOptionsTimeInsertId: AmplitudeStepExample = { ], }; +const analyticsDenied: WalkerOS.Consent = { analytics: false }; +const analyticsGranted: WalkerOS.Consent = { analytics: true }; + /** * Consent revoked -> amplitude.setOptOut(true). After analytics consent is * granted (the destination loads), revoking it toggles optOut via the @@ -409,8 +428,8 @@ export const consentRevokeOptOut: AmplitudeStepExample = { 'After analytics consent is granted (Amplitude loads), revoking it opts out of tracking via setOptOut(true).', command: 'consent', before: { analytics: true }, - in: { analytics: false } as WalkerOS.Consent, - settings: {} as Partial, + in: analyticsDenied, + settings: {}, out: [ ['amplitude.setOptOut', false], ['amplitude.setOptOut', true], @@ -425,8 +444,8 @@ export const consentGrantOptIn: AmplitudeStepExample = { description: 'A walker consent command with analytics granted opts back into Amplitude tracking via setOptOut(false).', command: 'consent', - in: { analytics: true } as WalkerOS.Consent, - settings: {} as Partial, + in: analyticsGranted, + settings: {}, out: [['amplitude.setOptOut', false]], }; diff --git a/packages/server/destinations/amplitude/src/index.ts b/packages/server/destinations/amplitude/src/index.ts index 1db6ac21b..3d1370ee4 100644 --- a/packages/server/destinations/amplitude/src/index.ts +++ b/packages/server/destinations/amplitude/src/index.ts @@ -1,10 +1,21 @@ import type { WalkerOS } from '@walkeros/core'; import type { Destination, Env } from './types'; +import { retryBudget } from '@walkeros/server-core'; import { getConfig } from './config'; import { push, pushBatch, getAmplitude } from './push'; +import { AmplitudeTransport } from './transport'; export * as DestinationAmplitude from './types'; +/** + * The SDK's longest total wait before `retries` retries: the first waits + * `flushIntervalMillis` (here the budget's wait), each later one a second + * longer than the one before (1 s, 2 s, ...). + */ +function amplitudeWaits(retries: number, wait: number): number { + return retries && wait + 500 * retries * (retries - 1); +} + export const destinationAmplitude: Destination = { type: 'amplitude', @@ -13,7 +24,7 @@ export const destinationAmplitude: Destination = { async init({ config: partialConfig, env }) { const config = getConfig(partialConfig); - const amp = getAmplitude(env as Record | undefined); + const amp = getAmplitude(env); // Destructure walkerOS-specific keys; the rest flow through to the SDK. const { @@ -25,8 +36,16 @@ export const destinationAmplitude: Destination = { } = config.settings; // Each push and each batch flushes the SDK and waits until Amplitude - // answered its calls. - await amp.init(apiKey, nodeOptions).promise; + // answered its calls. The SDK's own retry ends within config.timeout: a + // transport whose requests end after a quarter of it, and as many + // retries as fit with the SDK's waits. A value set in the settings wins. + const budget = retryBudget(partialConfig.timeout, amplitudeWaits); + await amp.init(apiKey, { + transportProvider: new AmplitudeTransport(budget.attempt), + flushMaxRetries: budget.retries + 1, + flushIntervalMillis: budget.wait, + ...nodeOptions, + }).promise; return config; }, @@ -36,9 +55,7 @@ export const destinationAmplitude: Destination = { on(type, context) { if (type !== 'consent' || !context.data) return; - const amp = getAmplitude( - context.env as Record | undefined, - ); + const amp = getAmplitude(context.env); const consent = context.data as WalkerOS.Consent; const required = context.config?.consent; @@ -51,7 +68,7 @@ export const destinationAmplitude: Destination = { }, async destroy({ env }) { - const amp = getAmplitude(env as Record | undefined); + const amp = getAmplitude(env); await amp.flush().promise; }, }; diff --git a/packages/server/destinations/amplitude/src/push.ts b/packages/server/destinations/amplitude/src/push.ts index 65e1bc75c..fad841b61 100644 --- a/packages/server/destinations/amplitude/src/push.ts +++ b/packages/server/destinations/amplitude/src/push.ts @@ -1,4 +1,4 @@ -import type { Collector, Destination, WalkerOS } from '@walkeros/core'; +import type { Collector, Destination, Logger, WalkerOS } from '@walkeros/core'; import type { PushFn, PushBatchFn, @@ -19,6 +19,7 @@ import { flattenIncludeSections, StepError, } from '@walkeros/core'; +import { batchOutcome } from '@walkeros/server-core'; import * as amplitudeSDK from '@amplitude/analytics-node'; type IdentifyOpKey = @@ -66,11 +67,17 @@ const REVENUE_FIELD_MAP: Record = { receiptSig: 'setReceiptSig', }; -export function getAmplitude( - env: Record | undefined, -): AmplitudeSDK { - const amplitude = env?.amplitude as AmplitudeSDK | undefined; - return amplitude ?? (amplitudeSDK as unknown as AmplitudeSDK); +/** An injected client (tests, simulation): one that can at least track. */ +function isAmplitudeSDK(value: unknown): value is AmplitudeSDK { + return isObject(value) && typeof value.track === 'function'; +} + +/** The client of `env.amplitude` when one is injected, else the SDK. */ +export function getAmplitude(env: unknown): AmplitudeSDK { + const injected = isObject(env) ? env.amplitude : undefined; + return isAmplitudeSDK(injected) + ? injected + : (amplitudeSDK as unknown as AmplitudeSDK); } /** @@ -205,6 +212,22 @@ function buildEventOptions( return opts; } +/** + * A configured mapping that resolved to nothing skips its call: a declared + * skip, said once at DEBUG with the event and the mapping key, never the + * value, so a wrong mapping path does not look like an event without data. + */ +function skipped( + logger: Logger.Instance, + event: WalkerOS.Event, + call: string, + key: string, +): void { + logger.debug(`Amplitude skipped ${call}: ${key} resolved to nothing`, { + event: event.name, + }); +} + /** A call's outcome, marked as handled until its batch reads it. */ function held(sent: Promise): Promise { sent.catch(() => undefined); @@ -228,6 +251,7 @@ async function enqueue( data: Destination.Data, settings: Partial, collector: Collector.Instance, + logger: Logger.Instance, ): Promise[]> { const mappingSettings = rule?.settings || {}; const calls: Promise[] = []; @@ -241,7 +265,7 @@ async function enqueue( }); if (isObject(resolved)) { identifyResolved = resolved as Record; - } + } else skipped(logger, event, 'identify', 'identify'); } // 2. Resolve eventOptions -- rule-level override wins over destination-level. @@ -264,6 +288,16 @@ async function enqueue( ); const hasEventOptions = Object.keys(eventOptions).length > 0; + // The id Amplitude deduplicates a call by (insert_id): the event id for + // the track call, the event id and the call type for the others, so a + // resent event is dropped and the calls of one event are not. A mapped + // insert_id wins. + const optionsFor = (type?: string): EventOptions | undefined => { + if (!event.id) return hasEventOptions ? eventOptions : undefined; + const insert_id = type ? `${event.id}-${type}` : event.id; + return { insert_id, ...eventOptions }; + }; + // 4. Identify operations -- fires amplitude.identify() if op keys are present. if (identifyResolved) { const id = buildIdentify(amp, identifyResolved); @@ -271,7 +305,7 @@ async function enqueue( calls.push( held( call('Amplitude identify failed', () => - amp.identify(id, hasEventOptions ? eventOptions : undefined), + amp.identify(id, optionsFor('identify')), ), ), ); @@ -282,24 +316,20 @@ async function enqueue( const resolved = await getMappingValue(event, mappingSettings.group, { collector, }); - if (isObject(resolved)) { - const { type, name } = resolved as { - type?: unknown; - name?: unknown; - }; - if (isString(type) && (isString(name) || isArray(name))) - calls.push( - held( - call('Amplitude setGroup failed', () => - amp.setGroup( - type, - name as string | string[], - hasEventOptions ? eventOptions : undefined, - ), - ), + const type = isObject(resolved) ? resolved.type : undefined; + const name = isObject(resolved) ? resolved.name : undefined; + if ( + isString(type) && + (isString(name) || (isArray(name) && name.every(isString))) + ) + calls.push( + held( + call('Amplitude setGroup failed', () => + amp.setGroup(type, name, optionsFor('group')), ), - ); - } + ), + ); + else skipped(logger, event, 'group', 'group'); } // 6. Group identify (properties on the group) @@ -309,28 +339,21 @@ async function enqueue( mappingSettings.groupIdentify, { collector }, ); - if (isObject(resolved)) { - const { type, name } = resolved as { - type?: unknown; - name?: unknown; - }; - if (isString(type) && isString(name)) { - const id = buildIdentify(amp, resolved as Record); - if (id) - calls.push( - held( - call('Amplitude groupIdentify failed', () => - amp.groupIdentify( - type, - name, - id, - hasEventOptions ? eventOptions : undefined, - ), - ), - ), - ); - } - } + const type = isObject(resolved) ? resolved.type : undefined; + const name = isObject(resolved) ? resolved.name : undefined; + const id = + isObject(resolved) && isString(type) && isString(name) + ? buildIdentify(amp, resolved) + : undefined; + if (id && isString(type) && isString(name)) + calls.push( + held( + call('Amplitude groupIdentify failed', () => + amp.groupIdentify(type, name, id, optionsFor('groupIdentify')), + ), + ), + ); + else skipped(logger, event, 'groupIdentify', 'groupIdentify'); } // 7. Revenue -- single object or array from loop -> N revenue() calls. @@ -343,14 +366,15 @@ async function enqueue( : isObject(resolved) ? [resolved as Record] : []; - for (const item of items) { + if (!items.some(isObject)) skipped(logger, event, 'revenue', 'revenue'); + for (const [index, item] of items.entries()) { if (!isObject(item)) continue; calls.push( held( call('Amplitude revenue failed', () => amp.revenue( buildRevenue(amp, item), - hasEventOptions ? eventOptions : undefined, + optionsFor(`revenue-${index}`), ), ), ), @@ -373,11 +397,7 @@ async function enqueue( calls.push( held( call('Amplitude track failed', () => - amp.track( - eventType, - eventProperties, - hasEventOptions ? eventOptions : undefined, - ), + amp.track(eventType, eventProperties, optionsFor()), ), ), ); @@ -405,11 +425,19 @@ function settingsOf( /** One event: its calls handed to the SDK, sent, and their answers awaited. */ export const push: PushFn = async function ( event, - { config, rule, data, collector, env }, + { config, rule, data, collector, env, logger }, ) { - const amp = getAmplitude(env as Record | undefined); + const amp = getAmplitude(env); const settings = settingsOf(config.settings); - const calls = await enqueue(amp, event, rule, data, settings, collector); + const calls = await enqueue( + amp, + event, + rule, + data, + settings, + collector, + logger, + ); if (calls.length) { const settled = Promise.allSettled(calls); await flush(amp); @@ -424,9 +452,9 @@ export const push: PushFn = async function ( */ export const pushBatch: PushBatchFn = async function ( batch, - { config, collector, env }, + { config, collector, env, logger }, ) { - const amp = getAmplitude(env as Record | undefined); + const amp = getAmplitude(env); const settings = settingsOf(config.settings); // Each entry's answers, or why its calls could not be handed over. @@ -443,6 +471,7 @@ export const pushBatch: PushBatchFn = async function ( entry.data, settings, collector, + logger, ); outcomes.push({ settled: Promise.allSettled(calls) }); } catch (error) { @@ -461,8 +490,5 @@ export const pushBatch: PushBatchFn = async function ( ); } - // Nothing landed: a whole-batch failure, which the breaker counts. - if (failed.length && failed.length === batch.entries.length) - throw failed[0].error; - if (failed.length) return { failed }; + return batchOutcome(batch.entries.length, failed); }; diff --git a/packages/server/destinations/amplitude/src/schemas/settings.ts b/packages/server/destinations/amplitude/src/schemas/settings.ts index 3b5bc7a96..b8214d846 100644 --- a/packages/server/destinations/amplitude/src/schemas/settings.ts +++ b/packages/server/destinations/amplitude/src/schemas/settings.ts @@ -18,7 +18,9 @@ export const SettingsSchema = z .number() .int() .positive() - .describe('How often (in ms) to flush the event queue. Default: 10000.') + .describe( + 'How long (in ms) the SDK waits before it sends queued events, and before its first retry of a failed request. Default: a tenth of config.timeout (1000 at the 10 s default). A value set here wins.', + ) .optional(), flushQueueSize: z .number() @@ -32,7 +34,9 @@ export const SettingsSchema = z .number() .int() .nonnegative() - .describe('Max retries on failed flush. Default: 12.') + .describe( + "Attempts per event, the first send included. Default: as many as end within config.timeout with the SDK's waits, each request bounded by a quarter of it (3 at the 10 s default). A value set here wins.", + ) .optional(), useBatch: z .boolean() diff --git a/packages/server/destinations/amplitude/src/transport.ts b/packages/server/destinations/amplitude/src/transport.ts new file mode 100644 index 000000000..640343337 --- /dev/null +++ b/packages/server/destinations/amplitude/src/transport.ts @@ -0,0 +1,53 @@ +import type { Payload, Response, Transport } from '@amplitude/analytics-core'; +import { BaseTransport, Status } from '@amplitude/analytics-core'; +import { isObject, StepError } from '@walkeros/core'; +import { sendServer } from '@walkeros/server-core'; + +/** + * The answers the SDK retries: a 5xx, a 408 and an unreadable answer. Under + * an explicit flush the SDK retries only a send that throws, so these throw. + */ +const RETRIED: Status[] = [Status.Failed, Status.Timeout, Status.Unknown]; + +/** + * The SDK's transport, on the global `fetch` through `sendServer`: every + * request ends after `timeout` ms (a quarter of `config.timeout`). The SDK's + * own Node transport has none, so a request Amplitude never answered kept + * its flush running and every later flush was skipped. A request that times + * out or fails on the network throws, as does an answer the SDK retries: the + * SDK then sends the events again, an explicit flush included, up to + * `flushMaxRetries` attempts. + */ +export class AmplitudeTransport extends BaseTransport implements Transport { + constructor(private readonly timeout: number) { + super(); + } + + async send(serverUrl: string, payload: Payload): Promise { + let status = 200; + let body: unknown; + try { + ({ data: body } = await sendServer(serverUrl, JSON.stringify(payload), { + timeout: this.timeout, + })); + } catch (error) { + // An answer other than 2xx carries its status and body; anything else + // (a timeout, a network failure) is a send that failed. + if (!(error instanceof StepError) || error.status === undefined) + throw error; + status = error.status; + body = error.body; + } + + const response = this.buildResponse({ + code: status, + ...(isObject(body) ? body : {}), + }); + if (!response || RETRIED.includes(response.status)) + throw new StepError(`Amplitude answered ${status}`, { + code: 'transport', + status, + }); + return response; + } +} diff --git a/packages/server/destinations/posthog/src/__tests__/failure.test.ts b/packages/server/destinations/posthog/src/__tests__/failure.test.ts index 10a875574..f98b5c371 100644 --- a/packages/server/destinations/posthog/src/__tests__/failure.test.ts +++ b/packages/server/destinations/posthog/src/__tests__/failure.test.ts @@ -118,4 +118,54 @@ describe('posthog send failures', () => { await collector.command('shutdown'); }); + + it('says at DEBUG which mapping a declared skip found nothing for', async () => { + const lines: Array<[string, unknown]> = []; + const { collector } = await startFlow({ + logger: { + level: 'DEBUG', + handler: (_level, message, context) => lines.push([message, context]), + }, + destinations: { + posthog: { + code: destinationPostHog, + config: { + settings: { apiKey: 'phc_test' }, + batch: undefined, + mapping: { + company: { + update: { + silent: true, + settings: { + group: { + map: { + type: { value: 'company' }, + key: 'data.company_id', + }, + }, + }, + }, + }, + }, + }, + env: failingEnv({}), + }, + }, + }); + + await collector.push( + createEvent({ name: 'company update', user: { id: 'us3r' }, data: {} }), + ); + + expect( + lines.filter(([message]) => message.includes('resolved to nothing')), + ).toEqual([ + [ + 'PostHog skipped group: group resolved to nothing', + expect.objectContaining({ event: 'company update' }), + ], + ]); + + await collector.command('shutdown'); + }); }); diff --git a/packages/server/destinations/posthog/src/__tests__/retry.test.ts b/packages/server/destinations/posthog/src/__tests__/retry.test.ts new file mode 100644 index 000000000..de6abfedc --- /dev/null +++ b/packages/server/destinations/posthog/src/__tests__/retry.test.ts @@ -0,0 +1,97 @@ +import { startFlow } from '@walkeros/collector'; +import { createEvent } from '@walkeros/core'; +import { destinationPostHog } from '..'; + +/** + * PostHog's endpoint behind the real SDK: each request gets the next answer, + * a status or no answer at all (the request then ends when the SDK aborts + * it). + */ +function endpoint(answers: Array) { + let requests = 0; + jest.spyOn(globalThis, 'fetch').mockImplementation((_url, init) => { + const answer = answers[Math.min(requests, answers.length - 1)]; + requests++; + if (answer === 'never') + return new Promise((_resolve, reject) => + init?.signal?.addEventListener('abort', () => + reject(init.signal?.reason), + ), + ); + return Promise.resolve(new Response('{}', { status: answer })); + }); + return { requests: () => requests }; +} + +/** Run fake time until the promise settles; answers how long it took. */ +async function settle( + promise: Promise, +): Promise<{ value: T; ms: number }> { + const started = Date.now(); + let done = false; + promise.then( + () => (done = true), + () => (done = true), + ); + for (let step = 0; !done && step < 1000; step++) + await jest.advanceTimersByTimeAsync(50); + return { value: await promise, ms: Date.now() - started }; +} + +async function flowWith(timeout?: number) { + const { collector } = await startFlow({ + logger: { level: 'ERROR', handler: () => undefined }, + destinations: { + posthog: { + code: destinationPostHog, + config: { + // Uncompressed, so no zlib work runs outside the fake clock. + settings: { apiKey: 'k', disableCompression: true }, + batch: undefined, + timeout, + }, + }, + }, + }); + return collector; +} + +describe('posthog retry within config.timeout', () => { + beforeEach(() => jest.useFakeTimers()); + afterEach(() => { + jest.useRealTimers(); + jest.restoreAllMocks(); + }); + + it('delivers once PostHog answers after two 503s, within the timeout', async () => { + const vendor = endpoint([503, 503, 200]); + const collector = await flowWith(); + + const { value: result, ms } = await settle( + collector.push(createEvent({ name: 'product view' })), + ); + + expect(result.ok).toBe(true); + expect(vendor.requests()).toBe(3); + expect(ms).toBeLessThan(10_000); + }); + + it('fails timeout once when PostHog never answers, before the race', async () => { + const vendor = endpoint(['never']); + const collector = await flowWith(); + + const { value: result, ms } = await settle( + collector.push(createEvent({ name: 'product view' })), + ); + + // The SDK's own timeout, not the collector's race: the package's message. + expect(result.failed?.posthog?.reason).toMatchObject({ + code: 'timeout', + message: 'PostHog send failed', + }); + expect(collector.status.destinations.posthog.failed).toBe(1); + // Three requests of 2.5 s, 1 s apart, end before 10 s. + expect(vendor.requests()).toBe(3); + expect(ms).toBeLessThan(10_000); + }); +}); diff --git a/packages/server/destinations/posthog/src/config.ts b/packages/server/destinations/posthog/src/config.ts index 81e0d9bd5..a6865947b 100644 --- a/packages/server/destinations/posthog/src/config.ts +++ b/packages/server/destinations/posthog/src/config.ts @@ -1,5 +1,6 @@ import type { Config, Env, PartialConfig, Settings } from './types'; import { StepError } from '@walkeros/core'; +import { retryBudget } from '@walkeros/server-core'; import { PostHog } from 'posthog-node'; export function getConfig( @@ -12,8 +13,17 @@ export function getConfig( if (!apiKey) throw new StepError('Config settings apiKey missing', { code: 'config' }); + // The SDK's own retry ends within config.timeout: a quarter of it per + // request, two retries a tenth of it apart. A value set in the settings + // wins. + const budget = retryBudget(partialConfig.timeout); + const options: Record = { + requestTimeout: budget.attempt, + fetchRetryCount: budget.retries, + fetchRetryDelay: budget.wait, + }; + // Extract PostHog SDK options from settings - const options: Record = {}; if (settings.host) options.host = settings.host; if (settings.flushAt !== undefined) options.flushAt = settings.flushAt; if (settings.flushInterval !== undefined) diff --git a/packages/server/destinations/posthog/src/examples/step.ts b/packages/server/destinations/posthog/src/examples/step.ts index 390760858..abeac89ab 100644 --- a/packages/server/destinations/posthog/src/examples/step.ts +++ b/packages/server/destinations/posthog/src/examples/step.ts @@ -27,11 +27,12 @@ export const defaultCapture: PostHogStepExample = { title: 'Default capture', description: 'A walker event becomes a PostHog capture call with the user id as distinctId and no extra properties.', - in: getEvent('product view', { timestamp: 1700000100 }), + in: getEvent('product view', { id: 'ev1700000100', timestamp: 1700000100 }), out: [ [ 'client.capture', { + uuid: 'b8a7f21f-1a9f-8aa7-8480-8d16fb64c44c', distinctId: 'us3r', event: 'product view', properties: {}, @@ -48,12 +49,13 @@ export const captureWithInclude: PostHogStepExample = { title: 'Capture with include', description: 'Destination-level include flattens data and globals sections into prefixed PostHog event properties.', - in: getEvent('order complete', { timestamp: 1700000101 }), + in: getEvent('order complete', { id: 'ev1700000101', timestamp: 1700000101 }), configInclude: ['data', 'globals'], out: [ [ 'client.capture', { + uuid: 'a5a5ede9-4f3b-85f4-b30f-3da8773fabfa', distinctId: 'us3r', event: 'order complete', properties: { @@ -184,6 +186,7 @@ export const captureWithGroupContext: PostHogStepExample = { description: 'A destination-level group mapping attaches the resolved group context to every PostHog capture call.', in: getEvent('page view', { + id: 'ev1700000104', timestamp: 1700000104, globals: { pagegroup: 'docs', company_id: 'company_123' }, }), @@ -199,6 +202,7 @@ export const captureWithGroupContext: PostHogStepExample = { [ 'client.capture', { + uuid: 'c6a47305-44fc-8835-b15f-3f494c1f9e20', distinctId: 'us3r', event: 'page view', properties: {}, @@ -208,6 +212,9 @@ export const captureWithGroupContext: PostHogStepExample = { ], }; +const analyticsDenied: WalkerOS.Consent = { analytics: false }; +const analyticsGranted: WalkerOS.Consent = { analytics: true }; + /** * Consent revoked - after analytics consent is granted (PostHog loads and is * enabled), revoking it calls client.disable() via the on('consent') handler. @@ -220,8 +227,8 @@ export const consentRevoke: PostHogStepExample = { 'A walker consent command with analytics denied calls client.disable on the PostHog client.', command: 'consent', before: { analytics: true }, - in: { analytics: false } as WalkerOS.Consent, - settings: {} as Partial, + in: analyticsDenied, + settings: {}, out: [['client.enable'], ['client.disable']], }; @@ -233,8 +240,8 @@ export const consentGrant: PostHogStepExample = { description: 'A walker consent command with analytics granted calls client.enable on the PostHog client.', command: 'consent', - in: { analytics: true } as WalkerOS.Consent, - settings: {} as Partial, + in: analyticsGranted, + settings: {}, out: [['client.enable']], }; diff --git a/packages/server/destinations/posthog/src/index.ts b/packages/server/destinations/posthog/src/index.ts index 81c01f3e7..33271599d 100644 --- a/packages/server/destinations/posthog/src/index.ts +++ b/packages/server/destinations/posthog/src/index.ts @@ -21,7 +21,7 @@ export const destinationPostHog: Destination = { pushBatch, async destroy({ config }) { - const client = (config?.settings as Settings | undefined)?.client; + const client = config?.settings?.client; if (client) { await client.shutdown(); } diff --git a/packages/server/destinations/posthog/src/push.ts b/packages/server/destinations/posthog/src/push.ts index b37f1e64d..beda610a2 100644 --- a/packages/server/destinations/posthog/src/push.ts +++ b/packages/server/destinations/posthog/src/push.ts @@ -1,4 +1,4 @@ -import type { Collector, Destination, WalkerOS } from '@walkeros/core'; +import type { Collector, Destination, Logger, WalkerOS } from '@walkeros/core'; import type { PostHogClient, PushBatchFn, @@ -6,7 +6,9 @@ import type { Rule, Settings, } from './types'; +import { createHash } from 'crypto'; import { getMappingValue, isObject, isString, StepError } from '@walkeros/core'; +import { batchOutcome } from '@walkeros/server-core'; /** * Run one flush to its end. The SDK reports a failed send as an `error` @@ -30,7 +32,54 @@ async function call( off?.(); } if (failure !== undefined) - throw new StepError(message, { code: 'transport', cause: failure }); + throw new StepError(message, { + code: timedOut(failure) ? 'timeout' : 'transport', + cause: failure, + }); +} + +/** + * Whether the SDK's last request timed out: it aborts a request after + * `requestTimeout` and reports a network error carrying the abort. + */ +function timedOut(error: unknown): boolean { + if (typeof error !== 'object' || error === null) return false; + const name: unknown = Reflect.get(error, 'name'); + if (name === 'AbortError' || name === 'TimeoutError') return true; + return timedOut(Reflect.get(error, 'error') ?? Reflect.get(error, 'cause')); +} + +/** + * The uuid PostHog deduplicates a captured event by, derived from the event + * id: the same event resent gets the same uuid. PostHog takes only a UUID, + * so the id is hashed into one (RFC 9562 version 8, name based). + */ +function uuidOf(id: string): string { + const hex = createHash('sha256').update(id).digest('hex'); + const variant = ((parseInt(hex[16], 16) & 0x3) | 0x8).toString(16); + return [ + hex.slice(0, 8), + hex.slice(8, 12), + `8${hex.slice(13, 16)}`, + `${variant}${hex.slice(17, 20)}`, + hex.slice(20, 32), + ].join('-'); +} + +/** + * A configured mapping that resolved to nothing skips its call: a declared + * skip, said once at DEBUG with the event and the mapping key, never the + * value, so a wrong mapping path does not look like an event without data. + */ +function skipped( + logger: Logger.Instance, + event: WalkerOS.Event, + call: string, + key: string, +): void { + logger.debug(`PostHog skipped ${call}: ${key} resolved to nothing`, { + event: event.name, + }); } /** Run one queueing call; an injected client may answer with a promise. */ @@ -72,11 +121,11 @@ function clientOf(settings: Partial | undefined): { /** One event: its calls queued and sent by one flush. */ export const push: PushFn = async function ( event, - { config, rule, data, collector }, + { config, rule, data, collector, logger }, ) { const { settings, client } = clientOf(config.settings); await exclusive(settings, async () => { - await send(event, rule, data, settings, client, collector); + await send(event, rule, data, settings, client, collector, logger); await call(client, 'PostHog send failed', () => client.flush()); }); }; @@ -88,12 +137,14 @@ export const push: PushFn = async function ( */ export const pushBatch: PushBatchFn = async function ( batch, - { config, collector }, + { config, collector, logger }, ) { const { settings, client } = clientOf(config.settings); const failed: Destination.BatchFailure[] = []; + let sendError: unknown; await exclusive(settings, async () => { + const queued: number[] = []; for (const [index, entry] of batch.entries.entries()) await send( entry.event, @@ -102,16 +153,23 @@ export const pushBatch: PushBatchFn = async function ( settings, client, collector, - ).catch((error: unknown) => failed.push({ index, error })); + logger, + ).then( + () => queued.push(index), + (error: unknown) => failed.push({ index, error }), + ); - if (failed.length < batch.entries.length) - await call(client, 'PostHog send failed', () => client.flush()); + // One request carries every queued entry: a failed flush fails them all. + if (queued.length) + await call(client, 'PostHog send failed', () => client.flush()).catch( + (error: unknown) => { + sendError = error; + for (const index of queued) failed.push({ index, error }); + }, + ); }); - // Nothing landed: a whole-batch failure, which the breaker counts. - if (failed.length && failed.length === batch.entries.length) - throw failed[0].error; - if (failed.length) return { failed }; + return batchOutcome(batch.entries.length, failed, sendError); }; /** The calls of one event (identify, group identify, capture). */ @@ -122,6 +180,7 @@ async function send( settings: Partial, client: PostHogClient, collector: Collector.Instance, + logger: Logger.Instance, ): Promise { const mappingSettings = (rule?.settings || {}) as Record; @@ -139,7 +198,7 @@ async function send( if (isString(identifyResolved.distinctId)) { distinctId = identifyResolved.distinctId as string; } - } + } else skipped(logger, event, 'identify', 'identify'); } // Fallback distinctId from event.user @@ -199,8 +258,8 @@ async function send( ); // Build groups object for capture groups = { [type]: key as string | number }; - } - } + } else skipped(logger, event, 'group', 'group'); + } else skipped(logger, event, 'group', 'group'); } // 3. Capture - unless rule opts out via silent @@ -208,15 +267,14 @@ async function send( const eventName = isString(rule?.name) ? rule.name : event.name; const properties = isObject(data) ? (data as Record) : {}; - const captureArgs: Record = { + const captureArgs: Parameters[0] = { distinctId, event: eventName, properties, }; if (groups) captureArgs.groups = groups; + if (event.id) captureArgs.uuid = uuidOf(event.id); - await queue('PostHog capture failed', () => - client.capture(captureArgs as Parameters[0]), - ); + await queue('PostHog capture failed', () => client.capture(captureArgs)); } } diff --git a/packages/server/destinations/posthog/src/schemas/settings.ts b/packages/server/destinations/posthog/src/schemas/settings.ts index 799b54698..2f06b6851 100644 --- a/packages/server/destinations/posthog/src/schemas/settings.ts +++ b/packages/server/destinations/posthog/src/schemas/settings.ts @@ -47,17 +47,19 @@ export const SettingsSchema = z requestTimeout: z .number() .describe( - 'Milliseconds before a request to PostHog times out. Default: 10000.', + 'Milliseconds before a request to PostHog times out. Default: a quarter of config.timeout (2500 at the 10 s default). A value set here wins.', ) .optional(), fetchRetryCount: z .number() - .describe('Retries of a failed request to PostHog. Default: 3.') + .describe( + 'Retries of a failed request to PostHog. Default: 2, which with the default timeout and delay ends within config.timeout. A value set here wins.', + ) .optional(), fetchRetryDelay: z .number() .describe( - 'Milliseconds between retries of a failed request. Default: 3000.', + 'Milliseconds between retries of a failed request. Default: a tenth of config.timeout (1000 at the 10 s default). A value set here wins.', ) .optional(), debug: z diff --git a/packages/server/destinations/rudderstack/src/__tests__/failure.test.ts b/packages/server/destinations/rudderstack/src/__tests__/failure.test.ts index cd58195e1..58854773c 100644 --- a/packages/server/destinations/rudderstack/src/__tests__/failure.test.ts +++ b/packages/server/destinations/rudderstack/src/__tests__/failure.test.ts @@ -80,4 +80,75 @@ describe('rudderstack send failures', () => { await collector.command('shutdown'); }); + + it('fails a call the SDK refuses at once with its own StepError', async () => { + const refusing: RudderStackAnalyticsMock = { + ...analyticsFailing(() => false), + track: () => { + throw new Error('Invalid event'); + }, + }; + const { collector } = await startFlow({ + logger: { level: 'ERROR', handler: () => undefined }, + destinations: { + rudderstack: { + code: destinationRudderStack, + config: { settings, batch: undefined }, + env: { analytics: refusing }, + }, + }, + }); + + const result = await collector.push(getEvent('product view')); + + expect(result.failed?.rudderstack?.reason).toMatchObject({ + code: 'transport', + message: 'RudderStack track failed', + }); + + await collector.command('shutdown'); + }); + + it('says at DEBUG which mapping a declared skip found nothing for', async () => { + const lines: Array<[string, unknown]> = []; + const { collector } = await startFlow({ + logger: { + level: 'DEBUG', + handler: (_level, message, context) => lines.push([message, context]), + }, + destinations: { + rudderstack: { + code: destinationRudderStack, + config: { + settings, + batch: undefined, + mapping: { + company: { + update: { + silent: true, + settings: { group: { map: { groupId: 'data.company_id' } } }, + }, + }, + }, + }, + env: { analytics: analyticsFailing(() => false) }, + }, + }, + }); + + await collector.push( + createEvent({ name: 'company update', user: { id: 'us3r' }, data: {} }), + ); + + expect( + lines.filter(([message]) => message.includes('resolved to nothing')), + ).toEqual([ + [ + 'RudderStack skipped group: group resolved to nothing', + expect.objectContaining({ event: 'company update' }), + ], + ]); + + await collector.command('shutdown'); + }); }); diff --git a/packages/server/destinations/rudderstack/src/__tests__/retry.test.ts b/packages/server/destinations/rudderstack/src/__tests__/retry.test.ts new file mode 100644 index 000000000..ccd9e0d68 --- /dev/null +++ b/packages/server/destinations/rudderstack/src/__tests__/retry.test.ts @@ -0,0 +1,83 @@ +import type { Server } from 'http'; +import { createServer } from 'http'; +import { startFlow } from '@walkeros/collector'; +import { createEvent } from '@walkeros/core'; +import { destinationRudderStack } from '..'; + +/** + * RudderStack's data plane behind the real SDK, on a local port: each + * request gets the next answer, a status or no answer at all. Real timers: + * axios bounds a request with the socket's own timer, which fake timers do + * not drive, so a short config.timeout keeps the run fast. + */ +async function dataPlane(answers: Array) { + let requests = 0; + const server: Server = createServer((req, res) => { + const answer = answers[Math.min(requests, answers.length - 1)]; + requests++; + req.resume(); + if (answer !== 'never') res.writeHead(answer).end('{}'); + }); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + const address = server.address(); + const port = typeof address === 'object' && address ? address.port : 0; + return { + url: `http://127.0.0.1:${port}`, + requests: () => requests, + close: () => { + server.closeAllConnections(); + return new Promise((resolve) => server.close(() => resolve())); + }, + }; +} + +/** 200 ms per request, two retries 80 ms apart: 760 ms at worst. */ +const TIMEOUT = 800; + +async function flowWith(dataPlaneUrl: string) { + const { collector } = await startFlow({ + logger: { level: 'ERROR', handler: () => undefined }, + destinations: { + rudderstack: { + code: destinationRudderStack, + config: { + settings: { writeKey: 'w', dataPlaneUrl }, + batch: undefined, + timeout: TIMEOUT, + }, + }, + }, + }); + return collector; +} + +describe('rudderstack retry within config.timeout', () => { + it('delivers once RudderStack answers after two 503s, within the timeout', async () => { + const plane = await dataPlane([503, 503, 200]); + const collector = await flowWith(plane.url); + + const started = Date.now(); + const result = await collector.push(createEvent({ name: 'product view' })); + + expect(result.ok).toBe(true); + expect(plane.requests()).toBe(3); + expect(Date.now() - started).toBeLessThan(TIMEOUT); + await collector.command('shutdown'); + await plane.close(); + }); + + it('fails timeout once when RudderStack never answers, before the race', async () => { + const plane = await dataPlane(['never']); + const collector = await flowWith(plane.url); + + const result = await collector.push(createEvent({ name: 'product view' })); + + // The SDK's own timeout, not the collector's race: the package's message. + expect(result.failed?.rudderstack?.reason).toMatchObject({ + code: 'timeout', + message: 'RudderStack track failed', + }); + expect(collector.status.destinations.rudderstack.failed).toBe(1); + await plane.close(); + }); +}); diff --git a/packages/server/destinations/rudderstack/src/examples/step.ts b/packages/server/destinations/rudderstack/src/examples/step.ts index 699a16fac..1c72ecb89 100644 --- a/packages/server/destinations/rudderstack/src/examples/step.ts +++ b/packages/server/destinations/rudderstack/src/examples/step.ts @@ -29,6 +29,7 @@ export const defaultTrack: RudderStackStepExample = { description: 'A walker event becomes a RudderStack analytics.track call with userId and anonymousId resolved from the event.', in: getEvent('product view', { + id: 'ev1700000100', timestamp: 1700000100, user: { id: 'us3r', session: 's3ss10n' }, }), @@ -36,6 +37,7 @@ export const defaultTrack: RudderStackStepExample = { [ 'analytics.track', { + messageId: 'ev1700000100', userId: 'us3r', anonymousId: 's3ss10n', event: 'product view', @@ -54,6 +56,7 @@ export const mappedEventName: RudderStackStepExample = { description: 'A mapping renames the event so the RudderStack track call uses a canonical name such as Order Completed.', in: getEvent('order complete', { + id: 'ev1700000101', timestamp: 1700000101, user: { id: 'us3r', session: 's3ss10n' }, }), @@ -64,6 +67,7 @@ export const mappedEventName: RudderStackStepExample = { [ 'analytics.track', { + messageId: 'ev1700000101', userId: 'us3r', anonymousId: 's3ss10n', event: 'Order Completed', @@ -83,6 +87,7 @@ export const destinationIdentify: RudderStackStepExample = { description: 'Destination-level identify fires analytics.identify with traits on the first push only, then the track call follows.', in: getEvent('page view', { + id: 'ev1700000102', timestamp: 1700000102, user: { id: 'us3r', session: 's3ss10n', email: 'user@example.com' }, }), @@ -101,6 +106,7 @@ export const destinationIdentify: RudderStackStepExample = { [ 'analytics.identify', { + messageId: 'ev1700000102-identify', userId: 'us3r', anonymousId: 's3ss10n', traits: { email: 'user@example.com' }, @@ -110,6 +116,7 @@ export const destinationIdentify: RudderStackStepExample = { [ 'analytics.track', { + messageId: 'ev1700000102', userId: 'us3r', anonymousId: 's3ss10n', event: 'page view', @@ -129,6 +136,7 @@ export const userLoginIdentify: RudderStackStepExample = { 'A user login fires only an analytics.identify call with the resolved userId and traits, skipping the track.', in: createEvent({ name: 'user login', + id: 'ev1700000103', timestamp: 1700000103, user: { id: 'us3r', session: 's3ss10n' }, data: { @@ -159,6 +167,7 @@ export const userLoginIdentify: RudderStackStepExample = { [ 'analytics.identify', { + messageId: 'ev1700000103-identify', userId: 'new-user-123', anonymousId: 's3ss10n', traits: { email: 'user@acme.com', name: 'Jane Doe', plan: 'premium' }, @@ -177,6 +186,7 @@ export const companyGroup: RudderStackStepExample = { 'A company update fires analytics.group with the groupId and group traits for B2B account tracking.', in: createEvent({ name: 'company update', + id: 'ev1700000104', timestamp: 1700000104, user: { id: 'us3r', session: 's3ss10n' }, data: { @@ -207,6 +217,7 @@ export const companyGroup: RudderStackStepExample = { [ 'analytics.group', { + messageId: 'ev1700000104-group', userId: 'us3r', anonymousId: 's3ss10n', groupId: 'comp-456', @@ -226,6 +237,7 @@ export const pageView: RudderStackStepExample = { description: 'A page view fires analytics.page with a resolved page name and properties instead of a generic track.', in: getEvent('page view', { + id: 'ev1700000105', timestamp: 1700000105, user: { id: 'us3r', session: 's3ss10n' }, data: { @@ -252,6 +264,7 @@ export const pageView: RudderStackStepExample = { [ 'analytics.page', { + messageId: 'ev1700000105-page', userId: 'us3r', anonymousId: 's3ss10n', name: 'Getting Started', @@ -272,6 +285,7 @@ export const screenView: RudderStackStepExample = { 'A screen view from a mobile app backend fires analytics.screen with a screen name and properties.', in: createEvent({ name: 'screen view', + id: 'ev1700000106', timestamp: 1700000106, user: { id: 'us3r', session: 's3ss10n' }, data: { @@ -299,6 +313,7 @@ export const screenView: RudderStackStepExample = { [ 'analytics.screen', { + messageId: 'ev1700000106-screen', userId: 'us3r', anonymousId: 's3ss10n', name: 'Welcome', @@ -319,6 +334,7 @@ export const aliasUser: RudderStackStepExample = { 'An identity merge fires analytics.alias to link a previous anonymous id to the newly registered user id.', in: createEvent({ name: 'identity merge', + id: 'ev1700000107', timestamp: 1700000107, user: { id: 'registered-456', session: 's3ss10n' }, data: { @@ -339,6 +355,7 @@ export const aliasUser: RudderStackStepExample = { [ 'analytics.alias', { + messageId: 'ev1700000107-alias', userId: 'registered-456', previousId: 'anonymous-123', timestamp: new Date(1700000107), @@ -355,6 +372,7 @@ export const anonymousOnly: RudderStackStepExample = { description: 'When no userId is resolved RudderStack accepts the track call keyed solely by anonymousId.', in: getEvent('product view', { + id: 'ev1700000108', timestamp: 1700000108, user: { session: 's3ss10n' }, }), @@ -365,6 +383,7 @@ export const anonymousOnly: RudderStackStepExample = { [ 'analytics.track', { + messageId: 'ev1700000108', anonymousId: 's3ss10n', event: 'product view', properties: {}, diff --git a/packages/server/destinations/rudderstack/src/index.ts b/packages/server/destinations/rudderstack/src/index.ts index 6e8d043b3..bf5f81f39 100644 --- a/packages/server/destinations/rudderstack/src/index.ts +++ b/packages/server/destinations/rudderstack/src/index.ts @@ -1,5 +1,6 @@ import type { Destination } from './types'; import { StepError } from '@walkeros/core'; +import { retryBudget } from '@walkeros/server-core'; import { getConfig } from './config'; import { push, pushBatch } from './push'; @@ -38,9 +39,18 @@ export const destinationRudderStack: Destination = { // Each push and each batch flushes the SDK and waits until its calls // are sent. A failed send reaches each call's callback; the error // handler only keeps the SDK from rethrowing it as an unhandled - // rejection. + // rejection. The SDK's own retry ends within config.timeout: a + // quarter of it per request, two retries a tenth of it apart. A value + // set in the settings wins. + const budget = retryBudget(partialConfig.timeout); settings._analytics = new Analytics(writeKey, { dataPlaneUrl, + timeout: budget.attempt, + retryCount: budget.retries, + axiosRetryConfig: { + retryDelay: () => budget.wait, + shouldResetTimeout: true, + }, ...sdkOptions, errorHandler: () => undefined, }); diff --git a/packages/server/destinations/rudderstack/src/push.ts b/packages/server/destinations/rudderstack/src/push.ts index fcff22522..378931ba3 100644 --- a/packages/server/destinations/rudderstack/src/push.ts +++ b/packages/server/destinations/rudderstack/src/push.ts @@ -1,4 +1,4 @@ -import type { Collector, Destination, WalkerOS } from '@walkeros/core'; +import type { Collector, Destination, Logger, WalkerOS } from '@walkeros/core'; import type { PushFn, PushBatchFn, @@ -16,22 +16,69 @@ import { isThenable, StepError, } from '@walkeros/core'; +import { batchOutcome } from '@walkeros/server-core'; /** * Hand one call to the SDK. The promise settles once RudderStack sent it: the * SDK answers through the callback, with the error when the send failed; an - * injected client may answer with a promise instead. + * injected client may answer with a promise instead. A call the SDK refuses + * at once (it validates each message) fails the same way. The promise is + * marked handled at once, since its event's outcome reads it only after the + * flush. */ function call( message: string, send: (done: RudderStackCallback) => unknown, ): Promise { - return new Promise((resolve, reject) => { - const fail = (error: unknown) => - reject(new StepError(message, { code: 'transport', cause: error })); - const answer: unknown = send((error) => (error ? fail(error) : resolve())); + const sent = new Promise((resolve, reject) => { + const fail = (error: unknown) => reject(failure(message, error)); + let answer: unknown; + try { + answer = send((error) => (error ? fail(error) : resolve())); + } catch (error) { + fail(error); + return; + } if (isThenable(answer)) answer.then(() => resolve(), fail); }); + sent.catch(() => undefined); + return sent; +} + +/** + * A failed send: `timeout` when the SDK's last request timed out (axios + * answers `ECONNABORTED`, or `ETIMEDOUT`), else `transport`. + */ +function failure(message: string, error: unknown): StepError { + const code: unknown = + typeof error === 'object' && error !== null + ? Reflect.get(error, 'code') + : undefined; + const timedOut = code === 'ECONNABORTED' || code === 'ETIMEDOUT'; + return new StepError(message, { + code: timedOut ? 'timeout' : 'transport', + cause: error, + }); +} + +/** + * The id RudderStack deduplicates a call by: the event id for the track + * call, the event id and the call type for the others, so a resent event is + * dropped and the calls of one event are not. + */ +function messageIdOf(event: WalkerOS.Event, type?: string): string | undefined { + if (!event.id) return undefined; + return type ? `${event.id}-${type}` : event.id; +} + +/** The identity of one call, with the call's own messageId. */ +function withMessageId( + identity: Record, + event: WalkerOS.Event, + type: string, +): Record { + const messageId = messageIdOf(event, type); + return messageId ? { ...identity, messageId } : identity; } /** @@ -47,6 +94,22 @@ function flush(settings: Partial, sdk: RudderStackAnalyticsMock) { return flushing; } +/** + * A configured mapping that resolved to nothing skips its call: a declared + * skip, said once at DEBUG with the event and the mapping key, never the + * value, so a wrong mapping path does not look like an event without data. + */ +function skipped( + logger: Logger.Instance, + event: WalkerOS.Event, + call: string, + key: string, +): void { + logger.debug(`RudderStack skipped ${call}: ${key} resolved to nothing`, { + event: event.name, + }); +} + /** The settled calls of one event: the first failure, if any. */ async function delivered(calls: Promise[]>) { for (const result of await calls) @@ -65,6 +128,7 @@ async function enqueue( settings: Partial, sdk: RudderStackAnalyticsMock, collector: Collector.Instance, + logger: Logger.Instance, ): Promise[]> { const state: RuntimeState = settings._state || {}; settings._state = state; @@ -108,12 +172,13 @@ async function enqueue( calls.push( applyIdentify( sdk, - resolved as Record, - identity, + resolved, + withMessageId(identity, event, 'identify'), state, timestamp, ), ); + else skipped(logger, event, 'identify', 'identify'); } // 3. Group -- rule-level overrides destination-level @@ -122,62 +187,60 @@ async function enqueue( const resolved = await getMappingValue(event, groupMapping, { collector, }); - if (isObject(resolved)) + if (isObject(resolved) && isString(resolved.groupId)) calls.push( applyGroup( sdk, - resolved as Record, - identity, + resolved, + withMessageId(identity, event, 'group'), state, timestamp, ), ); + else skipped(logger, event, 'group', 'group'); } // 4. Page (the call is not awaited here: it is delivered at the flush) - if (mappingSettings.page !== undefined) - calls.push( - ( - await applyPage( - sdk, - mappingSettings.page, - event, - identity, - collector, - timestamp, - ) - ).sent, + if (mappingSettings.page !== undefined) { + const { sent } = await applyPage( + sdk, + mappingSettings.page, + event, + withMessageId(identity, event, 'page'), + collector, + timestamp, ); + if (sent) calls.push(sent); + else skipped(logger, event, 'page', 'page'); + } // 5. Screen - if (mappingSettings.screen !== undefined) - calls.push( - ( - await applyScreen( - sdk, - mappingSettings.screen, - event, - identity, - collector, - timestamp, - ) - ).sent, + if (mappingSettings.screen !== undefined) { + const { sent } = await applyScreen( + sdk, + mappingSettings.screen, + event, + withMessageId(identity, event, 'screen'), + collector, + timestamp, ); + if (sent) calls.push(sent); + else skipped(logger, event, 'screen', 'screen'); + } // 6. Alias - if (mappingSettings.alias !== undefined) - calls.push( - ( - await applyAlias( - sdk, - mappingSettings.alias, - event, - identity, - collector, - timestamp, - ) - ).sent, + if (mappingSettings.alias !== undefined) { + const { sent } = await applyAlias( + sdk, + mappingSettings.alias, + event, + withMessageId(identity, event, 'alias'), + collector, + timestamp, ); + if (sent) calls.push(sent); + else skipped(logger, event, 'alias', 'alias'); + } // 7. Track (unless silent: true) if (rule?.silent !== true) { @@ -190,6 +253,8 @@ async function enqueue( properties, timestamp, }; + const messageId = messageIdOf(event); + if (messageId) params.messageId = messageId; calls.push( call('RudderStack track failed', (done) => sdk.track(params, done)), @@ -215,11 +280,19 @@ function sdkOf( /** One event: its calls handed to the SDK, sent, and their deliveries awaited. */ export const push: PushFn = async function ( event, - { config, rule, data, collector, env }, + { config, rule, data, collector, env, logger }, ) { const settings = config.settings ?? {}; const sdk = sdkOf(config.settings, env); - const calls = await enqueue(event, rule, data, settings, sdk, collector); + const calls = await enqueue( + event, + rule, + data, + settings, + sdk, + collector, + logger, + ); if (calls.length) { const settled = Promise.allSettled(calls); await flush(settings, sdk); @@ -234,7 +307,7 @@ export const push: PushFn = async function ( */ export const pushBatch: PushBatchFn = async function ( batch, - { config, collector, env }, + { config, collector, env, logger }, ) { const settings = config.settings ?? {}; const sdk = sdkOf(config.settings, env); @@ -253,6 +326,7 @@ export const pushBatch: PushBatchFn = async function ( settings, sdk, collector, + logger, ); outcomes.push({ settled: Promise.allSettled(calls) }); } catch (error) { @@ -271,10 +345,7 @@ export const pushBatch: PushBatchFn = async function ( ); } - // Nothing landed: a whole-batch failure, which the breaker counts. - if (failed.length && failed.length === batch.entries.length) - throw failed[0].error; - if (failed.length) return { failed }; + return batchOutcome(batch.entries.length, failed); }; function resolveString(value: unknown): string | undefined { @@ -485,6 +556,7 @@ async function applyAlias( }; // Alias uses userId (the new canonical ID) -- not full identity spread if (identity.userId) params.userId = identity.userId; + if (identity.messageId) params.messageId = identity.messageId; return { sent: call('RudderStack alias failed', (done) => sdk.alias(params, done)), diff --git a/packages/server/destinations/rudderstack/src/schemas/settings.ts b/packages/server/destinations/rudderstack/src/schemas/settings.ts index 6d7ff46d9..9536944c0 100644 --- a/packages/server/destinations/rudderstack/src/schemas/settings.ts +++ b/packages/server/destinations/rudderstack/src/schemas/settings.ts @@ -65,7 +65,17 @@ export const SettingsSchema = z .number() .int() .min(0) - .describe('Retry attempts for failed batches. Default: 3.') + .describe( + 'Retries the SDK makes for a failed request, a tenth of config.timeout apart. Default: 2, which ends within config.timeout. A value set here wins.', + ) + .optional(), + timeout: z + .number() + .int() + .positive() + .describe( + 'Time one request may take, in milliseconds. Default: a quarter of config.timeout (2500 at the 10 s default). A value set here wins.', + ) .optional(), enable: z .boolean() diff --git a/packages/server/destinations/rudderstack/src/types/index.ts b/packages/server/destinations/rudderstack/src/types/index.ts index 186e22aa2..8fda07e99 100644 --- a/packages/server/destinations/rudderstack/src/types/index.ts +++ b/packages/server/destinations/rudderstack/src/types/index.ts @@ -32,8 +32,10 @@ export interface Settings { maxInternalQueueSize?: number; /** Log level. Default: 'info'. */ logLevel?: string; - /** Retry attempts for failed batches. Default: 3. */ + /** Retries for a failed request. Default: 2, within config.timeout. */ retryCount?: number; + /** Time one request may take (ms). Default: a quarter of config.timeout. */ + timeout?: number; /** Completely disable the SDK. Default: false. */ enable?: boolean; /** Enable gzip compression. Default: true. */ diff --git a/packages/server/destinations/segment/src/__tests__/failure.test.ts b/packages/server/destinations/segment/src/__tests__/failure.test.ts index 2ffeb77f8..563e2cb16 100644 --- a/packages/server/destinations/segment/src/__tests__/failure.test.ts +++ b/packages/server/destinations/segment/src/__tests__/failure.test.ts @@ -73,4 +73,76 @@ describe('segment delivery failures', () => { await collector.command('shutdown'); }); + + it('fails a call the SDK refuses at once with its own StepError', async () => { + const refusing: SegmentAnalyticsMock = { + ...analyticsFailing(() => false), + track: () => { + throw new Error('invalid message'); + }, + }; + const { collector } = await startFlow({ + logger: { level: 'ERROR', handler: () => undefined }, + destinations: { + segment: { + code: destinationSegment, + config: { settings: { writeKey: 'test' }, batch: undefined }, + env: { analytics: refusing }, + }, + }, + }); + + const result = await collector.push(getEvent('product view')); + + expect(result.failed?.segment?.reason).toMatchObject({ + code: 'transport', + message: 'Segment track failed', + }); + + await collector.command('shutdown'); + }); + + it('says at DEBUG which mapping a declared skip found nothing for', async () => { + const lines: Array<[string, unknown]> = []; + const { collector } = await startFlow({ + logger: { + level: 'DEBUG', + handler: (_level, message, context) => lines.push([message, context]), + }, + destinations: { + segment: { + code: destinationSegment, + config: { + settings: { writeKey: 'test' }, + batch: undefined, + mapping: { + company: { + update: { + silent: true, + settings: { group: { map: { groupId: 'data.company_id' } } }, + }, + }, + }, + }, + env: { analytics: analyticsFailing(() => false) }, + }, + }, + }); + + await collector.push( + createEvent({ name: 'company update', user: { id: 'us3r' }, data: {} }), + ); + + const skips = lines.filter(([message]) => + message.includes('resolved to nothing'), + ); + expect(skips).toEqual([ + [ + 'Segment skipped group: group resolved to nothing', + expect.objectContaining({ event: 'company update' }), + ], + ]); + + await collector.command('shutdown'); + }); }); diff --git a/packages/server/destinations/segment/src/__tests__/retry.test.ts b/packages/server/destinations/segment/src/__tests__/retry.test.ts new file mode 100644 index 000000000..743c8edcd --- /dev/null +++ b/packages/server/destinations/segment/src/__tests__/retry.test.ts @@ -0,0 +1,93 @@ +import { startFlow } from '@walkeros/collector'; +import { createEvent } from '@walkeros/core'; +import { destinationSegment } from '..'; + +/** + * Segment's endpoint behind the real SDK: each request gets the next answer, + * a status or no answer at all (the request then ends when the SDK aborts + * it). + */ +function endpoint(answers: Array) { + let requests = 0; + jest.spyOn(globalThis, 'fetch').mockImplementation((_url, init) => { + const answer = answers[Math.min(requests, answers.length - 1)]; + requests++; + if (answer === 'never') + return new Promise((_resolve, reject) => + init?.signal?.addEventListener('abort', () => + reject(init.signal?.reason), + ), + ); + return Promise.resolve(new Response('{}', { status: answer })); + }); + return { requests: () => requests }; +} + +/** Run fake time until the promise settles; answers how long it took. */ +async function settle( + promise: Promise, +): Promise<{ value: T; ms: number }> { + const started = Date.now(); + let done = false; + promise.then( + () => (done = true), + () => (done = true), + ); + for (let step = 0; !done && step < 1000; step++) + await jest.advanceTimersByTimeAsync(50); + return { value: await promise, ms: Date.now() - started }; +} + +async function flowWith(timeout?: number) { + const { collector } = await startFlow({ + logger: { level: 'ERROR', handler: () => undefined }, + destinations: { + segment: { + code: destinationSegment, + config: { settings: { writeKey: 'w' }, batch: undefined, timeout }, + }, + }, + }); + return collector; +} + +describe('segment retry within config.timeout', () => { + beforeEach(() => jest.useFakeTimers()); + afterEach(() => { + jest.useRealTimers(); + jest.restoreAllMocks(); + }); + + it('delivers once Segment answers after two 503s, within the timeout', async () => { + const vendor = endpoint([503, 503, 200]); + // Three attempts with the SDK's backoff need more than the 10 s default. + const collector = await flowWith(30_000); + + const { value: result, ms } = await settle( + collector.push(createEvent({ name: 'product view' })), + ); + + expect(result.ok).toBe(true); + expect(vendor.requests()).toBe(3); + expect(ms).toBeLessThan(30_000); + }); + + it('fails timeout once when Segment never answers, before the race', async () => { + const vendor = endpoint(['never']); + const collector = await flowWith(); + + const { value: result, ms } = await settle( + collector.push(createEvent({ name: 'product view' })), + ); + + // The SDK's own timeout, not the collector's race: the package's message. + expect(result.failed?.segment?.reason).toMatchObject({ + code: 'timeout', + message: 'Segment track failed', + }); + expect(collector.status.destinations.segment.failed).toBe(1); + // Two requests of 2.5 s and the backoff between them end before 10 s. + expect(vendor.requests()).toBe(2); + expect(ms).toBeLessThan(10_000); + }); +}); diff --git a/packages/server/destinations/segment/src/examples/step.ts b/packages/server/destinations/segment/src/examples/step.ts index af04c60d6..695f2ca3e 100644 --- a/packages/server/destinations/segment/src/examples/step.ts +++ b/packages/server/destinations/segment/src/examples/step.ts @@ -29,6 +29,7 @@ export const defaultTrack: SegmentStepExample = { description: 'A walker event becomes a Segment analytics.track call with userId and anonymousId resolved from the event.', in: getEvent('product view', { + id: 'ev1700000100', timestamp: 1700000100, user: { id: 'us3r', session: 's3ss10n' }, }), @@ -36,6 +37,7 @@ export const defaultTrack: SegmentStepExample = { [ 'analytics.track', { + messageId: 'ev1700000100', userId: 'us3r', anonymousId: 's3ss10n', event: 'product view', @@ -54,6 +56,7 @@ export const mappedEventName: SegmentStepExample = { description: "A mapping renames the event so the Segment track call uses Segment's canonical 'Order Completed' name.", in: getEvent('order complete', { + id: 'ev1700000101', timestamp: 1700000101, user: { id: 'us3r', session: 's3ss10n' }, }), @@ -64,6 +67,7 @@ export const mappedEventName: SegmentStepExample = { [ 'analytics.track', { + messageId: 'ev1700000101', userId: 'us3r', anonymousId: 's3ss10n', event: 'Order Completed', @@ -83,6 +87,7 @@ export const destinationIdentify: SegmentStepExample = { description: 'Destination-level identify fires Segment analytics.identify with traits on the first push only, then the track call follows.', in: getEvent('page view', { + id: 'ev1700000102', timestamp: 1700000102, user: { id: 'us3r', session: 's3ss10n', email: 'user@example.com' }, }), @@ -101,6 +106,7 @@ export const destinationIdentify: SegmentStepExample = { [ 'analytics.identify', { + messageId: 'ev1700000102-identify', userId: 'us3r', anonymousId: 's3ss10n', traits: { email: 'user@example.com' }, @@ -110,6 +116,7 @@ export const destinationIdentify: SegmentStepExample = { [ 'analytics.track', { + messageId: 'ev1700000102', userId: 'us3r', anonymousId: 's3ss10n', event: 'page view', @@ -129,6 +136,7 @@ export const userLoginIdentify: SegmentStepExample = { 'A user login fires only Segment analytics.identify with the userId and traits, skipping the track.', in: createEvent({ name: 'user login', + id: 'ev1700000103', timestamp: 1700000103, user: { id: 'us3r', session: 's3ss10n' }, data: { @@ -159,6 +167,7 @@ export const userLoginIdentify: SegmentStepExample = { [ 'analytics.identify', { + messageId: 'ev1700000103-identify', userId: 'new-user-123', anonymousId: 's3ss10n', traits: { email: 'user@acme.com', name: 'Jane Doe', plan: 'premium' }, @@ -177,6 +186,7 @@ export const companyGroup: SegmentStepExample = { 'A company update fires Segment analytics.group with groupId and group traits for account-level tracking.', in: createEvent({ name: 'company update', + id: 'ev1700000104', timestamp: 1700000104, user: { id: 'us3r', session: 's3ss10n' }, data: { @@ -207,6 +217,7 @@ export const companyGroup: SegmentStepExample = { [ 'analytics.group', { + messageId: 'ev1700000104-group', userId: 'us3r', anonymousId: 's3ss10n', groupId: 'comp-456', @@ -226,6 +237,7 @@ export const pageView: SegmentStepExample = { description: 'A page view fires Segment analytics.page with category, name, and properties instead of a generic track.', in: getEvent('page view', { + id: 'ev1700000105', timestamp: 1700000105, user: { id: 'us3r', session: 's3ss10n' }, data: { @@ -254,6 +266,7 @@ export const pageView: SegmentStepExample = { [ 'analytics.page', { + messageId: 'ev1700000105-page', userId: 'us3r', anonymousId: 's3ss10n', category: 'docs', @@ -274,6 +287,7 @@ export const screenView: SegmentStepExample = { 'A screen view fires Segment analytics.screen with name, category, and properties for mobile app tracking.', in: createEvent({ name: 'screen view', + id: 'ev1700000106', timestamp: 1700000106, user: { id: 'us3r', session: 's3ss10n' }, data: { @@ -302,6 +316,7 @@ export const screenView: SegmentStepExample = { [ 'analytics.screen', { + messageId: 'ev1700000106-screen', userId: 'us3r', anonymousId: 's3ss10n', name: 'Welcome', @@ -321,6 +336,7 @@ export const anonymousOnly: SegmentStepExample = { description: 'When no userId is resolved Segment accepts a track call keyed solely by anonymousId.', in: getEvent('product view', { + id: 'ev1700000107', timestamp: 1700000107, user: { session: 's3ss10n' }, }), @@ -331,6 +347,7 @@ export const anonymousOnly: SegmentStepExample = { [ 'analytics.track', { + messageId: 'ev1700000107', anonymousId: 's3ss10n', event: 'product view', properties: {}, @@ -349,6 +366,7 @@ export const consentForwarding: SegmentStepExample = { description: 'Walker consent keys are mapped to Segment categoryPreferences on the analytics context for downstream filtering.', in: getEvent('product view', { + id: 'ev1700000108', timestamp: 1700000108, user: { id: 'us3r', session: 's3ss10n' }, consent: { analytics: true, marketing: true }, @@ -363,6 +381,7 @@ export const consentForwarding: SegmentStepExample = { [ 'analytics.track', { + messageId: 'ev1700000108', userId: 'us3r', anonymousId: 's3ss10n', event: 'product view', diff --git a/packages/server/destinations/segment/src/index.ts b/packages/server/destinations/segment/src/index.ts index 13abf108c..90c999e04 100644 --- a/packages/server/destinations/segment/src/index.ts +++ b/packages/server/destinations/segment/src/index.ts @@ -1,11 +1,20 @@ import type { Destination } from './types'; import { StepError } from '@walkeros/core'; +import { retryBudget } from '@walkeros/server-core'; import { getConfig } from './config'; import { push, pushBatch } from './push'; // Types export * as DestinationSegment from './types'; +/** + * The SDK's longest total wait before `retries` retries: its backoff waits up + * to 1 s times two to the retry's power (1 to 2 s, then 2 to 4 s). + */ +function segmentBackoff(retries: number): number { + return 1000 * (2 ** (retries + 1) - 2); +} + export const destinationSegment: Destination = { type: 'segment', @@ -35,8 +44,18 @@ export const destinationSegment: Destination = { } = settings; // Each push and each batch flushes the SDK and waits until Segment - // delivered its calls. - settings._analytics = new Analytics({ writeKey, ...sdkOptions }); + // delivered its calls. The SDK's own retry ends within config.timeout: + // a quarter of it per request, the retries its backoff leaves room + // for, rate-limit waits bounded the same way. A value set in the + // settings wins. + const budget = retryBudget(partialConfig.timeout, segmentBackoff); + settings._analytics = new Analytics({ + writeKey, + httpRequestTimeout: budget.attempt, + maxRetries: budget.retries, + maxRateLimitDuration: budget.attempt / 1000, + ...sdkOptions, + }); } catch (err) { throw new StepError('Failed to initialize the Segment SDK', { code: 'not-ready', diff --git a/packages/server/destinations/segment/src/push.ts b/packages/server/destinations/segment/src/push.ts index 809dcc800..c4784b3d3 100644 --- a/packages/server/destinations/segment/src/push.ts +++ b/packages/server/destinations/segment/src/push.ts @@ -1,4 +1,4 @@ -import type { Collector, Destination, WalkerOS } from '@walkeros/core'; +import type { Collector, Destination, Logger, WalkerOS } from '@walkeros/core'; import type { PushFn, PushBatchFn, @@ -16,24 +16,71 @@ import { isThenable, StepError, } from '@walkeros/core'; +import { batchOutcome } from '@walkeros/server-core'; /** * Hand one call to the SDK. The promise settles once Segment delivered it: * the SDK answers through the callback, with the reason when the delivery - * failed; an injected client may answer with a promise instead. + * failed; an injected client may answer with a promise instead. A call the + * SDK refuses at once fails the same way. The promise is marked handled at + * once, since its event's outcome reads it only after the flush. */ function call( message: string, send: (done: SegmentCallback) => unknown, ): Promise { - return new Promise((resolve, reject) => { - const fail = (error: unknown) => - reject(new StepError(message, { code: 'transport', cause: error })); - const answer: unknown = send((error) => - error === undefined || error === null ? resolve() : fail(error), - ); + const sent = new Promise((resolve, reject) => { + const fail = (error: unknown) => reject(failure(message, error)); + let answer: unknown; + try { + answer = send((error) => + error === undefined || error === null ? resolve() : fail(error), + ); + } catch (error) { + fail(error); + return; + } if (isThenable(answer)) answer.then(() => resolve(), fail); }); + sent.catch(() => undefined); + return sent; +} + +/** + * A failed delivery: `timeout` when the SDK's last request timed out (it + * aborts a request after `httpRequestTimeout`), else `transport`. + */ +function failure(message: string, error: unknown): StepError { + // A DOMException, from another realm in some runtimes: read by its name. + const name: unknown = + typeof error === 'object' && error !== null + ? Reflect.get(error, 'name') + : undefined; + const timedOut = name === 'AbortError' || name === 'TimeoutError'; + return new StepError(message, { + code: timedOut ? 'timeout' : 'transport', + cause: error, + }); +} + +/** + * The id Segment deduplicates a call by: the event id for the track call, + * the event id and the call type for the others, so a resent event is + * dropped by Segment and the calls of one event are not. + */ +function messageIdOf(event: WalkerOS.Event, type?: string): string | undefined { + if (!event.id) return undefined; + return type ? `${event.id}-${type}` : event.id; +} + +/** The identity of one call, with the call's own messageId. */ +function withMessageId( + identity: Record, + event: WalkerOS.Event, + type: string, +): Record { + const messageId = messageIdOf(event, type); + return messageId ? { ...identity, messageId } : identity; } /** @@ -49,6 +96,22 @@ function flush(settings: Partial, sdk: SegmentAnalyticsMock) { return flushing; } +/** + * A configured mapping that resolved to nothing skips its call: a declared + * skip, said once at DEBUG with the event and the mapping key, never the + * value, so a wrong mapping path does not look like an event without data. + */ +function skipped( + logger: Logger.Instance, + event: WalkerOS.Event, + call: string, + key: string, +): void { + logger.debug(`Segment skipped ${call}: ${key} resolved to nothing`, { + event: event.name, + }); +} + /** The settled calls of one event: the first failure, if any. */ async function delivered(calls: Promise[]>) { for (const result of await calls) @@ -67,6 +130,7 @@ async function enqueue( settings: Partial, sdk: SegmentAnalyticsMock, collector: Collector.Instance, + logger: Logger.Instance, ): Promise[]> { const state: RuntimeState = settings._state || {}; settings._state = state; @@ -109,13 +173,14 @@ async function enqueue( calls.push( applyIdentify( sdk, - resolved as Record, - identity, + resolved, + withMessageId(identity, event, 'identify'), state, timestamp, context, ), ); + else skipped(logger, event, 'identify', 'identify'); } // 3. Group -- rule-level overrides destination-level @@ -124,50 +189,49 @@ async function enqueue( const resolved = await getMappingValue(event, groupMapping, { collector, }); - if (isObject(resolved)) + if (isObject(resolved) && isString(resolved.groupId)) calls.push( applyGroup( sdk, - resolved as Record, - identity, + resolved, + withMessageId(identity, event, 'group'), state, timestamp, context, ), ); + else skipped(logger, event, 'group', 'group'); } // 4. Page (the call is not awaited here: it is delivered at the flush) - if (mappingSettings.page !== undefined) - calls.push( - ( - await applyPage( - sdk, - mappingSettings.page, - event, - identity, - collector, - timestamp, - context, - ) - ).sent, + if (mappingSettings.page !== undefined) { + const { sent } = await applyPage( + sdk, + mappingSettings.page, + event, + withMessageId(identity, event, 'page'), + collector, + timestamp, + context, ); + if (sent) calls.push(sent); + else skipped(logger, event, 'page', 'page'); + } // 5. Screen - if (mappingSettings.screen !== undefined) - calls.push( - ( - await applyScreen( - sdk, - mappingSettings.screen, - event, - identity, - collector, - timestamp, - context, - ) - ).sent, + if (mappingSettings.screen !== undefined) { + const { sent } = await applyScreen( + sdk, + mappingSettings.screen, + event, + withMessageId(identity, event, 'screen'), + collector, + timestamp, + context, ); + if (sent) calls.push(sent); + else skipped(logger, event, 'screen', 'screen'); + } // 6. Track (unless silent: true) if (rule?.silent !== true) { @@ -180,6 +244,8 @@ async function enqueue( properties, timestamp, }; + const messageId = messageIdOf(event); + if (messageId) params.messageId = messageId; if (context) params.context = context; calls.push(call('Segment track failed', (done) => sdk.track(params, done))); @@ -204,11 +270,19 @@ function sdkOf( /** One event: its calls handed to the SDK, sent, and their deliveries awaited. */ export const push: PushFn = async function ( event, - { config, rule, data, collector, env }, + { config, rule, data, collector, env, logger }, ) { const settings = config.settings ?? {}; const sdk = sdkOf(config.settings, env); - const calls = await enqueue(event, rule, data, settings, sdk, collector); + const calls = await enqueue( + event, + rule, + data, + settings, + sdk, + collector, + logger, + ); if (calls.length) { const settled = Promise.allSettled(calls); await flush(settings, sdk); @@ -223,7 +297,7 @@ export const push: PushFn = async function ( */ export const pushBatch: PushBatchFn = async function ( batch, - { config, collector, env }, + { config, collector, env, logger }, ) { const settings = config.settings ?? {}; const sdk = sdkOf(config.settings, env); @@ -242,6 +316,7 @@ export const pushBatch: PushBatchFn = async function ( settings, sdk, collector, + logger, ); outcomes.push({ settled: Promise.allSettled(calls) }); } catch (error) { @@ -260,10 +335,7 @@ export const pushBatch: PushBatchFn = async function ( ); } - // Nothing landed: a whole-batch failure, which the breaker counts. - if (failed.length && failed.length === batch.entries.length) - throw failed[0].error; - if (failed.length) return { failed }; + return batchOutcome(batch.entries.length, failed); }; function resolveString(value: unknown): string | undefined { diff --git a/packages/server/destinations/segment/src/schemas/settings.ts b/packages/server/destinations/segment/src/schemas/settings.ts index 31f830db9..d761873cb 100644 --- a/packages/server/destinations/segment/src/schemas/settings.ts +++ b/packages/server/destinations/segment/src/schemas/settings.ts @@ -46,13 +46,17 @@ export const SettingsSchema = z .number() .int() .min(0) - .describe('Retry attempts for failed batches. Default: 3.') + .describe( + "Retries the SDK makes for a failed request. Default: as many as fit config.timeout with the SDK's backoff (one at the 10 s default, two from 24 s). A value set here wins.", + ) .optional(), httpRequestTimeout: z .number() .int() .positive() - .describe('HTTP request timeout in milliseconds. Default: 10000.') + .describe( + 'Time one request may take, in milliseconds. Default: a quarter of config.timeout (2500 at the 10 s default). A value set here wins.', + ) .optional(), disable: z .boolean() diff --git a/packages/server/destinations/segment/src/types/index.ts b/packages/server/destinations/segment/src/types/index.ts index a488a57eb..d1460a3ef 100644 --- a/packages/server/destinations/segment/src/types/index.ts +++ b/packages/server/destinations/segment/src/types/index.ts @@ -28,9 +28,9 @@ export interface Settings { flushAt?: number; /** Max milliseconds before auto-flush. Default: 10000. */ flushInterval?: number; - /** Retry attempts for failed batches. Default: 3. */ + /** Retries for a failed request. Default: as many as fit config.timeout. */ maxRetries?: number; - /** HTTP request timeout (ms). Default: 10000. */ + /** Time one request may take (ms). Default: a quarter of config.timeout. */ httpRequestTimeout?: number; /** Completely disable the SDK. Default: false. */ disable?: boolean; diff --git a/packages/server/destinations/sqlite/src/__tests__/init.test.ts b/packages/server/destinations/sqlite/src/__tests__/init.test.ts index 048e5aac3..fe4aa7bac 100644 --- a/packages/server/destinations/sqlite/src/__tests__/init.test.ts +++ b/packages/server/destinations/sqlite/src/__tests__/init.test.ts @@ -62,17 +62,17 @@ const initOf = destinationSQLite.init; describe('init', () => { it('stays offline: no query, no CREATE TABLE and no prepare', async () => { - const logger = makeLogger(); const { client, calls } = makeClientSpy(); const query = jest.fn(client.query); client.query = query; - await destinationSQLite.init({ + await initOf({ config: { settings: { sqlite: { url: ':memory:', schema: 'auto' } } }, env: { client }, - logger, + logger: createMockLogger(), id: 'test', - } as never); + collector: createMockContext().collector, + }); expect(calls).toEqual([]); expect(query).not.toHaveBeenCalled(); @@ -182,16 +182,16 @@ describe('init', () => { }); it('does NOT auto-create when setup is unset and table missing; hard-fails with setup hint', async () => { - const logger = makeLogger(); const { client } = makeClientSpy(); // Default makeClientSpy.query returns [], so tableExists -> false. - const result = await destinationSQLite.init({ + const result = await initOf({ config: { settings: { sqlite: { url: ':memory:' } } }, env: { client }, - logger, + logger: createMockLogger(), id: 'sqlite', - } as never); + collector: createMockContext().collector, + }); await expect(firstPush(result)).rejects.toMatchObject({ name: 'StepError', From 2e19d35ba00fab426461107f79de68665600e981 Mon Sep 17 00:00:00 2001 From: alexanderkirtzel Date: Wed, 21 Oct 2026 08:59:06 +0000 Subject: [PATCH 32/41] web skips amplitude --- packages/web/destinations/amplitude/README.md | 7 +- .../amplitude/src/__tests__/index.test.ts | 73 +++++- .../src/__tests__/realSdkPlugins.test.ts | 74 +++++++ .../src/__tests__/stepExamples.test.ts | 11 +- .../amplitude/src/examples/env.ts | 6 +- .../amplitude/src/examples/step.ts | 16 +- .../web/destinations/amplitude/src/index.ts | 207 ++++++++++-------- .../amplitude/src/schemas/settings.ts | 140 ++++++------ .../destinations/amplitude/src/types/index.ts | 12 +- .../clarity/src/__tests__/index.test.ts | 42 +++- .../web/destinations/clarity/src/index.ts | 53 +++-- .../fullstory/src/__tests__/index.test.ts | 42 +++- .../web/destinations/fullstory/src/index.ts | 42 ++-- .../hotjar/src/__tests__/stepExamples.test.ts | 42 +++- packages/web/destinations/hotjar/src/index.ts | 57 +++-- .../src/__tests__/stepExamples.test.ts | 43 +++- .../web/destinations/mixpanel/src/index.ts | 69 ++++-- .../src/__tests__/stepExamples.test.ts | 42 +++- .../web/destinations/pinterest/src/index.ts | 25 ++- .../src/__tests__/stepExamples.test.ts | 43 +++- .../web/destinations/posthog/src/index.ts | 45 ++-- .../segment/src/__tests__/index.test.ts | 49 ++++- .../web/destinations/segment/src/index.ts | 69 ++++-- .../tiktok/src/__tests__/stepExamples.test.ts | 42 +++- packages/web/destinations/tiktok/src/index.ts | 40 +++- 25 files changed, 950 insertions(+), 341 deletions(-) create mode 100644 packages/web/destinations/amplitude/src/__tests__/realSdkPlugins.test.ts diff --git a/packages/web/destinations/amplitude/README.md b/packages/web/destinations/amplitude/README.md index 6094d9be1..d1275a5fc 100644 --- a/packages/web/destinations/amplitude/README.md +++ b/packages/web/destinations/amplitude/README.md @@ -48,10 +48,9 @@ npm install @walkeros/web-destination-amplitude ## Plugins With only `apiKey` and the analytics options, only Amplitude analytics starts. -Setting `sessionReplay`, `experiment` (with a `deploymentKey`) or `engagement` -starts the plugins through `@amplitude/unified`, which sets up all three -together: enabling one also sets up Session Replay, Experiment and Guides & -Surveys. +Each plugin starts only from its own setting: `sessionReplay` starts Session +Replay, `experiment` with a `deploymentKey` starts Experiment, and `engagement` +starts Guides & Surveys, which loads its SDK from cdn.amplitude.com. ## Documentation diff --git a/packages/web/destinations/amplitude/src/__tests__/index.test.ts b/packages/web/destinations/amplitude/src/__tests__/index.test.ts index 8bb1e4abf..1eae7fbc9 100644 --- a/packages/web/destinations/amplitude/src/__tests__/index.test.ts +++ b/packages/web/destinations/amplitude/src/__tests__/index.test.ts @@ -1,5 +1,6 @@ import { startFlow } from '@walkeros/collector'; -import { clone, getEvent } from '@walkeros/core'; +import type { Logger } from '@walkeros/core'; +import { clone, createEvent, getEvent, Level } from '@walkeros/core'; import { destinationAmplitude } from '..'; import { examples } from '../dev'; import type { Settings } from '../types'; @@ -13,9 +14,9 @@ async function initCalls(settings: Settings): Promise { calls.push('init'); return { promise: Promise.resolve() }; }; - amplitude.initAll = () => { - calls.push('initAll'); - return Promise.resolve(); + amplitude.add = (plugin) => { + calls.push(`add ${plugin.name}`); + return { promise: Promise.resolve() }; }; const { elb } = await startFlow({ destinations: { @@ -31,11 +32,63 @@ describe('amplitude destination', () => { expect(await initCalls({ apiKey: 'key' })).toEqual(['init']); }); - it.each<[string, Partial]>([ - ['sessionReplay', { sessionReplay: { sampleRate: 1 } }], - ['experiment', { experiment: { deploymentKey: 'deployment' } }], - ['engagement', { engagement: true }], - ])('starts the plugins when %s is set', async (_name, plugin) => { - expect(await initCalls({ apiKey: 'key', ...plugin })).toEqual(['initAll']); + it.each<[Partial, string]>([ + [ + { sessionReplay: { sampleRate: 1 } }, + '@amplitude/plugin-session-replay-browser', + ], + [ + { experiment: { deploymentKey: 'deployment' } }, + '@amplitude/experiment-analytics-plugin', + ], + [{ engagement: true }, '@amplitude/engagement-browser'], + ])( + 'starts analytics and only the plugin %j enables', + async (plugin, name) => { + expect(await initCalls({ apiKey: 'key', ...plugin })).toEqual([ + 'init', + `add ${name}`, + ]); + }, + ); + + it('writes one DEBUG line naming the event and the mapping key for a declared skip', async () => { + const lines: Array<[Level, string, unknown]> = []; + const handler: Logger.Handler = (level, message, context) => { + lines.push([level, message, context]); + }; + const { elb } = await startFlow({ + logger: { level: 'DEBUG', handler }, + destinations: { + amplitude: { + code: destinationAmplitude, + config: { + settings: { apiKey: 'key' }, + env: clone(examples.env.push), + mapping: { + company: { + update: { + silent: true, + settings: { + group: { + map: { type: { value: 'company' }, name: 'data.none' }, + }, + }, + }, + }, + }, + }, + }, + }, + }); + + expect((await elb(createEvent({ name: 'company update' }))).ok).toBe(true); + expect(lines.filter(([, message]) => message.includes('skipped'))).toEqual([ + [ + Level.DEBUG, + 'Amplitude skipped setGroup: group resolved to nothing', + expect.objectContaining({ event: 'company update' }), + ], + ]); }); }); diff --git a/packages/web/destinations/amplitude/src/__tests__/realSdkPlugins.test.ts b/packages/web/destinations/amplitude/src/__tests__/realSdkPlugins.test.ts new file mode 100644 index 000000000..a94f65424 --- /dev/null +++ b/packages/web/destinations/amplitude/src/__tests__/realSdkPlugins.test.ts @@ -0,0 +1,74 @@ +import type { Logger } from '@walkeros/core'; +import type * as Unified from '@amplitude/unified'; +import type { UnifiedClient } from '@amplitude/unified'; +import { startFlow } from '@walkeros/collector'; +import { getEvent, Level } from '@walkeros/core'; +import { destinationAmplitude } from '..'; + +// The real @amplitude/unified, with the client the destination creates kept +// and the name of each plugin it adds. A file of its own: the destination +// keeps one client per page. +const mockClients: UnifiedClient[] = []; +const mockAdded: string[] = []; +jest.mock('@amplitude/unified', () => { + const actual = jest.requireActual('@amplitude/unified'); + return { + ...actual, + createInstance: (): UnifiedClient => { + const client = actual.createInstance(); + mockClients.push(client); + return { + ...client, + add: (plugin) => { + mockAdded.push(plugin.name ?? ''); + return client.add(plugin); + }, + }; + }, + }; +}); + +describe('amplitude plugins on the real SDK', () => { + const fetchBefore = globalThis.fetch; + const requests: string[] = []; + + beforeEach(() => { + jest.useRealTimers(); + globalThis.fetch = async (input) => { + requests.push(String(input)); + return new Response('{}', { status: 200 }); + }; + }); + + afterEach(() => { + globalThis.fetch = fetchBefore; + }); + + it('starts Session Replay alone: no Experiment, no Guides & Surveys, no engagement script', async () => { + const levels: Logger.Level[] = []; + const { elb } = await startFlow({ + logger: { handler: (level) => levels.push(level) }, + destinations: { + amplitude: { + code: destinationAmplitude, + config: { + settings: { + apiKey: 'a1b2c3d4e5f6789012345678abcdef12', + sessionReplay: { sampleRate: 0 }, + }, + }, + }, + }, + }); + + expect((await elb(getEvent('page view'))).ok).toBe(true); + + expect(levels.filter((level) => level === Level.ERROR)).toEqual([]); + expect(mockClients).toHaveLength(1); + const [client] = mockClients; + expect(mockAdded).toEqual(['@amplitude/plugin-session-replay-browser']); + expect(client.experiment()).toBeUndefined(); + expect(document.querySelectorAll('script')).toHaveLength(0); + expect(requests.filter((url) => url.includes('engagement'))).toEqual([]); + }); +}); diff --git a/packages/web/destinations/amplitude/src/__tests__/stepExamples.test.ts b/packages/web/destinations/amplitude/src/__tests__/stepExamples.test.ts index 90121fa7b..3365390f5 100644 --- a/packages/web/destinations/amplitude/src/__tests__/stepExamples.test.ts +++ b/packages/web/destinations/amplitude/src/__tests__/stepExamples.test.ts @@ -123,9 +123,10 @@ function spyEnv(env: Env): { env: Env; collected: () => CallRecord[] } { calls.push(['amplitude.init', ...args]); return { promise: Promise.resolve() }; }, - initAll: (async (...args: unknown[]) => { - calls.push(['amplitude.initAll', ...args]); - }) as NonNullable['initAll'], + add: (...args: unknown[]) => { + calls.push(['amplitude.add', ...args]); + return { promise: Promise.resolve() }; + }, track: ((eventType: string, props?: Record) => { calls.push(['amplitude.track', eventType, props ?? {}]); }) as NonNullable['track'], @@ -216,9 +217,7 @@ describe('amplitude destination — step examples', () => { // Drop init — every example triggers init once; it's not part of `out`. const expected = (example.out ?? []) as ReadonlyArray; - const actual = collected().filter( - ([path]) => path !== 'amplitude.initAll' && path !== 'amplitude.init', - ); + const actual = collected().filter(([path]) => path !== 'amplitude.init'); expect(actual).toEqual(expected); }, diff --git a/packages/web/destinations/amplitude/src/examples/env.ts b/packages/web/destinations/amplitude/src/examples/env.ts index e71d10df1..5b35cd298 100644 --- a/packages/web/destinations/amplitude/src/examples/env.ts +++ b/packages/web/destinations/amplitude/src/examples/env.ts @@ -113,7 +113,7 @@ class MockRevenue implements RevenueInstance { export const init: Env | undefined = { amplitude: { init: noopPromise, - initAll: () => Promise.resolve(), + add: noopPromise, track: noop, identify: noop, revenue: noop, @@ -136,7 +136,7 @@ export const init: Env | undefined = { export const push: Env = { amplitude: { init: noopPromise, - initAll: () => Promise.resolve(), + add: noopPromise, track: noop, identify: noop, revenue: noop, @@ -155,7 +155,7 @@ export const push: Env = { /** Simulation tracking paths for CLI --simulate. */ export const simulation = [ 'call:amplitude.init', - 'call:amplitude.initAll', + 'call:amplitude.add', 'call:amplitude.track', 'call:amplitude.identify', 'call:amplitude.revenue', diff --git a/packages/web/destinations/amplitude/src/examples/step.ts b/packages/web/destinations/amplitude/src/examples/step.ts index cf525484c..e691c446b 100644 --- a/packages/web/destinations/amplitude/src/examples/step.ts +++ b/packages/web/destinations/amplitude/src/examples/step.ts @@ -447,22 +447,14 @@ export const silentRule: AmplitudeStepExample = { /** * The silent group rule of groupAssignmentWithProperties on an update - * without data: no group name resolves and the rule suppresses the track, - * so nothing is sent. + * without data: no group resolves, so nothing is sent. A declared skip: an + * unmet mapping, not a failure. */ export const groupAssignmentWithoutData: AmplitudeStepExample = { public: false, description: - 'A silent rule whose group mapping finds no company sends nothing, so the push fails with missing-input instead of counting as delivered.', + 'A silent rule whose group mapping finds no company sends nothing: an unmet mapping, not a failure, so the push returns normally.', in: { name: 'company update', entity: 'company', action: 'update' }, - mapping: { - silent: true, - settings: { - group: { - map: { type: { value: 'company' }, name: 'data.company' }, - }, - }, - }, + mapping: groupAssignmentWithProperties.mapping, out: [], - outcome: { status: 'failed', code: 'missing-input' }, }; diff --git a/packages/web/destinations/amplitude/src/index.ts b/packages/web/destinations/amplitude/src/index.ts index b4b4a797e..5e736cc09 100644 --- a/packages/web/destinations/amplitude/src/index.ts +++ b/packages/web/destinations/amplitude/src/index.ts @@ -1,4 +1,4 @@ -import type { Collector, WalkerOS } from '@walkeros/core'; +import type { Collector, Logger, WalkerOS } from '@walkeros/core'; import { getGrantedConsent, getMappingValue, @@ -8,9 +8,14 @@ import { isObject, isString, StepError, + stepCall, } from '@walkeros/core'; import * as amplitudeSDK from '@amplitude/unified'; +import { sessionReplayPlugin } from '@amplitude/plugin-session-replay-browser'; +import { experimentPlugin } from '@amplitude/plugin-experiment-browser'; +import { plugin as engagementPlugin } from '@amplitude/engagement-browser'; import type { + AmplitudePlugin, AmplitudeSDK, Destination, IdentifyInstance, @@ -29,8 +34,8 @@ let real: RealSDK | undefined; /** * The real SDK: one @amplitude/unified client, as the module's own default - * client is one per page. Created on first use; its `init` sets up analytics - * alone, `initAll` adds the three plugins. + * client is one per page. Created on first use; its `init` sets up analytics, + * `add` one plugin at a time. */ function realAmplitude(): RealSDK { if (real) return real; @@ -39,7 +44,7 @@ function realAmplitude(): RealSDK { // The unified client forwards three arguments, and with three the // analytics SDK reads the second as a user id: the options go third. init: (apiKey, options) => client.init(apiKey, undefined, options), - initAll: (apiKey, options) => client.initAll(apiKey, options), + add: (plugin) => client.add(plugin), track: (eventType, eventProperties) => client.track(eventType, eventProperties), identify: (identify) => client.identify(identify), @@ -72,7 +77,7 @@ function isAmplitudeSDK(value: unknown): value is AmplitudeSDK { isObject(value) && [ 'init', - 'initAll', + 'add', 'track', 'identify', 'revenue', @@ -89,27 +94,31 @@ function isAmplitudeSDK(value: unknown): value is AmplitudeSDK { ); } +const send = (fn: () => unknown) => + stepCall(fn, 'transport', 'Amplitude call failed'); + /** - * Waits for one SDK call: its promise, or the `promise` an AmplitudeReturn - * carries. A throw or a rejection becomes a StepError. + * A configured mapping that resolved to nothing skips its call: a declared + * skip, said once at DEBUG with the event and the mapping key, never the + * value, so a wrong mapping path does not look like an event without data. */ -async function call( - fn: () => unknown, - code: string, - message: string, - settle = false, -): Promise { - try { - const result = await fn(); - if (settle && isObject(result) && result.promise instanceof Promise) - await result.promise; - } catch (error) { - throw new StepError(message, { code, cause: error }); - } +function skipped( + logger: Logger.Instance, + event: WalkerOS.Event, + call: string, + key: string, +): void { + logger.debug(`Amplitude skipped ${call}: ${key} resolved to nothing`, { + event: event.name, + }); } -const send = (fn: () => unknown) => - call(fn, 'transport', 'Amplitude call failed'); +/** Waits for a call and the `promise` its AmplitudeReturn carries. */ +async function settled(fn: () => unknown): Promise { + const result = await fn(); + if (isObject(result) && result.promise instanceof Promise) + await result.promise; +} /** * Deterministic string → positive integer hash (djb2). @@ -134,7 +143,7 @@ async function applyIdentitySetters< R extends RevenueInstance, >( amp: AmplitudeSDK, - sent: (fn: () => unknown) => Promise, + send: (fn: () => unknown) => Promise, resolved: Record, lastIdentity: RuntimeState['lastIdentity'] = {}, ): Promise> { @@ -143,10 +152,10 @@ async function applyIdentitySetters< if ('user' in resolved) { const user = resolved.user; if (user === null || user === '') { - await sent(() => amp.setUserId(undefined)); + await send(() => amp.setUserId(undefined)); delete updated.user; } else if (isString(user) && user !== lastIdentity.user) { - await sent(() => amp.setUserId(user)); + await send(() => amp.setUserId(user)); updated.user = user; } } @@ -154,7 +163,7 @@ async function applyIdentitySetters< if ('device' in resolved) { const device = resolved.device; if (isString(device) && device !== lastIdentity.device) { - await sent(() => amp.setDeviceId(device)); + await send(() => amp.setDeviceId(device)); updated.device = device; } } @@ -174,7 +183,7 @@ async function applyIdentitySetters< ? hashStringToInt(session) : undefined; if (asNumber !== undefined && asNumber !== lastIdentity.session) { - await sent(() => amp.setSessionId(asNumber)); + await send(() => amp.setSessionId(asNumber)); updated.session = asNumber; } } @@ -264,7 +273,7 @@ function buildRevenue( return rev; } -/** Starts the SDK: analytics alone, or with the plugins the settings enable. */ +/** Starts analytics, then each plugin the settings enable, in that order. */ async function start( amp: AmplitudeSDK, settings: Settings, @@ -285,35 +294,46 @@ async function start( ...browserOptions, }; - // Session Replay, Experiment and Guides & Surveys are opt-in: without any - // of them only analytics starts. @amplitude/unified sets up all three - // together, so one enabled plugin brings the other two. - const plugins = Boolean( - sessionReplay || experiment?.deploymentKey || engagement, + await stepCall( + () => settled(() => amp.init(apiKey, analyticsOptions)), + 'vendor-not-ready', + 'Amplitude init failed', ); - if (!plugins) - return call( - () => amp.init(apiKey, analyticsOptions), + // Session Replay, Experiment and Guides & Surveys are opt-in, each on its + // own: only the ones the settings enable start, on the analytics server + // zone and instance. + const { serverZone, instanceName } = analyticsOptions; + const shared = { + ...(serverZone ? { serverZone } : {}), + ...(instanceName ? { instanceName } : {}), + }; + const plugins: Array<[string, AmplitudePlugin]> = []; + if (sessionReplay) + plugins.push([ + 'Session Replay', + sessionReplayPlugin({ ...sessionReplay, ...shared }), + ]); + if (experiment?.deploymentKey) + plugins.push([ + 'Experiment', + experimentPlugin({ ...experiment, ...shared }), + ]); + if (engagement) + plugins.push([ + 'Guides & Surveys', + engagementPlugin({ + ...(engagement === true ? {} : engagement), + ...shared, + }), + ]); + + for (const [name, plugin] of plugins) + await stepCall( + () => settled(() => amp.add(plugin)), 'vendor-not-ready', - 'Amplitude init failed', - true, + `Amplitude ${name} failed to start`, ); - - return call( - () => - amp.initAll(apiKey, { - serverZone: analyticsOptions.serverZone, - analytics: analyticsOptions, - ...(sessionReplay ? { sessionReplay } : {}), - ...(experiment?.deploymentKey ? { experiment } : {}), - ...(engagement - ? { engagement: engagement === true ? undefined : engagement } - : {}), - }), - 'vendor-not-ready', - 'Amplitude init failed', - ); } async function pushTo( @@ -324,13 +344,9 @@ async function pushTo( rule: { name?: unknown; silent?: boolean } | undefined, data: unknown, collector: Collector.Instance, + logger: Logger.Instance, ): Promise { const state: RuntimeState = settings._state || {}; - let calls = 0; - const sent = (fn: () => unknown) => { - calls++; - return send(fn); - }; // 1. Reset - fires first so subsequent identity calls start clean. if (mappingSettings.reset !== undefined) { @@ -341,7 +357,7 @@ async function pushTo( collector, }); if (resolved) { - await sent(() => amp.reset()); + await send(() => amp.reset()); state.lastIdentity = {}; } } @@ -357,18 +373,18 @@ async function pushTo( identifyResolved = resolved; state.lastIdentity = await applyIdentitySetters( amp, - sent, + send, resolved, state.lastIdentity, ); - } + } else skipped(logger, event, 'identify', 'identify'); } // 3. Identify operations - fires amplitude.identify() if the resolved // object has any op keys (set, setOnce, add, ...). if (identifyResolved) { const id = buildIdentify(amp, identifyResolved); - if (id) await sent(() => amp.identify(id)); + if (id) await send(() => amp.identify(id)); } // 4. Group assignment @@ -376,11 +392,10 @@ async function pushTo( const resolved = await getMappingValue(event, mappingSettings.group, { collector, }); - if (isObject(resolved)) { - const { type, name } = resolved; - if (isString(type) && (isString(name) || isStringArray(name))) - await sent(() => amp.setGroup(type, name)); - } + const { type, name } = isObject(resolved) ? resolved : {}; + if (isString(type) && (isString(name) || isStringArray(name))) + await send(() => amp.setGroup(type, name)); + else skipped(logger, event, 'setGroup', 'group'); } // 5. Group identify (properties on the group) @@ -390,13 +405,14 @@ async function pushTo( mappingSettings.groupIdentify, { collector }, ); - if (isObject(resolved)) { - const { type, name } = resolved; - if (isString(type) && isString(name)) { - const id = buildIdentify(amp, resolved); - if (id) await sent(() => amp.groupIdentify(type, name, id)); - } - } + const { type, name } = isObject(resolved) ? resolved : {}; + const id = + isObject(resolved) && isString(type) && isString(name) + ? buildIdentify(amp, resolved) + : undefined; + if (id && isString(type) && isString(name)) + await send(() => amp.groupIdentify(type, name, id)); + else skipped(logger, event, 'groupIdentify', 'groupIdentify'); } // 6. Revenue - single object or array from loop → N revenue() calls. @@ -404,36 +420,27 @@ async function pushTo( const resolved = await getMappingValue(event, mappingSettings.revenue, { collector, }); - const items = isArray(resolved) ? resolved : [resolved]; - for (const item of items) { + let sent = 0; + for (const item of isArray(resolved) ? resolved : [resolved]) { if (!isObject(item)) continue; const revenue = buildRevenue(amp, item); - await sent(() => amp.revenue(revenue)); + await send(() => amp.revenue(revenue)); + sent++; } + if (!sent) skipped(logger, event, 'revenue', 'revenue'); } // 7. Default track if (rule?.silent !== true) { const eventType = isString(rule?.name) ? rule.name : event.name; const eventProperties = isObject(data) ? data : {}; - await sent(() => amp.track(eventType, eventProperties)); + await send(() => amp.track(eventType, eventProperties)); } - // Persist state mutations back onto config.settings. + // Persist state mutations back onto config.settings. A silent rule whose + // settings resolve nothing (or an identity that is already current) sends + // nothing: a declared skip, not a failure. settings._state = state; - - // A silent rule whose own settings resolved nothing sent nothing. - const ruleSettings = - mappingSettings.reset !== undefined || - mappingSettings.identify !== undefined || - mappingSettings.group !== undefined || - mappingSettings.groupIdentify !== undefined || - mappingSettings.revenue !== undefined; - if (!calls && ruleSettings) - throw new StepError( - `Amplitude: the silent rule of "${event.name}" resolved no identity, group or revenue`, - { code: 'missing-input' }, - ); } function isStringArray(value: unknown): value is string[] { @@ -468,12 +475,21 @@ export const destinationAmplitude: Destination = { return { ...config, settings: { ...settings, _state } }; }, - async push(event, { config, rule, env, data, collector }) { + async push(event, { config, rule, env, data, collector, logger }) { const settings: Settings = { apiKey: '', ...config.settings }; const mappingSettings: Mapping = rule?.settings || {}; const mock = mockOf(env); await (mock - ? pushTo(mock, event, settings, mappingSettings, rule, data, collector) + ? pushTo( + mock, + event, + settings, + mappingSettings, + rule, + data, + collector, + logger, + ) : pushTo( realAmplitude(), event, @@ -482,6 +498,7 @@ export const destinationAmplitude: Destination = { rule, data, collector, + logger, )); }, @@ -494,7 +511,7 @@ export const destinationAmplitude: Destination = { const granted = getGrantedConsent(required, collector.consent) !== false; const amp = mockOf(env) ?? realAmplitude(); - await call( + await stepCall( () => amp.setOptOut(!granted), 'consent-not-applied', 'Amplitude did not take the consent update', diff --git a/packages/web/destinations/amplitude/src/schemas/settings.ts b/packages/web/destinations/amplitude/src/schemas/settings.ts index ffcc83230..2be299815 100644 --- a/packages/web/destinations/amplitude/src/schemas/settings.ts +++ b/packages/web/destinations/amplitude/src/schemas/settings.ts @@ -1,72 +1,76 @@ import { z } from '@walkeros/core/dev'; -export const SettingsSchema = z.object({ - apiKey: z - .string() - .min(1) - .describe( - 'Your Amplitude project API key. Find it in your Amplitude project settings under "General" → "API Keys" (like a1b2c3d4e5f6789012345678abcdef12).', - ), - serverZone: z - .enum(['US', 'EU']) - .describe('Amplitude server zone. Default: US.') - .optional(), - flushIntervalMillis: z - .number() - .int() - .positive() - .describe('How often (in ms) to flush the event queue. Default: 1000.') - .optional(), - flushQueueSize: z - .number() - .int() - .positive() - .describe('Max queued events before a flush. Default: 30.') - .optional(), - flushMaxRetries: z - .number() - .int() - .nonnegative() - .describe('Max retries on failed flush. Default: 5.') - .optional(), - transport: z - .enum(['fetch', 'xhr', 'beacon']) - .describe('HTTP transport. Default: fetch.') - .optional(), - useBatch: z - .boolean() - .describe( - 'Use the Amplitude batch endpoint instead of the standard endpoint. Default: false.', - ) - .optional(), - appVersion: z - .string() - .describe('Application version; tagged onto every event.') - .optional(), - identify: z - .unknown() - .describe( - 'walkerOS mapping value resolving to an identity object. Keys: user, device, session, set, setOnce, add, append, prepend, preInsert, postInsert, remove, unset, clearAll.', - ) - .optional(), - sessionReplay: z - .unknown() - .describe( - 'Session Replay options, passed to the Session Replay plugin. Without this, experiment.deploymentKey or engagement only analytics starts. With any of the three set, @amplitude/unified sets up all three plugins together. It records a sampled share of sessions, set by the project sampling settings in Amplitude when present, else by sampleRate (default 0).', - ) - .optional(), - experiment: z - .unknown() - .describe( - 'Feature Experiment options, passed to the Experiment plugin only when they include `deploymentKey`. Without it, sessionReplay or engagement only analytics starts. With any of the three set, @amplitude/unified sets up all three plugins together.', - ) - .optional(), - engagement: z - .unknown() - .describe( - 'Guides & Surveys options object; `true` passes no options. Without it, sessionReplay or experiment.deploymentKey only analytics starts. With any of the three set, @amplitude/unified sets up all three plugins together, and Guides & Surveys loads its SDK from cdn.amplitude.com.', - ) - .optional(), -}); +export const SettingsSchema = z + .object({ + apiKey: z + .string() + .min(1) + .describe( + 'Your Amplitude project API key. Find it in your Amplitude project settings under "General" → "API Keys" (like a1b2c3d4e5f6789012345678abcdef12).', + ), + serverZone: z + .enum(['US', 'EU']) + .describe('Amplitude server zone. Default: US.') + .optional(), + flushIntervalMillis: z + .number() + .int() + .positive() + .describe('How often (in ms) to flush the event queue. Default: 1000.') + .optional(), + flushQueueSize: z + .number() + .int() + .positive() + .describe('Max queued events before a flush. Default: 30.') + .optional(), + flushMaxRetries: z + .number() + .int() + .nonnegative() + .describe('Max retries on failed flush. Default: 5.') + .optional(), + transport: z + .enum(['fetch', 'xhr', 'beacon']) + .describe('HTTP transport. Default: fetch.') + .optional(), + useBatch: z + .boolean() + .describe( + 'Use the Amplitude batch endpoint instead of the standard endpoint. Default: false.', + ) + .optional(), + appVersion: z + .string() + .describe('Application version; tagged onto every event.') + .optional(), + identify: z + .unknown() + .describe( + 'walkerOS mapping value resolving to an identity object. Keys: user, device, session, set, setOnce, add, append, prepend, preInsert, postInsert, remove, unset, clearAll.', + ) + .optional(), + sessionReplay: z + .unknown() + .describe( + 'Session Replay options. When set, the Session Replay plugin starts; the other plugins start only from their own settings. It records a sampled share of sessions, set by the project sampling settings in Amplitude when present, else by sampleRate (default 0).', + ) + .optional(), + experiment: z + .unknown() + .describe( + 'Feature Experiment options. The Experiment plugin starts only when they include `deploymentKey`; the other plugins start only from their own settings.', + ) + .optional(), + engagement: z + .unknown() + .describe( + 'Guides & Surveys options object; `true` passes no options. When set, the Guides & Surveys plugin starts and loads its SDK from cdn.amplitude.com; the other plugins start only from their own settings.', + ) + .optional(), + }) + // Pass-through settings are loose: every other Amplitude BrowserOptions + // key (serverUrl, defaultTracking, ...) is forwarded to the SDK as given. + .loose(); export type Settings = z.infer; diff --git a/packages/web/destinations/amplitude/src/types/index.ts b/packages/web/destinations/amplitude/src/types/index.ts index 2784896c5..34f5d05e4 100644 --- a/packages/web/destinations/amplitude/src/types/index.ts +++ b/packages/web/destinations/amplitude/src/types/index.ts @@ -3,10 +3,7 @@ import type { Destination as CoreDestination, } from '@walkeros/core'; import type { DestinationWeb } from '@walkeros/web-core'; -import type { - Types as AmplitudeTypes, - UnifiedOptions, -} from '@amplitude/unified'; +import type { Types as AmplitudeTypes } from '@amplitude/unified'; import type { SessionReplayOptions } from '@amplitude/plugin-session-replay-browser'; import type { ExperimentPluginConfig } from '@amplitude/plugin-experiment-browser'; import type { InitOptions as EngagementInitOptions } from '@amplitude/engagement-browser'; @@ -15,6 +12,7 @@ import type { InitOptions as EngagementInitOptions } from '@amplitude/engagement // walkerOS mapping-based `identify`. Omit it so Settings can re-declare it. type BrowserOptions = Omit; export type AmplitudeBrowserOptions = AmplitudeTypes.BrowserOptions; +export type AmplitudePlugin = AmplitudeTypes.Plugin; /** * Destination-level settings. @@ -71,10 +69,10 @@ export interface AmplitudeSDK< I extends IdentifyInstance = IdentifyInstance, R extends RevenueInstance = RevenueInstance, > { - /** Analytics only, the plugins none. */ + /** Analytics. */ init: (apiKey: string, options?: AmplitudeBrowserOptions) => unknown; - /** Analytics plus Session Replay, Experiment and Guides & Surveys. */ - initAll: (apiKey: string, options?: UnifiedOptions) => unknown; + /** One plugin: Session Replay, Experiment or Guides & Surveys. */ + add: (plugin: AmplitudePlugin) => unknown; track: ( eventType: string, eventProperties?: Record, diff --git a/packages/web/destinations/clarity/src/__tests__/index.test.ts b/packages/web/destinations/clarity/src/__tests__/index.test.ts index ec795cf39..fcff73ee2 100644 --- a/packages/web/destinations/clarity/src/__tests__/index.test.ts +++ b/packages/web/destinations/clarity/src/__tests__/index.test.ts @@ -1,5 +1,6 @@ import { startFlow } from '@walkeros/collector'; -import { getEvent } from '@walkeros/core'; +import type { Logger } from '@walkeros/core'; +import { createEvent, getEvent, Level } from '@walkeros/core'; import { destinationClarity } from '..'; import type { ClaritySDK } from '../types'; @@ -99,4 +100,43 @@ describe('clarity destination', () => { code: 'vendor-not-loaded', }); }); + + it('writes one DEBUG line naming the event and the mapping key for a declared skip', async () => { + const lines: Array<[Level, string, unknown]> = []; + const handler: Logger.Handler = (level, message, context) => { + lines.push([level, message, context]); + }; + const calls: unknown[][] = []; + const { elb } = await startFlow({ + logger: { level: 'DEBUG', handler }, + destinations: { + clarity: { + code: { + ...destinationClarity, + env: { clarity: recordingClarity(calls) }, + }, + config: { + settings: { apiKey: 'key' }, + mapping: { + company: { + update: { + silent: true, + settings: { set: { map: { page_id: 'data.none' } } }, + }, + }, + }, + }, + }, + }, + }); + + expect((await elb(createEvent({ name: 'company update' }))).ok).toBe(true); + expect(lines.filter(([, message]) => message.includes('skipped'))).toEqual([ + [ + Level.DEBUG, + 'Clarity skipped setTag: set resolved to nothing', + expect.objectContaining({ event: 'company update' }), + ], + ]); + }); }); diff --git a/packages/web/destinations/clarity/src/index.ts b/packages/web/destinations/clarity/src/index.ts index 429701d46..7bd3d78c4 100644 --- a/packages/web/destinations/clarity/src/index.ts +++ b/packages/web/destinations/clarity/src/index.ts @@ -1,4 +1,4 @@ -import type { WalkerOS } from '@walkeros/core'; +import type { Logger, WalkerOS } from '@walkeros/core'; import { getGrantedConsent, getMappingValue, @@ -194,6 +194,22 @@ function consentOf( : { analytics_Storage: 'denied', ad_Storage: 'denied' }; } +/** + * A configured mapping that resolved to nothing skips its call: a declared + * skip, said once at DEBUG with the event and the mapping key, never the + * value, so a wrong mapping path does not look like an event without data. + */ +function skipped( + logger: Logger.Instance, + event: WalkerOS.Event, + call: string, + key: string, +): void { + logger.debug(`Clarity skipped ${call}: ${key} resolved to nothing`, { + event: event.name, + }); +} + export const destinationClarity: Destination = { type: 'clarity', @@ -211,7 +227,7 @@ export const destinationClarity: Destination = { ); }, - async push(event, { config, rule, env, data, collector }) { + async push(event, { config, rule, env, data, collector, logger }) { const settings: Partial = config.settings || {}; const clarity = getClarity(env, settings.apiKey ?? '', config.timeout); const mappingSettings: Mapping = rule?.settings || {}; @@ -224,21 +240,19 @@ export const destinationClarity: Destination = { const resolved = await getMappingValue(event, identifyMapping, { collector, }); - if (isObject(resolved)) { - const { customId, customSessionId, customPageId, friendlyName } = - resolved; - if (isString(customId) && customId) { - await send(() => - callIdentify( - clarity, - customId, - isString(customSessionId) ? customSessionId : undefined, - isString(customPageId) ? customPageId : undefined, - isString(friendlyName) ? friendlyName : undefined, - ), - ); - } - } + const { customId, customSessionId, customPageId, friendlyName } = + isObject(resolved) ? resolved : {}; + if (isString(customId) && customId) { + await send(() => + callIdentify( + clarity, + customId, + isString(customSessionId) ? customSessionId : undefined, + isString(customPageId) ? customPageId : undefined, + isString(friendlyName) ? friendlyName : undefined, + ), + ); + } else skipped(logger, event, 'identify', 'identify'); } // 2. Include tags - pre-flattened by core into context.data @@ -250,8 +264,10 @@ export const destinationClarity: Destination = { const resolved = await getMappingValue(event, mappingSettings.set, { collector, }); - for (const [key, value] of tagsOf(resolved)) + const tags = tagsOf(resolved); + for (const [key, value] of tags) await send(() => clarity.setTag(key, value)); + if (!tags.length) skipped(logger, event, 'setTag', 'set'); } // 4. Session priority upgrade @@ -260,6 +276,7 @@ export const destinationClarity: Destination = { collector, }); if (isString(reason) && reason) await send(() => clarity.upgrade(reason)); + else skipped(logger, event, 'upgrade', 'upgrade'); } // 5. Default event forwarding - unless rule.silent is set diff --git a/packages/web/destinations/fullstory/src/__tests__/index.test.ts b/packages/web/destinations/fullstory/src/__tests__/index.test.ts index 200319549..caf17bb7d 100644 --- a/packages/web/destinations/fullstory/src/__tests__/index.test.ts +++ b/packages/web/destinations/fullstory/src/__tests__/index.test.ts @@ -1,5 +1,6 @@ import { startFlow } from '@walkeros/collector'; -import { getEvent } from '@walkeros/core'; +import type { Logger } from '@walkeros/core'; +import { createEvent, getEvent, Level } from '@walkeros/core'; import { destinationFullStory } from '..'; import type { FullStorySDK } from '../types'; @@ -45,4 +46,43 @@ describe('fullstory destination', () => { ['shutdown'], ]); }); + + it('writes one DEBUG line naming the event and the mapping key for a declared skip', async () => { + const lines: Array<[Level, string, unknown]> = []; + const handler: Logger.Handler = (level, message, context) => { + lines.push([level, message, context]); + }; + const calls: unknown[][] = []; + const { elb } = await startFlow({ + logger: { level: 'DEBUG', handler }, + destinations: { + fullstory: { + code: { + ...destinationFullStory, + env: { fullstory: recordingFS(calls) }, + }, + config: { + settings: { orgId: 'o-1' }, + mapping: { + company: { + update: { + silent: true, + settings: { identify: { map: { uid: 'data.none' } } }, + }, + }, + }, + }, + }, + }, + }); + + expect((await elb(createEvent({ name: 'company update' }))).ok).toBe(true); + expect(lines.filter(([, message]) => message.includes('skipped'))).toEqual([ + [ + Level.DEBUG, + 'FullStory skipped setIdentity: identify resolved to nothing', + expect.objectContaining({ event: 'company update' }), + ], + ]); + }); }); diff --git a/packages/web/destinations/fullstory/src/index.ts b/packages/web/destinations/fullstory/src/index.ts index 8b8779ac9..ea45ffd33 100644 --- a/packages/web/destinations/fullstory/src/index.ts +++ b/packages/web/destinations/fullstory/src/index.ts @@ -1,4 +1,4 @@ -import type { WalkerOS } from '@walkeros/core'; +import type { Logger, WalkerOS } from '@walkeros/core'; import { getGrantedConsent, getMappingValue, @@ -108,6 +108,22 @@ function consentCalls( return [() => (granted ? fs.start() : fs.shutdown())]; } +/** + * A configured mapping that resolved to nothing skips its call: a declared + * skip, said once at DEBUG with the event and the mapping key, never the + * value, so a wrong mapping path does not look like an event without data. + */ +function skipped( + logger: Logger.Instance, + event: WalkerOS.Event, + call: string, + key: string, +): void { + logger.debug(`FullStory skipped ${call}: ${key} resolved to nothing`, { + event: event.name, + }); +} + export const destinationFullStory: Destination = { type: 'fullstory', @@ -127,7 +143,7 @@ export const destinationFullStory: Destination = { ); }, - async push(event, { config, rule, env, data, collector }) { + async push(event, { config, rule, env, data, collector, logger }) { const fs = getFS(env); const settings: Partial = config.settings || {}; const mappingSettings: Mapping = rule?.settings || {}; @@ -138,17 +154,15 @@ export const destinationFullStory: Destination = { const resolved = await getMappingValue(event, identifyMapping, { collector, }); - if (isObject(resolved)) { - const { uid, properties } = resolved; - if (isString(uid) && uid) { - const identityOpts: { - uid: string; - properties?: Record; - } = { uid }; - if (isObject(properties)) identityOpts.properties = properties; - await send(() => fs.setIdentity(identityOpts)); - } - } + const { uid, properties } = isObject(resolved) ? resolved : {}; + if (isString(uid) && uid) { + const identityOpts: { + uid: string; + properties?: Record; + } = { uid }; + if (isObject(properties)) identityOpts.properties = properties; + await send(() => fs.setIdentity(identityOpts)); + } else skipped(logger, event, 'setIdentity', 'identify'); } // 2. setProperties -- user or page scope @@ -161,7 +175,7 @@ export const destinationFullStory: Destination = { await send(() => fs.setProperties({ type: propertyType, properties: resolved }), ); - } + } else skipped(logger, event, 'setProperties', 'set'); } // 3. Default trackEvent -- unless rule.silent is set diff --git a/packages/web/destinations/hotjar/src/__tests__/stepExamples.test.ts b/packages/web/destinations/hotjar/src/__tests__/stepExamples.test.ts index ae85af6b0..b8841572b 100644 --- a/packages/web/destinations/hotjar/src/__tests__/stepExamples.test.ts +++ b/packages/web/destinations/hotjar/src/__tests__/stepExamples.test.ts @@ -6,7 +6,9 @@ jest.mock('@hotjar/browser', () => ({ import type { WalkerOS, Mapping as WalkerOSMapping } from '@walkeros/core'; import { startFlow } from '@walkeros/collector'; -import { clone } from '@walkeros/core'; +import type { Logger } from '@walkeros/core'; +import { clone, createEvent, Level } from '@walkeros/core'; +import { destinationHotjar } from '..'; import { examples } from '../dev'; import type { Env, Settings } from '../types'; import { expectSimulationResolves } from '@walkeros/core/dev'; @@ -95,6 +97,44 @@ describe('hotjar destination -- step examples', () => { expect(actual).toEqual(expected); }); + + it('writes one DEBUG line naming the event and the mapping key for a declared skip', async () => { + const lines: Array<[Level, string, unknown]> = []; + const handler: Logger.Handler = (level, message, context) => { + lines.push([level, message, context]); + }; + const { elb } = await startFlow({ + logger: { level: 'DEBUG', handler }, + destinations: { + hotjar: { + code: { + ...destinationHotjar, + env: spyEnv(clone(examples.env.push)).env, + }, + config: { + settings: { siteId: 1234567 }, + mapping: { + company: { + update: { + silent: true, + settings: { stateChange: 'data.none' }, + }, + }, + }, + }, + }, + }, + }); + + expect((await elb(createEvent({ name: 'company update' }))).ok).toBe(true); + expect(lines.filter(([, message]) => message.includes('skipped'))).toEqual([ + [ + Level.DEBUG, + 'Hotjar skipped stateChange: stateChange resolved to nothing', + expect.objectContaining({ event: 'company update' }), + ], + ]); + }); }); it('declares simulation paths that resolve', () => diff --git a/packages/web/destinations/hotjar/src/index.ts b/packages/web/destinations/hotjar/src/index.ts index 62c9bab88..26f1f57d7 100644 --- a/packages/web/destinations/hotjar/src/index.ts +++ b/packages/web/destinations/hotjar/src/index.ts @@ -1,3 +1,4 @@ +import type { Logger, WalkerOS } from '@walkeros/core'; import { getMappingValue, isObject, @@ -32,6 +33,22 @@ const call = (fn: () => unknown, code: string, message: string) => message, ); +/** + * A configured mapping that resolved to nothing skips its call: a declared + * skip, said once at DEBUG with the event and the mapping key, never the + * value, so a wrong mapping path does not look like an event without data. + */ +function skipped( + logger: Logger.Instance, + event: WalkerOS.Event, + call: string, + key: string, +): void { + logger.debug(`Hotjar skipped ${call}: ${key} resolved to nothing`, { + event: event.name, + }); +} + export const destinationHotjar: Destination = { type: 'hotjar', @@ -62,7 +79,7 @@ export const destinationHotjar: Destination = { return config; }, - async push(event, { config, rule, env, collector }) { + async push(event, { config, rule, env, collector, logger }) { const hotjar = getHotjar(env as Env | undefined); const settings = (config.settings || {}) as Partial; const mappingSettings = (rule?.settings || {}) as Mapping; @@ -75,27 +92,25 @@ export const destinationHotjar: Destination = { const resolved = await getMappingValue(event, identifyMapping, { collector, }); - if (isObject(resolved)) { - const { userId, ...attributes } = resolved as Record; - if (isString(userId) && userId) { - // Build typed user attributes, filtering out non-primitive values - const userInfo: Record = {}; - for (const [key, val] of Object.entries(attributes)) { - if ( - isString(val) || - typeof val === 'number' || - typeof val === 'boolean' - ) { - userInfo[key] = val; - } + const { userId, ...attributes } = isObject(resolved) ? resolved : {}; + if (isString(userId) && userId) { + // Build typed user attributes, filtering out non-primitive values + const userInfo: Record = {}; + for (const [key, val] of Object.entries(attributes)) { + if ( + isString(val) || + typeof val === 'number' || + typeof val === 'boolean' + ) { + userInfo[key] = val; } - await call( - () => hotjar.identify(userId, userInfo), - 'transport', - 'Hotjar identify failed', - ); } - } + await call( + () => hotjar.identify(userId, userInfo), + 'transport', + 'Hotjar identify failed', + ); + } else skipped(logger, event, 'identify', 'identify'); } // 2. SPA state change -- stateChange mapping resolves to path string. @@ -111,7 +126,7 @@ export const destinationHotjar: Destination = { 'transport', 'Hotjar stateChange failed', ); - } + } else skipped(logger, event, 'stateChange', 'stateChange'); } // 3. Default event forwarding -- unless rule.silent is set diff --git a/packages/web/destinations/mixpanel/src/__tests__/stepExamples.test.ts b/packages/web/destinations/mixpanel/src/__tests__/stepExamples.test.ts index f46736b4c..fdcfb7e63 100644 --- a/packages/web/destinations/mixpanel/src/__tests__/stepExamples.test.ts +++ b/packages/web/destinations/mixpanel/src/__tests__/stepExamples.test.ts @@ -9,7 +9,9 @@ jest.mock('mixpanel-browser', () => ({ import type { WalkerOS, Mapping as WalkerOSMapping } from '@walkeros/core'; import { startFlow } from '@walkeros/collector'; -import { clone } from '@walkeros/core'; +import type { Logger } from '@walkeros/core'; +import { clone, createEvent, Level } from '@walkeros/core'; +import { destinationMixpanel } from '..'; import { examples } from '../dev'; import type { MixpanelStepExample } from '../examples/step'; import type { Env, MixpanelGroup, Settings } from '../types'; @@ -179,6 +181,45 @@ describe('mixpanel destination — step examples', () => { expect(actual).toEqual(expected); }, ); + + it('writes one DEBUG line naming the event and the mapping key for a declared skip', async () => { + const lines: Array<[Level, string, unknown]> = []; + const handler: Logger.Handler = (level, message, context) => { + lines.push([level, message, context]); + }; + const { elb } = await startFlow({ + logger: { level: 'DEBUG', handler }, + destinations: { + mixpanel: { + code: { ...destinationMixpanel, env: clone(examples.env.push) }, + config: { + settings: { apiKey: 'k' }, + mapping: { + company: { + update: { + silent: true, + settings: { + groupProfile: { + map: { key: { value: 'company' }, id: 'data.none' }, + }, + }, + }, + }, + }, + }, + }, + }, + }); + + expect((await elb(createEvent({ name: 'company update' }))).ok).toBe(true); + expect(lines.filter(([, message]) => message.includes('skipped'))).toEqual([ + [ + Level.DEBUG, + 'Mixpanel skipped groupProfile: groupProfile resolved to nothing', + expect.objectContaining({ event: 'company update' }), + ], + ]); + }); }); it('declares simulation paths that resolve', () => diff --git a/packages/web/destinations/mixpanel/src/index.ts b/packages/web/destinations/mixpanel/src/index.ts index 4358a38c3..28665bcd7 100644 --- a/packages/web/destinations/mixpanel/src/index.ts +++ b/packages/web/destinations/mixpanel/src/index.ts @@ -1,3 +1,4 @@ +import type { Logger, WalkerOS } from '@walkeros/core'; import { getGrantedConsent, getMappingValue, @@ -179,6 +180,26 @@ async function applyGroupProfile( return applied; } +function isStringArray(value: unknown): value is string[] { + return isArray(value) && value.every(isString); +} + +/** + * A configured mapping that resolved to nothing skips its call: a declared + * skip, said once at DEBUG with the event and the mapping key, never the + * value, so a wrong mapping path does not look like an event without data. + */ +function skipped( + logger: Logger.Instance, + event: WalkerOS.Event, + call: string, + key: string, +): void { + logger.debug(`Mixpanel skipped ${call}: ${key} resolved to nothing`, { + event: event.name, + }); +} + export const destinationMixpanel: Destination = { type: 'mixpanel', @@ -223,7 +244,7 @@ export const destinationMixpanel: Destination = { return { ...config, settings: { ...settings, _state } }; }, - async push(event, { config, rule, env, data, collector }) { + async push(event, { config, rule, env, data, collector, logger }) { const mp = getMixpanel(env as Env | undefined); const settings = (config.settings || {}) as Settings; const mappingSettings = rule?.settings || {}; @@ -246,13 +267,17 @@ export const destinationMixpanel: Destination = { const resolved = await getMappingValue(event, identifyMapping, { collector, }); - if (isObject(resolved)) { + if ( + isObject(resolved) && + isString(resolved.distinctId) && + resolved.distinctId !== '' + ) state.lastIdentity = await applyIdentify( mp, - resolved as Record, + resolved, state.lastIdentity, ); - } + else skipped(logger, event, 'identify', 'identify'); } // 3. People operations - fires zero or more mp.people.* calls. @@ -260,8 +285,8 @@ export const destinationMixpanel: Destination = { const resolved = await getMappingValue(event, mappingSettings.people, { collector, }); - if (isObject(resolved)) - await applyPeople(mp, resolved as Record); + if (!(isObject(resolved) && (await applyPeople(mp, resolved)))) + skipped(logger, event, 'people', 'people'); } // 4. Group assignment - resolves to { key, id } and calls set_group. @@ -270,21 +295,19 @@ export const destinationMixpanel: Destination = { const resolved = await getMappingValue(event, groupMapping, { collector, }); - if (isObject(resolved)) { - const { key, id } = resolved as { key?: unknown; id?: unknown }; - if ( - isString(key) && - key !== '' && - (isString(id) || isArray(id)) && - id !== '' - ) { - await stepCall( - () => mp.set_group(key, id as string | string[]), - 'transport', - 'Mixpanel set_group failed', - ); - } - } + const { key, id } = isObject(resolved) ? resolved : {}; + if ( + isString(key) && + key !== '' && + (isString(id) || isStringArray(id)) && + id !== '' + ) + await stepCall( + () => mp.set_group(key, id), + 'transport', + 'Mixpanel set_group failed', + ); + else skipped(logger, event, 'set_group', 'group'); } // 5. Group profile - resolves to { key, id, set?, set_once?, ... } @@ -294,8 +317,8 @@ export const destinationMixpanel: Destination = { mappingSettings.groupProfile, { collector }, ); - if (isObject(resolved)) - await applyGroupProfile(mp, resolved as Record); + if (!(isObject(resolved) && (await applyGroupProfile(mp, resolved)))) + skipped(logger, event, 'groupProfile', 'groupProfile'); } // 6. Default track - unless the rule opts out via silent. diff --git a/packages/web/destinations/pinterest/src/__tests__/stepExamples.test.ts b/packages/web/destinations/pinterest/src/__tests__/stepExamples.test.ts index 972fc6184..832735aca 100644 --- a/packages/web/destinations/pinterest/src/__tests__/stepExamples.test.ts +++ b/packages/web/destinations/pinterest/src/__tests__/stepExamples.test.ts @@ -9,7 +9,9 @@ import type { WalkerOS, Mapping as WalkerOSMapping } from '@walkeros/core'; import { startFlow } from '@walkeros/collector'; -import { clone, isObject, isString } from '@walkeros/core'; +import type { Logger } from '@walkeros/core'; +import { clone, createEvent, isObject, isString, Level } from '@walkeros/core'; +import { destinationPinterest } from '..'; import { examples } from '../dev'; import type { Env, Pintrk } from '../types'; @@ -97,4 +99,42 @@ describe('pinterest destination — step examples', () => { expect(actual).toEqual([...(example.out ?? [])]); }); + + it('writes one DEBUG line naming the event and the mapping key for a declared skip', async () => { + const lines: Array<[Level, string, unknown]> = []; + const handler: Logger.Handler = (level, message, context) => { + lines.push([level, message, context]); + }; + const { elb } = await startFlow({ + logger: { level: 'DEBUG', handler }, + destinations: { + pinterest: { + code: { + ...destinationPinterest, + env: spyEnv(clone(examples.env.push)).env, + }, + config: { + settings: { apiKey: '2612345678901' }, + mapping: { + company: { + update: { + silent: true, + settings: { identify: { map: { em: 'data.none' } } }, + }, + }, + }, + }, + }, + }, + }); + + expect((await elb(createEvent({ name: 'company update' }))).ok).toBe(true); + expect(lines.filter(([, message]) => message.includes('skipped'))).toEqual([ + [ + Level.DEBUG, + 'Pinterest skipped set: identify resolved to nothing', + expect.objectContaining({ event: 'company update' }), + ], + ]); + }); }); diff --git a/packages/web/destinations/pinterest/src/index.ts b/packages/web/destinations/pinterest/src/index.ts index c2c8b9b58..7dc512064 100644 --- a/packages/web/destinations/pinterest/src/index.ts +++ b/packages/web/destinations/pinterest/src/index.ts @@ -8,7 +8,7 @@ import { StepError, stepCall, } from '@walkeros/core'; -import type { Collector, Mapping, WalkerOS } from '@walkeros/core'; +import type { Collector, Logger, Mapping, WalkerOS } from '@walkeros/core'; import type { Destination, EventData, @@ -94,6 +94,22 @@ function identifyEqual(a?: IdentifyFields, b?: IdentifyFields): boolean { return a.em === b.em && a.external_id === b.external_id; } +/** + * A configured mapping that resolved to nothing skips its call: a declared + * skip, said once at DEBUG with the event and the mapping key, never the + * value, so a wrong mapping path does not look like an event without data. + */ +function skipped( + logger: Logger.Instance, + event: WalkerOS.Event, + call: string, + key: string, +): void { + logger.debug(`Pinterest skipped ${call}: ${key} resolved to nothing`, { + event: event.name, + }); +} + export const destinationPinterest: Destination = { type: 'pinterest', @@ -137,7 +153,7 @@ export const destinationPinterest: Destination = { }; }, - async push(event, { config, rule, data, env, collector }) { + async push(event, { config, rule, data, env, collector, logger }) { vendor(config, env).check(); const settings: Partial = config.settings || {}; @@ -152,7 +168,10 @@ export const destinationPinterest: Destination = { // is already current) sends nothing: a declared skip, not a failure. const identifyMapping = rule?.settings?.identify ?? settings.identify; const identify = await resolveIdentify(event, identifyMapping, collector); - if (identify && !identifyEqual(identify, state.lastIdentify)) { + if (!identify) { + if (identifyMapping !== undefined) + skipped(logger, event, 'set', 'identify'); + } else if (!identifyEqual(identify, state.lastIdentify)) { await stepCall( () => pintrk('set', identify), 'transport', diff --git a/packages/web/destinations/posthog/src/__tests__/stepExamples.test.ts b/packages/web/destinations/posthog/src/__tests__/stepExamples.test.ts index 04fe420ff..1a859118e 100644 --- a/packages/web/destinations/posthog/src/__tests__/stepExamples.test.ts +++ b/packages/web/destinations/posthog/src/__tests__/stepExamples.test.ts @@ -9,7 +9,9 @@ jest.mock('posthog-js', () => ({ import type { WalkerOS, Mapping as WalkerOSMapping } from '@walkeros/core'; import { startFlow } from '@walkeros/collector'; -import { clone } from '@walkeros/core'; +import type { Logger } from '@walkeros/core'; +import { clone, createEvent, Level } from '@walkeros/core'; +import { destinationPostHog } from '..'; import { examples } from '../dev'; import type { PostHogStepExample } from '../examples/step'; import type { Env, PostHogSDK, Settings } from '../types'; @@ -127,6 +129,45 @@ describe('posthog destination — step examples', () => { expect(actual).toEqual(expected); }, ); + + it('writes one DEBUG line naming the event and the mapping key for a declared skip', async () => { + const lines: Array<[Level, string, unknown]> = []; + const handler: Logger.Handler = (level, message, context) => { + lines.push([level, message, context]); + }; + const { elb } = await startFlow({ + logger: { level: 'DEBUG', handler }, + destinations: { + posthog: { + code: { ...destinationPostHog, env: clone(examples.env.push) }, + config: { + settings: { apiKey: 'k' }, + mapping: { + company: { + update: { + silent: true, + settings: { + group: { + map: { type: { value: 'company' }, key: 'data.none' }, + }, + }, + }, + }, + }, + }, + }, + }, + }); + + expect((await elb(createEvent({ name: 'company update' }))).ok).toBe(true); + expect(lines.filter(([, message]) => message.includes('skipped'))).toEqual([ + [ + Level.DEBUG, + 'PostHog skipped group: group resolved to nothing', + expect.objectContaining({ event: 'company update' }), + ], + ]); + }); }); it('declares simulation paths that resolve', () => diff --git a/packages/web/destinations/posthog/src/index.ts b/packages/web/destinations/posthog/src/index.ts index dee3e387c..7d59583be 100644 --- a/packages/web/destinations/posthog/src/index.ts +++ b/packages/web/destinations/posthog/src/index.ts @@ -1,3 +1,4 @@ +import type { Logger, WalkerOS } from '@walkeros/core'; import { getGrantedConsent, getMappingValue, @@ -117,6 +118,22 @@ async function applyGroup( return { lastGroup: updated, applied }; } +/** + * A configured mapping that resolved to nothing skips its call: a declared + * skip, said once at DEBUG with the event and the mapping key, never the + * value, so a wrong mapping path does not look like an event without data. + */ +function skipped( + logger: Logger.Instance, + event: WalkerOS.Event, + call: string, + key: string, +): void { + logger.debug(`PostHog skipped ${call}: ${key} resolved to nothing`, { + event: event.name, + }); +} + export const destinationPostHog: Destination = { type: 'posthog', @@ -172,7 +189,7 @@ export const destinationPostHog: Destination = { return { ...config, settings: { ...settings, _state } }; }, - async push(event, { config, rule, env, data, collector }) { + async push(event, { config, rule, env, data, collector, logger }) { const posthog = getPostHog(env as Env | undefined); const settings = (config.settings || {}) as Settings; const mappingSettings = rule?.settings || {}; @@ -203,14 +220,11 @@ export const destinationPostHog: Destination = { const resolved = await getMappingValue(event, identifyMapping, { collector, }); - if (isObject(resolved)) { - const identity = await applyIdentity( - posthog, - resolved as Record, - state.lastIdentity, - ); - state.lastIdentity = identity.lastIdentity; - } + const identity = isObject(resolved) + ? await applyIdentity(posthog, resolved, state.lastIdentity) + : undefined; + if (identity?.applied) state.lastIdentity = identity.lastIdentity; + else skipped(logger, event, 'identify', 'identify'); } // 3. Group - rule-level override wins over destination-level. @@ -219,14 +233,11 @@ export const destinationPostHog: Destination = { const resolved = await getMappingValue(event, groupMapping, { collector, }); - if (isObject(resolved)) { - const group = await applyGroup( - posthog, - resolved as Record, - state.lastGroup, - ); - state.lastGroup = group.lastGroup; - } + const group = isObject(resolved) + ? await applyGroup(posthog, resolved, state.lastGroup) + : undefined; + if (group?.applied) state.lastGroup = group.lastGroup; + else skipped(logger, event, 'group', 'group'); } // 4. Default capture - unless rule opts out via silent. diff --git a/packages/web/destinations/segment/src/__tests__/index.test.ts b/packages/web/destinations/segment/src/__tests__/index.test.ts index 261b3e491..70a9c8cbf 100644 --- a/packages/web/destinations/segment/src/__tests__/index.test.ts +++ b/packages/web/destinations/segment/src/__tests__/index.test.ts @@ -1,5 +1,6 @@ import { startFlow } from '@walkeros/collector'; -import { getEvent } from '@walkeros/core'; +import type { Logger } from '@walkeros/core'; +import { createEvent, getEvent, Level } from '@walkeros/core'; import { destinationSegment } from '..'; import type { SegmentAnalytics, SegmentSDK } from '../types'; @@ -42,4 +43,50 @@ describe('segment destination', () => { ], ]); }); + + it('writes one DEBUG line naming the event and the mapping key for a declared skip', async () => { + const lines: Array<[Level, string, unknown]> = []; + const handler: Logger.Handler = (level, message, context) => { + lines.push([level, message, context]); + }; + const noop = () => undefined; + const instance: SegmentAnalytics = { + track: noop, + identify: noop, + group: noop, + page: noop, + alias: noop, + reset: noop, + setAnonymousId: noop, + }; + const analytics: SegmentSDK = { load: () => instance }; + const { elb } = await startFlow({ + logger: { level: 'DEBUG', handler }, + destinations: { + segment: { + code: { ...destinationSegment, env: { analytics } }, + config: { + settings: { apiKey: 'key' }, + mapping: { + company: { + update: { + silent: true, + settings: { group: { map: { groupId: 'data.none' } } }, + }, + }, + }, + }, + }, + }, + }); + + expect((await elb(createEvent({ name: 'company update' }))).ok).toBe(true); + expect(lines.filter(([, message]) => message.includes('skipped'))).toEqual([ + [ + Level.DEBUG, + 'Segment skipped group: group resolved to nothing', + expect.objectContaining({ event: 'company update' }), + ], + ]); + }); }); diff --git a/packages/web/destinations/segment/src/index.ts b/packages/web/destinations/segment/src/index.ts index 85c527010..bd3ff9f8e 100644 --- a/packages/web/destinations/segment/src/index.ts +++ b/packages/web/destinations/segment/src/index.ts @@ -1,4 +1,4 @@ -import type { WalkerOS } from '@walkeros/core'; +import type { Logger, WalkerOS } from '@walkeros/core'; import { getMappingValue, isArray, @@ -131,16 +131,33 @@ function buildEventOptions( }; } +/** + * A configured mapping that resolved to nothing skips its call: a declared + * skip, said once at DEBUG with the event and the mapping key, never the + * value, so a wrong mapping path does not look like an event without data. + */ +function skipped( + logger: Logger.Instance, + event: WalkerOS.Event, + call: string, + key: string, +): void { + logger.debug(`Segment skipped ${call}: ${key} resolved to nothing`, { + event: event.name, + }); +} + /** * Resolve and apply identity mapping. Calls analytics.identify() only - * when the resolved value differs from the last-fired identity. + * when the resolved value differs from the last-fired identity. False when + * no userId, anonymousId or traits resolved. */ async function applyIdentify( analytics: SegmentAnalytics, resolved: Record, state: RuntimeState, options?: SegmentEventOptions, -): Promise { +): Promise { const last = state.lastIdentity || {}; const userId = isString(resolved.userId) ? resolved.userId : undefined; const anonymousId = isString(resolved.anonymousId) @@ -154,12 +171,14 @@ async function applyIdentify( if (anonymousId && anonymousId !== last.anonymousId) await send(() => analytics.setAnonymousId(anonymousId)); + if (!userId && !anonymousId && !traits) return false; + const traitsHash = hashTraits(traits); const userIdChanged = userId !== last.userId; const traitsChanged = traitsHash !== (last.traitsHash ?? ''); const hasChange = userIdChanged || (traits !== undefined && traitsChanged); - if (!hasChange && !anonymousId) return; + if (!hasChange && !anonymousId) return true; await send(() => options @@ -172,17 +191,18 @@ async function applyIdentify( anonymousId: anonymousId ?? last.anonymousId, traitsHash, }; + return true; } -/** Resolve and apply the group mapping, once per change. */ +/** Resolve and apply the group mapping, once per change. False without a groupId. */ async function applyGroup( analytics: SegmentAnalytics, resolved: Record, state: RuntimeState, options?: SegmentEventOptions, -): Promise { +): Promise { const groupId = isString(resolved.groupId) ? resolved.groupId : undefined; - if (!groupId) return; + if (!groupId) return false; const traits = isObject(resolved.traits) && !isArray(resolved.traits) @@ -193,7 +213,7 @@ async function applyGroup( const traitsHash = hashTraits(traits); const groupIdChanged = groupId !== last.groupId; const traitsChanged = traitsHash !== (last.traitsHash ?? ''); - if (!groupIdChanged && !traitsChanged) return; + if (!groupIdChanged && !traitsChanged) return true; await send(() => options @@ -202,16 +222,17 @@ async function applyGroup( ); state.lastGroup = { groupId, traitsHash }; + return true; } -/** Resolve and apply the page mapping. */ +/** Resolve and apply the page mapping. False when it resolved to nothing. */ async function applyPage( analytics: SegmentAnalytics, pageMapping: unknown, event: WalkerOS.Event, collector: unknown, options?: SegmentEventOptions, -): Promise { +): Promise { // Boolean true → call page() with no args (SDK auto-collect). if (pageMapping === true) { await send(() => @@ -219,7 +240,7 @@ async function applyPage( ? analytics.page(undefined, undefined, undefined, options) : analytics.page(), ); - return; + return true; } const resolved = await getMappingValue( @@ -227,7 +248,7 @@ async function applyPage( pageMapping as Parameters[1], { collector } as Parameters[2], ); - if (!isObject(resolved)) return; + if (!isObject(resolved)) return false; const category = isString(resolved.category) ? resolved.category : undefined; const name = isString(resolved.name) ? resolved.name : undefined; @@ -250,6 +271,7 @@ async function applyPage( return analytics.page(category ?? null, name ?? null); return analytics.page(); }); + return true; } export const destinationSegment: Destination = { @@ -300,7 +322,7 @@ export const destinationSegment: Destination = { return { ...config, settings: { ...settings, _state } }; }, - async push(event, { config, rule, data, collector }) { + async push(event, { config, rule, data, collector, logger }) { const settings: Partial = config.settings || {}; const state: RuntimeState = settings._state || {}; const analytics = state.analytics; @@ -335,8 +357,10 @@ export const destinationSegment: Destination = { const resolved = await getMappingValue(event, identifyMapping, { collector, }); - if (isObject(resolved)) - await applyIdentify(analytics, resolved, state, options); + const applied = + isObject(resolved) && + (await applyIdentify(analytics, resolved, state, options)); + if (!applied) skipped(logger, event, 'identify', 'identify'); } // 3. Group - rule-level override wins over destination-level. @@ -345,19 +369,24 @@ export const destinationSegment: Destination = { const resolved = await getMappingValue(event, groupMapping, { collector, }); - if (isObject(resolved)) - await applyGroup(analytics, resolved, state, options); + const applied = + isObject(resolved) && + (await applyGroup(analytics, resolved, state, options)); + if (!applied) skipped(logger, event, 'group', 'group'); } // 4. Page (explicit first-class Segment pattern) - if (mappingSettings.page !== undefined) - await applyPage( + if ( + mappingSettings.page !== undefined && + !(await applyPage( analytics, mappingSettings.page, event, collector, options, - ); + )) + ) + skipped(logger, event, 'page', 'page'); // 5. Default track call unless rule opts out via silent. if (rule?.silent !== true) { diff --git a/packages/web/destinations/tiktok/src/__tests__/stepExamples.test.ts b/packages/web/destinations/tiktok/src/__tests__/stepExamples.test.ts index 124e6e139..e7a033135 100644 --- a/packages/web/destinations/tiktok/src/__tests__/stepExamples.test.ts +++ b/packages/web/destinations/tiktok/src/__tests__/stepExamples.test.ts @@ -1,6 +1,8 @@ import type { WalkerOS, Mapping as WalkerOSMapping } from '@walkeros/core'; import { startFlow } from '@walkeros/collector'; -import { clone } from '@walkeros/core'; +import type { Logger } from '@walkeros/core'; +import { clone, createEvent, Level } from '@walkeros/core'; +import { destinationTikTok } from '..'; import { examples } from '../dev'; import type { TikTokStepExample } from '../examples/step'; import type { Env, Settings, TTQ } from '../types'; @@ -96,4 +98,42 @@ describe('tiktok destination — step examples', () => { expect(actual).toEqual(expected); }, ); + + it('writes one DEBUG line naming the event and the mapping key for a declared skip', async () => { + const lines: Array<[Level, string, unknown]> = []; + const handler: Logger.Handler = (level, message, context) => { + lines.push([level, message, context]); + }; + const { elb } = await startFlow({ + logger: { level: 'DEBUG', handler }, + destinations: { + tiktok: { + code: { + ...destinationTikTok, + env: spyTtq(clone(examples.env.push)).env, + }, + config: { + settings: { apiKey: 'test-pixel-id' }, + mapping: { + company: { + update: { + silent: true, + settings: { identify: { map: { email: 'data.none' } } }, + }, + }, + }, + }, + }, + }, + }); + + expect((await elb(createEvent({ name: 'company update' }))).ok).toBe(true); + expect(lines.filter(([, message]) => message.includes('skipped'))).toEqual([ + [ + Level.DEBUG, + 'TikTok skipped identify: identify resolved to nothing', + expect.objectContaining({ event: 'company update' }), + ], + ]); + }); }); diff --git a/packages/web/destinations/tiktok/src/index.ts b/packages/web/destinations/tiktok/src/index.ts index ce1a90654..061b08155 100644 --- a/packages/web/destinations/tiktok/src/index.ts +++ b/packages/web/destinations/tiktok/src/index.ts @@ -1,3 +1,4 @@ +import type { Logger, WalkerOS } from '@walkeros/core'; import { getGrantedConsent, getMappingValue, @@ -79,7 +80,7 @@ async function applyIdentify( ttq: TTQ, resolved: Record, lastIdentity: NonNullable = {}, -): Promise> { +): Promise | undefined> { const next: NonNullable = { ...lastIdentity }; const params: IdentifyParams = {}; let changed = false; @@ -94,9 +95,10 @@ async function applyIdentify( } } - // Skip ttq.identify() entirely if the resolved object has no non-empty - // values (anonymous visitor) or if nothing changed. - if (!changed || Object.keys(params).length === 0) return next; + // No non-empty value resolved: nothing to identify. An identity that is + // already current skips ttq.identify() as well. + if (Object.keys(params).length === 0) return undefined; + if (!changed) return next; await stepCall( () => ttq.identify(params), @@ -106,6 +108,22 @@ async function applyIdentify( return next; } +/** + * A configured mapping that resolved to nothing skips its call: a declared + * skip, said once at DEBUG with the event and the mapping key, never the + * value, so a wrong mapping path does not look like an event without data. + */ +function skipped( + logger: Logger.Instance, + event: WalkerOS.Event, + call: string, + key: string, +): void { + logger.debug(`TikTok skipped ${call}: ${key} resolved to nothing`, { + event: event.name, + }); +} + export const destinationTikTok: Destination = { type: 'tiktok', @@ -149,7 +167,7 @@ export const destinationTikTok: Destination = { return { ...config, settings: { ...settings, _state } }; }, - async push(event, { config, rule, data, env, collector }) { + async push(event, { config, rule, data, env, collector, logger }) { vendor(config, env).check(); const ttq = getTTQ(env); const settings: Partial = config.settings || {}; @@ -166,13 +184,11 @@ export const destinationTikTok: Destination = { const resolved = await getMappingValue(event, identifyMapping, { collector, }); - if (isObject(resolved)) { - state.lastIdentity = await applyIdentify( - ttq, - resolved, - state.lastIdentity, - ); - } + const identity = isObject(resolved) + ? await applyIdentify(ttq, resolved, state.lastIdentity) + : undefined; + if (identity) state.lastIdentity = identity; + else skipped(logger, event, 'identify', 'identify'); } // Default track. rule.silent suppresses the call entirely (e.g., for From 436b8d0b8899ec4b36b5e1cc5920d5ba6d61e60e Mon Sep 17 00:00:00 2001 From: alexanderkirtzel Date: Wed, 21 Oct 2026 09:06:53 +0000 Subject: [PATCH 33/41] snowplow consent --- packages/web/destinations/snowplow/README.md | 15 ++++ .../snowplow/src/__tests__/consent.test.ts | 61 ++++++++++++++++- .../destinations/snowplow/src/index.test.ts | 36 ++++++++++ .../web/destinations/snowplow/src/index.ts | 68 ++++++++++++++----- .../destinations/snowplow/src/types/index.ts | 2 +- 5 files changed, 162 insertions(+), 20 deletions(-) diff --git a/packages/web/destinations/snowplow/README.md b/packages/web/destinations/snowplow/README.md index dfef24d37..1ab2618dc 100644 --- a/packages/web/destinations/snowplow/README.md +++ b/packages/web/destinations/snowplow/README.md @@ -42,6 +42,21 @@ npm install @walkeros/web-destination-snowplow } ``` +## Plugins + +URL plugins in `settings.plugins` load with sp.js. With `settings.tracker` (your +own browser tracker), init fails with a `config` error if any are set: load them +on your own tracker, or drop `settings.tracker` so walkerOS loads them. + +## Consent + +With `config.consent` set, a revocation of that consent stops walkerOS's events +and the senders that run on their own: activity pings (`activityTracking`) and +the sp.js plugins for link clicks, forms, buttons and YouTube. The next grant +starts them again. The error tracking and FocalMeter plugins have no off switch, +so a revocation fails with `consent-not-applied` while they keep sending: load +them only after consent. Other plugins are not switched. + ## Documentation Full configuration, mapping, and examples live in the docs: diff --git a/packages/web/destinations/snowplow/src/__tests__/consent.test.ts b/packages/web/destinations/snowplow/src/__tests__/consent.test.ts index 4cbe296b1..c027479d2 100644 --- a/packages/web/destinations/snowplow/src/__tests__/consent.test.ts +++ b/packages/web/destinations/snowplow/src/__tests__/consent.test.ts @@ -1,10 +1,11 @@ import { startFlow } from '@walkeros/collector'; -import { getEvent } from '@walkeros/core'; +import type { Logger } from '@walkeros/core'; +import { getEvent, Level } from '@walkeros/core'; import { destinationSnowplow } from '..'; import type { Env, SnowplowFunction } from '../types'; describe('snowplow destination consent', () => { - it('turns the tracker senders off on a revocation and on again on a grant', async () => { + it('turns activity pings and the plugin senders off on a revocation and on again on a grant', async () => { const calls: unknown[][] = []; const snowplow: SnowplowFunction = Object.assign( (...args: unknown[]) => { @@ -15,6 +16,7 @@ describe('snowplow destination consent', () => { const env: Env = { window: { snowplow } }; const activityTracking = { minimumVisitLength: 30, heartbeatDelay: 10 }; const linkClicks = { trackContent: true }; + const forms = { options: { forms: { allowlist: ['checkout'] } } }; const { elb } = await startFlow({ destinations: { snowplow: { @@ -33,6 +35,11 @@ describe('snowplow destination consent', () => { ], options: linkClicks, }, + { + url: 'https://cdn.example.com/form-tracking.js', + name: ['snowplowFormTracking', 'FormTrackingPlugin'], + options: forms, + }, ], }, }, @@ -48,6 +55,7 @@ describe('snowplow destination consent', () => { expect(calls).toEqual([ ['disableActivityTracking'], ['disableLinkClickTracking'], + ['disableFormTracking'], ]); calls.length = 0; @@ -55,6 +63,55 @@ describe('snowplow destination consent', () => { expect(calls).toEqual([ ['enableActivityTracking', activityTracking], ['enableLinkClickTracking', linkClicks], + ['enableFormTracking', forms], + ]); + }); + + it('fails a revocation with consent-not-applied for a plugin without an off switch', async () => { + const lines: Array<[Level, string, unknown]> = []; + const handler: Logger.Handler = (level, message, context) => { + lines.push([level, message, context]); + }; + const snowplow: SnowplowFunction = Object.assign(() => undefined, { + q: [], + }); + const env: Env = { window: { snowplow } }; + const { elb } = await startFlow({ + // The default level: only ERROR lines reach the handler. + logger: { handler }, + destinations: { + snowplow: { + code: { ...destinationSnowplow, env }, + config: { + consent: { marketing: true }, + settings: { + collectorUrl: 'https://collector.example.com', + plugins: [ + { + url: 'https://cdn.example.com/error-tracking.js', + name: ['snowplowErrorTracking', 'ErrorTrackingPlugin'], + }, + ], + }, + }, + }, + }, + }); + + await elb('walker consent', { marketing: true }); + expect(lines).toEqual([]); + + await elb('walker consent', { marketing: false }); + expect(lines).toEqual([ + [ + Level.ERROR, + expect.stringContaining('consent not applied'), + expect.objectContaining({ + code: 'consent-not-applied', + error: + 'Snowplow ErrorTrackingPlugin has no off switch, so it keeps sending after a revocation', + }), + ], ]); }); }); diff --git a/packages/web/destinations/snowplow/src/index.test.ts b/packages/web/destinations/snowplow/src/index.test.ts index 2e998607a..0fac7bf5e 100644 --- a/packages/web/destinations/snowplow/src/index.test.ts +++ b/packages/web/destinations/snowplow/src/index.test.ts @@ -5,6 +5,7 @@ import { startFlow } from '@walkeros/collector'; import { createEvent, getEvent, mockEnv } from '@walkeros/core'; import { clearUserData, + destinationSnowplow, enableAnonymousTracking, disableAnonymousTracking, SCHEMAS, @@ -714,6 +715,41 @@ describe('destination snowplow', () => { ); expect(enableCall).toBeDefined(); }); + + test('fails init with config when settings.tracker meets URL plugins', async () => { + const { elb: push } = await startFlow({ + destinations: { + snowplow: { + // The module's own export: StepError's class must be the + // collector's, which a requireActual copy is not. + code: destinationSnowplow, + config: { + settings: { + collectorUrl: 'https://collector.example.com', + tracker: { + newTracker: () => undefined, + trackSelfDescribingEvent: () => undefined, + }, + plugins: [ + { + url: 'https://cdn.example.com/form-tracking.js', + name: ['snowplowFormTracking', 'FormTrackingPlugin'], + }, + ], + }, + }, + }, + }, + }); + + const result = await push(getEvent('page view')); + expect(result.ok).toBe(false); + expect(result.failed?.snowplow?.reason).toMatchObject({ + code: 'config', + message: + 'Snowplow loads the URL plugins in settings.plugins (FormTrackingPlugin) only with sp.js, not with settings.tracker: load them on your own tracker, or drop settings.tracker so walkerOS loads them', + }); + }); }); describe('anonymousTracking', () => { diff --git a/packages/web/destinations/snowplow/src/index.ts b/packages/web/destinations/snowplow/src/index.ts index e67a6c1d5..8a61eee39 100644 --- a/packages/web/destinations/snowplow/src/index.ts +++ b/packages/web/destinations/snowplow/src/index.ts @@ -56,11 +56,29 @@ function vendor( } /** - * Turns the tracker's own senders off on a revocation of the destination's - * required consent and on again on the next grant: activity pings, and link - * click tracking where a URL plugin enabled it. The required consent decides - * on the whole consent state, as it gates the pushes, and only a change that - * names a required key counts. + * The senders a URL plugin's enable method starts, each with the call that + * stops it (tracker and plugins 4.8.1). An empty entry is a sender without + * an off switch: a revocation cannot stop it. + */ +const OFF_SWITCHES = new Map([ + ['enableLinkClickTracking', 'disableLinkClickTracking'], + ['enableFormTracking', 'disableFormTracking'], + ['enableButtonClickTracking', 'disableButtonClickTracking'], + ['enableYouTubeTracking', 'disableYouTubeTracking'], + ['enableErrorTracking', undefined], + ['enableFocalMeterIntegration', undefined], +]); + +/** + * Follows the destination's required consent with the senders that run + * outside walkerOS's pushes. A revocation calls disableActivityTracking + * (when activityTracking is set) and the off switch of every sp.js URL + * plugin listed in OFF_SWITCHES; the next grant enables them again with + * their options. A listed plugin without an off switch (error tracking, + * FocalMeter) fails the revocation with `consent-not-applied`. Other + * plugins are left as they are. The required consent decides on the whole + * consent state, as it gates the pushes, and only a change that names a + * required key counts. */ async function followConsent( settings: Partial, @@ -76,13 +94,14 @@ async function followConsent( const granted = getGrantedConsent(required, consent) !== false; if (granted !== Boolean(state.paused)) return; - const linkClicks = (settings.plugins ?? []) + const senders = (settings.plugins ?? []) .filter(isUrlBasedPlugin) - .filter( - (plugin) => - (plugin.enableMethod ?? deriveEnableMethod(plugin.name[1])) === - 'enableLinkClickTracking', - ); + .flatMap((plugin) => { + const enable = plugin.enableMethod ?? deriveEnableMethod(plugin.name[1]); + return OFF_SWITCHES.has(enable) + ? [{ plugin, enable, disable: OFF_SWITCHES.get(enable) }] + : []; + }); const { activityTracking } = settings; await stepCall( @@ -91,17 +110,24 @@ async function followConsent( if (granted) adapter.enableActivityTracking(activityTracking); else adapter.disableActivityTracking(); } - for (const plugin of linkClicks) { - if (!granted) adapter.call('disableLinkClickTracking'); - else if (plugin.options) - adapter.call('enableLinkClickTracking', plugin.options); - else adapter.call('enableLinkClickTracking'); + for (const { plugin, enable, disable } of senders) { + if (!disable) continue; + if (!granted) adapter.call(disable); + else if (plugin.options) adapter.call(enable, plugin.options); + else adapter.call(enable); } }, 'consent-not-applied', 'Snowplow did not take the consent update', ); state.paused = !granted; + + const unstoppable = senders.filter(({ disable }) => !disable); + if (!granted && unstoppable.length) + throw new StepError( + `Snowplow ${unstoppable.map(({ plugin }) => plugin.name[1]).join(', ')} has no off switch, so it keeps sending after a revocation`, + { code: 'consent-not-applied' }, + ); } /** @@ -279,6 +305,13 @@ export const destinationSnowplow: Destination = { 'tracker.trackSelfDescribingEvent is required for browser-tracker mode', { code: 'config' }, ); + // URL plugins load through sp.js, which this mode does not use. + const urlPlugins = (settings.plugins ?? []).filter(isUrlBasedPlugin); + if (urlPlugins.length) + throw new StepError( + `Snowplow loads the URL plugins in settings.plugins (${urlPlugins.map((plugin) => plugin.name[1]).join(', ')}) only with sp.js, not with settings.tracker: load them on your own tracker, or drop settings.tracker so walkerOS loads them`, + { code: 'config' }, + ); // Initialize tracker await stepCall( @@ -453,7 +486,8 @@ export const destinationSnowplow: Destination = { // Types; tracked as a follow-up. These two `as` casts are that boundary. const settings = (context.config?.settings || {}) as Partial; - // The tracker's own senders follow the destination's required consent + // Activity pings and the plugin senders with an off switch follow the + // destination's required consent await followConsent( settings, context.config.consent, diff --git a/packages/web/destinations/snowplow/src/types/index.ts b/packages/web/destinations/snowplow/src/types/index.ts index c05116910..a10ef55ff 100644 --- a/packages/web/destinations/snowplow/src/types/index.ts +++ b/packages/web/destinations/snowplow/src/types/index.ts @@ -388,7 +388,7 @@ export interface RuntimeState { userIdSet?: boolean; /** The initialized adapter instance */ adapter?: SnowplowAdapter; - /** The tracker's own senders are off after a consent revocation */ + /** Activity pings and the plugin senders with an off switch are off after a consent revocation */ paused?: boolean; } From 7e5fc2753102c815cb49f5f636ef90f1f004d551 Mon Sep 17 00:00:00 2001 From: alexanderkirtzel Date: Wed, 21 Oct 2026 09:20:22 +0000 Subject: [PATCH 34/41] docs fixes --- .changeset/cli-server-flow-defaults.md | 3 +- .changeset/collector-held-event-consent.md | 8 +++++ .changeset/collector-no-effect-config.md | 2 +- .changeset/collector-transformer-timeout.md | 7 ++++ .changeset/collector-walkerjs-size.md | 4 +-- .changeset/core-json-log-truncation.md | 8 +++++ .changeset/runner-flow-logger.md | 7 ++-- ...erver-destination-amplitude-group-names.md | 7 ++++ ...server-destination-gcp-rejected-appends.md | 7 ++++ .changeset/server-destination-kafka-retry.md | 7 ++++ ...rver-destination-slack-thread-broadcast.md | 7 ++++ .../server-destination-sqlite-final-errors.md | 7 ++++ .../server-destinations-report-failures.md | 2 +- .changeset/server-destinations-retry-dedup.md | 11 ++++++ .changeset/server-source-express-body-log.md | 7 ++-- .changeset/server-source-fetch-batch-lines.md | 7 ++++ .changeset/server-store-fs-directory-miss.md | 7 ++++ .changeset/transformer-ga4-max-events.md | 7 ++++ .changeset/web-destination-clarity-consent.md | 8 +++++ .../web-destination-snowplow-consent.md | 8 +++++ .changeset/web-destinations-consent-state.md | 11 ++++++ .changeset/web-destinations-declared-skips.md | 18 ++++++++++ .../web-destinations-passthrough-settings.md | 13 +++++++ .changeset/web-destinations-unmapped-fail.md | 9 +++++ skills/walkeros-create-destination/SKILL.md | 11 ++++-- .../templates/simple/index.ts | 36 ++++++++++++------- skills/walkeros-using-step-examples/SKILL.md | 8 +++-- website/docs/apps/cli.mdx | 2 +- website/docs/collector/index.mdx | 5 ++- website/docs/collector/logger.mdx | 16 +++++---- website/docs/destinations/create-your-own.mdx | 33 +++++++++-------- website/docs/destinations/index.mdx | 10 +++++- .../docs/destinations/server/amplitude.mdx | 15 ++------ website/docs/destinations/server/gcp.mdx | 1 + website/docs/destinations/server/kafka.mdx | 2 +- website/docs/destinations/server/klaviyo.mdx | 2 +- website/docs/destinations/server/linkedin.mdx | 2 +- website/docs/destinations/server/piwikpro.mdx | 13 ++++--- website/docs/destinations/server/posthog.mdx | 13 +------ .../docs/destinations/server/rudderstack.mdx | 13 +------ website/docs/destinations/server/segment.mdx | 13 +------ website/docs/destinations/server/slack.mdx | 2 ++ website/docs/destinations/server/sqlite.mdx | 2 +- website/docs/destinations/server/twitter.mdx | 2 +- website/docs/destinations/web/amplitude.mdx | 14 ++++---- website/docs/destinations/web/clarity.mdx | 2 +- website/docs/destinations/web/linkedin.mdx | 8 +++-- website/docs/destinations/web/matomo.mdx | 10 +++--- website/docs/destinations/web/optimizely.mdx | 10 +++--- website/docs/destinations/web/piwikpro.mdx | 9 +++-- website/docs/destinations/web/snowplow.mdx | 8 +++++ website/docs/mapping/value.mdx | 2 +- website/docs/sources/server/aws.mdx | 10 +----- website/docs/sources/server/express.mdx | 12 ++----- website/docs/sources/server/fetch.mdx | 12 ++----- website/docs/sources/server/gcp.mdx | 10 +----- website/docs/stores/index.mdx | 2 +- website/docs/transformers/fingerprint.mdx | 2 +- website/docs/transformers/ga4.mdx | 2 +- website/docs/transformers/index.mdx | 9 ++++- 60 files changed, 325 insertions(+), 180 deletions(-) create mode 100644 .changeset/collector-held-event-consent.md create mode 100644 .changeset/collector-transformer-timeout.md create mode 100644 .changeset/core-json-log-truncation.md create mode 100644 .changeset/server-destination-amplitude-group-names.md create mode 100644 .changeset/server-destination-gcp-rejected-appends.md create mode 100644 .changeset/server-destination-kafka-retry.md create mode 100644 .changeset/server-destination-slack-thread-broadcast.md create mode 100644 .changeset/server-destination-sqlite-final-errors.md create mode 100644 .changeset/server-destinations-retry-dedup.md create mode 100644 .changeset/server-source-fetch-batch-lines.md create mode 100644 .changeset/server-store-fs-directory-miss.md create mode 100644 .changeset/transformer-ga4-max-events.md create mode 100644 .changeset/web-destination-clarity-consent.md create mode 100644 .changeset/web-destination-snowplow-consent.md create mode 100644 .changeset/web-destinations-consent-state.md create mode 100644 .changeset/web-destinations-declared-skips.md create mode 100644 .changeset/web-destinations-passthrough-settings.md create mode 100644 .changeset/web-destinations-unmapped-fail.md diff --git a/.changeset/cli-server-flow-defaults.md b/.changeset/cli-server-flow-defaults.md index cdadf1df0..9a60f5117 100644 --- a/.changeset/cli-server-flow-defaults.md +++ b/.changeset/cli-server-flow-defaults.md @@ -6,4 +6,5 @@ Server flows skip consent-denied events by default (`collector.consentDenied: 'skip'`; an authored value wins). A flow's own `logger` level, format and handler apply under `runneros`. `walkeros setup` fails with the reason when a step's `init` throws or returns `false`. Inline -`on`, `pushBatch` and `destroy` reach the bundle. +`on`, `pushBatch` and `destroy` reach the bundle. A flow's custom log handler +gets a no-op `originalHandler`. diff --git a/.changeset/collector-held-event-consent.md b/.changeset/collector-held-event-consent.md new file mode 100644 index 000000000..404c5f37f --- /dev/null +++ b/.changeset/collector-held-event-consent.md @@ -0,0 +1,8 @@ +--- +'@walkeros/collector': patch +--- + +A held event keeps the consent it was admitted with: when its destination's hold +ends it is delivered even if the collector's consent changed in the meantime. A +mapping value problem now logs once per mapping rule instead of once for all +rules, so a second rule's problem is no longer hidden behind the first. diff --git a/.changeset/collector-no-effect-config.md b/.changeset/collector-no-effect-config.md index c9a911a76..1023e63a7 100644 --- a/.changeset/collector-no-effect-config.md +++ b/.changeset/collector-no-effect-config.md @@ -4,4 +4,4 @@ A destination whose config sets `batch` without a `pushBatch`, or a mapping key with no effect at its position, logs one ERROR when it registers and still runs. -The orphan connection error that lost no event logs at WARN instead of ERROR. +A connection error that lost no event logs at WARN instead of ERROR. diff --git a/.changeset/collector-transformer-timeout.md b/.changeset/collector-transformer-timeout.md new file mode 100644 index 000000000..0d18fe320 --- /dev/null +++ b/.changeset/collector-transformer-timeout.md @@ -0,0 +1,7 @@ +--- +'@walkeros/collector': patch +--- + +Transformers accept `config.timeout` in milliseconds (default 10000). It bounds +`init` and each `push`; a push that does not settle fails the event with code +`timeout` instead of leaving the push unanswered. diff --git a/.changeset/collector-walkerjs-size.md b/.changeset/collector-walkerjs-size.md index ff7cc19e3..2c0b78f41 100644 --- a/.changeset/collector-walkerjs-size.md +++ b/.changeset/collector-walkerjs-size.md @@ -3,5 +3,5 @@ --- The web bundle `walker.js` carries the full failure contract, the circuit -breaker included, with no lean variant that leaves it out. It grows from 35,181 -bytes to 39,334 bytes, measured as gzip -9, and the bundle is not size-gated. +breaker included. It grows from 35,181 bytes to 39,334 bytes, measured as gzip +-9, and its size is not gated by a budget. diff --git a/.changeset/core-json-log-truncation.md b/.changeset/core-json-log-truncation.md new file mode 100644 index 000000000..e009e7d5a --- /dev/null +++ b/.changeset/core-json-log-truncation.md @@ -0,0 +1,8 @@ +--- +'@walkeros/core': patch +--- + +A JSON log line above 32 KiB now truncates its message as well as its context +and sets `messageTruncated: true`, so one oversized line can no longer break a +log pipeline. Mapping problems report under the step `mapping.`, one line +per rule. diff --git a/.changeset/runner-flow-logger.md b/.changeset/runner-flow-logger.md index e219b4c7c..8cec6a09e 100644 --- a/.changeset/runner-flow-logger.md +++ b/.changeset/runner-flow-logger.md @@ -3,6 +3,7 @@ --- A flow's `logger.level` and `logger.format` apply to the collector's lines, and -the format to the runner's own lines; `--verbose` still forces DEBUG. -`flow handler error` logs at ERROR, a rejected heartbeat logs a WARN, and Node -process warnings go through the logger at WARN. +the format to the runner's own lines; `--verbose` still forces DEBUG. A flow's +`logger.level`, DEBUG included, applies to the collector's lines without +`--verbose`. `flow handler error` logs at ERROR, a rejected heartbeat at WARN, +and Node process warnings go through the logger at WARN. diff --git a/.changeset/server-destination-amplitude-group-names.md b/.changeset/server-destination-amplitude-group-names.md new file mode 100644 index 000000000..31dc31b25 --- /dev/null +++ b/.changeset/server-destination-amplitude-group-names.md @@ -0,0 +1,7 @@ +--- +'@walkeros/server-destination-amplitude': patch +--- + +A group name list that holds a non-string is skipped with a DEBUG line naming +the call and the mapping key instead of being sent to Amplitude, so malformed +group data is never delivered. diff --git a/.changeset/server-destination-gcp-rejected-appends.md b/.changeset/server-destination-gcp-rejected-appends.md new file mode 100644 index 000000000..e25d6223c --- /dev/null +++ b/.changeset/server-destination-gcp-rejected-appends.md @@ -0,0 +1,7 @@ +--- +'@walkeros/server-destination-gcp': patch +--- + +An append that the BigQuery server rejects (quota, permission, a dropped table) +now fails the push instead of counting as delivered. Pub/Sub retries only the +error codes its SDK retries. diff --git a/.changeset/server-destination-kafka-retry.md b/.changeset/server-destination-kafka-retry.md new file mode 100644 index 000000000..22f137670 --- /dev/null +++ b/.changeset/server-destination-kafka-retry.md @@ -0,0 +1,7 @@ +--- +'@walkeros/server-destination-kafka': patch +--- + +A send is retried only when kafkajs marks the error as retriable. Deterministic +failures, such as an unknown topic or a rejected message, are final and fail the +push once instead of being retried. diff --git a/.changeset/server-destination-slack-thread-broadcast.md b/.changeset/server-destination-slack-thread-broadcast.md new file mode 100644 index 000000000..537cf95d9 --- /dev/null +++ b/.changeset/server-destination-slack-thread-broadcast.md @@ -0,0 +1,7 @@ +--- +'@walkeros/server-destination-slack': patch +--- + +`reply_broadcast` is sent only together with a thread. A broadcast flag set on a +message that has no thread to reply to is no longer passed on to Slack, so the +message is posted as an ordinary message. diff --git a/.changeset/server-destination-sqlite-final-errors.md b/.changeset/server-destination-sqlite-final-errors.md new file mode 100644 index 000000000..3fa472cce --- /dev/null +++ b/.changeset/server-destination-sqlite-final-errors.md @@ -0,0 +1,7 @@ +--- +'@walkeros/server-destination-sqlite': patch +--- + +A constraint error on insert is final and not retried, and a missing table named +by a rule's `table` setting is a `config` error. The legacy `schema: 'auto'` +also creates the rule tables. diff --git a/.changeset/server-destinations-report-failures.md b/.changeset/server-destinations-report-failures.md index fe37f9e0e..24c824274 100644 --- a/.changeset/server-destinations-report-failures.md +++ b/.changeset/server-destinations-report-failures.md @@ -21,4 +21,4 @@ A failed request is reported as a failed push (dead-letter queue, one ERROR line with the status) instead of counting as delivered or being logged at debug. Events a destination cannot send, such as a missing user identifier, fail with -`missing-input`. Rules with `silent: true` send nothing. +`missing-input`. Rules with `silent: true` send no event of their own. diff --git a/.changeset/server-destinations-retry-dedup.md b/.changeset/server-destinations-retry-dedup.md new file mode 100644 index 000000000..e0f753261 --- /dev/null +++ b/.changeset/server-destinations-retry-dedup.md @@ -0,0 +1,11 @@ +--- +'@walkeros/server-destination-segment': patch +'@walkeros/server-destination-rudderstack': patch +'@walkeros/server-destination-posthog': patch +'@walkeros/server-destination-amplitude': patch +--- + +Retries stay inside the destination's `config.timeout`: a quarter of it per +attempt, at most two retries, and a value set in the settings wins. RudderStack +gains a `timeout` setting. Each event carries `event.id` as the vendor's dedup +id, so at-least-once delivery drops duplicates. diff --git a/.changeset/server-source-express-body-log.md b/.changeset/server-source-express-body-log.md index 782c7f707..ec160c64b 100644 --- a/.changeset/server-source-express-body-log.md +++ b/.changeset/server-source-express-body-log.md @@ -2,6 +2,7 @@ '@walkeros/server-source-express': patch --- -The `Request body rejected` debug line carries only the status and the content -type, no longer a body sample or the parser message, so a rejected request body -can never leak its content into the logs. +The `Request body rejected` debug line carries only the status and content type, +never a body sample or the parser message. In respond-first mode a later +per-event failure logs at DEBUG, because the collector's ERROR line is the +signal. diff --git a/.changeset/server-source-fetch-batch-lines.md b/.changeset/server-source-fetch-batch-lines.md new file mode 100644 index 000000000..fe0feec76 --- /dev/null +++ b/.changeset/server-source-fetch-batch-lines.md @@ -0,0 +1,7 @@ +--- +'@walkeros/server-source-fetch': patch +--- + +A per-event "not processed" answer in a batch logs at DEBUG, because the +collector's ERROR line is the signal. A push that throws inside a batch logs an +ERROR and answers `Internal server error`, never the error text. diff --git a/.changeset/server-store-fs-directory-miss.md b/.changeset/server-store-fs-directory-miss.md new file mode 100644 index 000000000..954e0d977 --- /dev/null +++ b/.changeset/server-store-fs-directory-miss.md @@ -0,0 +1,7 @@ +--- +'@walkeros/server-store-fs': patch +--- + +A path that is a directory is a miss, like a missing file, instead of a failed +read. A read that fails for another reason, such as a permission error, still +throws and is reported by the collector. diff --git a/.changeset/transformer-ga4-max-events.md b/.changeset/transformer-ga4-max-events.md new file mode 100644 index 000000000..efe624cca --- /dev/null +++ b/.changeset/transformer-ga4-max-events.md @@ -0,0 +1,7 @@ +--- +'@walkeros/transformer-ga4': patch +--- + +A request with more events than `settings.maxEvents` is dropped whole and now +writes an ERROR line, no longer a WARN, because those events are lost. Raise +`maxEvents` if your own server-to-server calls send larger batches. diff --git a/.changeset/web-destination-clarity-consent.md b/.changeset/web-destination-clarity-consent.md new file mode 100644 index 000000000..04697f6a2 --- /dev/null +++ b/.changeset/web-destination-clarity-consent.md @@ -0,0 +1,8 @@ +--- +'@walkeros/web-destination-clarity': patch +--- + +Without `settings.consent`, a consent-gated destination grants +`analytics_Storage` only and denies both categories on a revocation. +`ad_Storage` follows only `settings.consent`, so a grant no longer switches +advertising storage on by itself. diff --git a/.changeset/web-destination-snowplow-consent.md b/.changeset/web-destination-snowplow-consent.md new file mode 100644 index 000000000..45ee0d843 --- /dev/null +++ b/.changeset/web-destination-snowplow-consent.md @@ -0,0 +1,8 @@ +--- +'@walkeros/web-destination-snowplow': patch +--- + +On a revocation, activity tracking and the link, form, button and YouTube +plugins switch off and come back on a grant. Error tracking and FocalMeter have +no off switch, so a revocation fails visibly (`consent-not-applied`). With +`settings.tracker`, URL plugins fail at `init` with a `config` error. diff --git a/.changeset/web-destinations-consent-state.md b/.changeset/web-destinations-consent-state.md new file mode 100644 index 000000000..71e770664 --- /dev/null +++ b/.changeset/web-destinations-consent-state.md @@ -0,0 +1,11 @@ +--- +'@walkeros/web-destination-mixpanel': patch +'@walkeros/web-destination-posthog': patch +'@walkeros/web-destination-tiktok': patch +'@walkeros/web-destination-pinterest': patch +'@walkeros/web-destination-optimizely': patch +--- + +A consent handler decides on the whole consent state against the destination's +own `config.consent` and acts only when a change touches its required keys. +Optimizely keeps its client across a revoke and a grant, instead of closing it. diff --git a/.changeset/web-destinations-declared-skips.md b/.changeset/web-destinations-declared-skips.md new file mode 100644 index 000000000..0170720a9 --- /dev/null +++ b/.changeset/web-destinations-declared-skips.md @@ -0,0 +1,18 @@ +--- +'@walkeros/web-destination-amplitude': patch +'@walkeros/web-destination-segment': patch +'@walkeros/web-destination-mixpanel': patch +'@walkeros/web-destination-posthog': patch +'@walkeros/web-destination-fullstory': patch +'@walkeros/web-destination-heap': patch +'@walkeros/web-destination-piwikpro': patch +'@walkeros/web-destination-optimizely': patch +'@walkeros/web-destination-pinterest': patch +'@walkeros/web-destination-tiktok': patch +'@walkeros/web-destination-snowplow': patch +--- + +A silent rule whose identify, group or similar mapping resolves to nothing sends +nothing, counts as delivered and writes one DEBUG line naming the call and +mapping key, never the value. An identity or group that is already current +writes nothing. diff --git a/.changeset/web-destinations-passthrough-settings.md b/.changeset/web-destinations-passthrough-settings.md new file mode 100644 index 000000000..081732bc0 --- /dev/null +++ b/.changeset/web-destinations-passthrough-settings.md @@ -0,0 +1,13 @@ +--- +'@walkeros/web-destination-amplitude': patch +'@walkeros/web-destination-segment': patch +'@walkeros/web-destination-mixpanel': patch +'@walkeros/web-destination-posthog': patch +'@walkeros/web-destination-tiktok': patch +'@walkeros/web-destination-d8a': patch +--- + +Settings accept SDK options that the schema does not list and pass them through +to the SDK, so a newer SDK option no longer fails validation. Amplitude events +from a setup with plugins no longer carry the `amplitude-ts-unified` library +prefix. diff --git a/.changeset/web-destinations-unmapped-fail.md b/.changeset/web-destinations-unmapped-fail.md new file mode 100644 index 000000000..7373822ac --- /dev/null +++ b/.changeset/web-destinations-unmapped-fail.md @@ -0,0 +1,9 @@ +--- +'@walkeros/web-destination-snowplow': patch +'@walkeros/web-destination-optimizely': patch +--- + +An event that no rule maps fails with `missing-input` in Snowplow; add an +`ignore: true` rule for events you do not send. Optimizely fails events from +visitors without the configured `userId` with `missing-input` instead of +dropping them. diff --git a/skills/walkeros-create-destination/SKILL.md b/skills/walkeros-create-destination/SKILL.md index 80a6d2b2e..7fc89ed96 100644 --- a/skills/walkeros-create-destination/SKILL.md +++ b/skills/walkeros-create-destination/SKILL.md @@ -669,9 +669,14 @@ Use these templates as your starting point: `push`; for any other, `await ready()`. Pass attributes the tag needs (`data-domain`) through `options.attributes`. 10. **Honour `rule.silent`.** A rule with `silent: true` sends no event of its - own and runs only the side effects it maps (identify, group). With nothing - to send it returns normally. Every destination ships a `silent` step example - (see Phase 3). + own and runs only the side effects it maps (identify, group). Do not + `return` early for it (`walkeros/no-silent-return` is an error): write + `push = async (event, context) => { if (!context.rule?.silent) await send(event, context); }` + and run the side effects around it. A silent rule whose side-effect mapping + resolves to nothing sends nothing, counts as delivered and writes one DEBUG + line (` skipped : resolved to nothing`, the event + name, never the value). Every destination ships a `silent` step example (see + Phase 3). 11. **Consent reaches the vendor.** `config.consent` decides: in `on('consent')`, tell the vendor about a revocation and a grant when it has a switch (`getGrantedConsent(config.consent, collector.consent)`), and throw diff --git a/skills/walkeros-create-destination/templates/simple/index.ts b/skills/walkeros-create-destination/templates/simple/index.ts index da20cb29b..d2ba7d924 100644 --- a/skills/walkeros-create-destination/templates/simple/index.ts +++ b/skills/walkeros-create-destination/templates/simple/index.ts @@ -1,4 +1,5 @@ -import type { Config, Destination, Env } from './types'; +import type { Config, Destination, Env, Types } from './types'; +import type { Destination as CoreDestination, WalkerOS } from '@walkeros/core'; import { getGrantedConsent, isObject, StepError } from '@walkeros/core'; import { getEnv, loadVendor } from '@walkeros/web-core'; @@ -56,6 +57,19 @@ async function call( } } +/** The vendor call that sends one event. */ +async function send( + event: WalkerOS.Event, + { data, env }: CoreDestination.PushContext, +): Promise { + const params = isObject(data) ? data : {}; + await call( + () => sdk(env)('track', event.name, params), + 'transport', + 'Vendor call failed', + ); +} + export const destinationVendor: Destination = { type: 'vendor', config: {}, @@ -88,21 +102,17 @@ export const destinationVendor: Destination = { /** * Push event to destination. `data` is the mapped data, `rule` the matching - * mapping rule. + * mapping rule. A silent rule sends no event of its own: `push` hands the + * event to `send` unless the rule is silent, and never returns early (the + * `walkeros/no-silent-return` lint rule is an error). A destination with + * side-effect mappings (identify, group) runs them in `push` for a silent + * rule too, and writes one DEBUG line when they resolve to nothing. */ - async push(event, { config, rule, data, env }) { + async push(event, context) { // A script that fired `error` fails every later push - vendor(config, env).check(); + vendor(context.config, context.env).check(); - // A silent rule sends no event of its own - if (rule?.silent) return; - - const params = isObject(data) ? data : {}; - await call( - () => sdk(env)('track', event.name, params), - 'transport', - 'Vendor call failed', - ); + if (!context.rule?.silent) await send(event, context); }, /** Tell the vendor about a consent change when it has a switch for it. */ diff --git a/skills/walkeros-using-step-examples/SKILL.md b/skills/walkeros-using-step-examples/SKILL.md index 16b2e47d3..cfc643786 100644 --- a/skills/walkeros-using-step-examples/SKILL.md +++ b/skills/walkeros-using-step-examples/SKILL.md @@ -186,8 +186,10 @@ flatten into a single array in execution order: import type { Flow } from '@walkeros/core'; import { getEvent } from '@walkeros/core'; +// getEvent pins the timestamp (1700000000000) and derives the id from the event +// name; pass `id` when the expected output must name a readable one. export const purchase: Flow.StepExample = { - in: getEvent('order complete', { timestamp: 1700000000 }), + in: getEvent('order complete', { id: '1700000000-gr0up-1' }), mapping: { name: 'Purchase', data: { map: { value: 'data.value', currency: { value: 'EUR' } } }, @@ -321,7 +323,7 @@ import type { Flow } from '@walkeros/core'; import { getEvent } from '@walkeros/core'; export const purchase: Flow.StepExample = { - in: getEvent('order complete', { timestamp: 1700000000 }), + in: getEvent('order complete', { id: '1700000000-gr0up-1' }), mapping: { name: 'Purchase', data: { map: { value: 'data.value' } } }, out: ['track', 'Purchase', { value: 555 }, { eventID: '1700000000-gr0up-1' }], }; @@ -461,7 +463,7 @@ export const purchase: Flow.StepExample = { title: 'Purchase', description: 'A completed order mapped to the Meta Pixel Purchase standard event.', - in: getEvent('order complete', { timestamp: 1700000000 }), + in: getEvent('order complete', { id: '1700000000-gr0up-1' }), mapping: { name: 'Purchase', data: { map: { value: 'data.value' } } }, out: [['fbq', 'track', 'Purchase', { value: 555, currency: 'EUR' }]], }; diff --git a/website/docs/apps/cli.mdx b/website/docs/apps/cli.mdx index 13b70832d..1b3311893 100644 --- a/website/docs/apps/cli.mdx +++ b/website/docs/apps/cli.mdx @@ -1311,7 +1311,7 @@ When the file has a root `contract`, the default run validates it exactly as `wa Every step with a `package` has its resolved `config.settings` checked against the package's settings schema, at the version the bundle would install: the `config.bundle.packages` pin, else the inline `@version`, else latest. A local `path` package is read from disk. -- An unknown settings key is an error (`ENTRY_SCHEMA`, keyword `additionalProperties`) where the package closes its schema. Other findings are warnings (`ENTRY_SCHEMA`, with `keyword`) for one minor release; they become errors after that. `--strict` fails on them now. +- An unknown settings key is an error (`ENTRY_SCHEMA`, keyword `additionalProperties`) where the package closes its schema. Other findings are warnings (`ENTRY_SCHEMA`, with `keyword`); `--strict` fails on them. - Placeholders: `$var` is resolved first and the value checked. `$env.NAME:default` is checked using the default, and an error names the default, never the placeholder. `$flow` resolves statically; an unresolvable `$flow` is an error. Values known only at runtime (`$env.NAME` without default, `$secret`, `$store`, `$code`) are not checked and are listed in `details.deferred`. - A schema that cannot be fetched is a skip (`SCHEMA_UNAVAILABLE`); a package without a settings schema is a skip (`NO_SETTINGS_SCHEMA`). - **Steps that import a named export** are checked against that export's schema. The export is picked as the bundler picks it: the step's `import`, else the first `imports` entry of its `config.bundle.packages` pin, else the default export. A package with several exports publishes a settings schema per export (`exportSchemas` in its `walkerOS.json`); a package with one export uses its `settings` schema for a named import too. An export missing from `exportSchemas`, a multi-export package version published before per-export schemas, or a package version that does not list its exports, is a skip (`NO_SETTINGS_SCHEMA`), so `--strict` fails on it. diff --git a/website/docs/collector/index.mdx b/website/docs/collector/index.mdx index ed85f515f..29a868e58 100644 --- a/website/docs/collector/index.mdx +++ b/website/docs/collector/index.mdx @@ -263,7 +263,7 @@ event ends in one of three places: * `done`: the destination delivered the event, * `queued`: the event is waiting, with a `reason`: `consent` (the destination waits for a grant), `batch` (it sits in a batch until the flush) or `held` - (the destination had a failure and retries after a window), + (a state delivery to the destination, such as its consent handler, is running or failed; the destination retries once per window), * `failed`: the destination did not deliver it, with a `reason` that carries a `code`, a `message` and `retryable` (see the table in [Logger](./logger#failure-codes)). @@ -351,6 +351,9 @@ warning. language="json" /> +A held event keeps the consent it was admitted with: when its destination's +hold ends it is delivered even if the collector's consent changed meanwhile. + ### Flushing batches `walker flush` sends every pending destination batch at once and leaves the diff --git a/website/docs/collector/logger.mdx b/website/docs/collector/logger.mdx index 2513b87ca..5d2e52152 100644 --- a/website/docs/collector/logger.mdx +++ b/website/docs/collector/logger.mdx @@ -203,10 +203,10 @@ see each line. | Code | Cause | Retryable | Fix | | ---- | ----- | --------- | --- | | `transport` | A destination's request failed or threw an unknown error | yes, except a `4xx` other than `429` | Check the endpoint and credentials; the status is on the line | -| `timeout` | A push or an `init` did not settle within `config.timeout` (10 s) | yes | Raise `config.timeout` or check the vendor | +| `timeout` | A destination or transformer push or `init` did not settle within its `config.timeout` (10 s by default) | yes | Raise `config.timeout` or check the vendor | | `not-ready` | The destination or transformer could not start | no | Fix the cause its message names | -| `breaker-open` | Repeated retryable failures opened the circuit breaker; the destination is paused | yes | Wait, or fix the outage; the destination retries after a pause | -| `blocked` | The destination was held for longer than one retry window | yes | Fix the failing call; later events retry | +| `breaker-open` | `config.breaker` is set and `threshold` retryable failures in a row opened it; events fail at once | yes | Fix the outage. After `breaker.cooldown` (30 s) the next event probes the destination and a success closes the breaker; events failed while it was open stay in the dead-letter queue | +| `blocked` | A state delivery to the destination (its `on` handler, for example consent) failed and did not recover within `config.timeout` | yes | Fix the destination's `on` handler (see its `consent-not-applied` line); later pushes retry the delivery, and events fail until one succeeds | | `queue-full` | A queue refused the event: the collector has not run yet, or a held destination's queue is full | yes | Start the collector sooner or raise `queueMax` | | `mapping` | A mapping rule `condition` threw, or a mapped value could not be built | no | Fix the condition or the value | | `transformer` | A transformer threw | no | Fix the transformer | @@ -216,7 +216,10 @@ see each line. | `vendor-not-ready` | The vendor SDK is not ready yet | yes | Usually resolves on the next event | | `vendor-timeout` | The vendor script did not become ready in time | yes | Check the network and the script URL | | `beacon-rejected` | The browser refused a `sendBeacon` | only up to 64 KiB | Send smaller batches | -| `consent-not-applied` | A destination's `on('consent')` handler failed | n/a | Fix the handler; the destination's events wait | +| `consent-not-applied` | A destination's `on('consent')` handler failed | n/a | Fix the handler; the destination's events wait for one window of `config.timeout`, then fail with `blocked` | +| `dlq-overflow` | A destination's dead-letter queue was full and evicted its oldest entry | n/a | Raise `dlqMax` or fix the failing destination; counted in `status.dropped` | +| `on-overflow` | A destination's queue of waiting `on()` deliveries was full and dropped the oldest | n/a | Raise `queueMax`; counted in `status.dropped` | +| `decode` | A store holds a value that cannot be decoded | no | Repair or delete the stored value; the read counts as a miss for that event | | `invalid` | An event was rejected as invalid input (a `WARN`) | no | Fix the producer | A step's own `StepError.code` wins over these. A code outside this table with @@ -237,10 +240,9 @@ platforms such as Cloud Run read as structured logs. language="json" /> -A flow's own `logger` level and format apply under `runneros` as well. `--verbose` -still forces `DEBUG`. A web flow has no output format, so `walkeros validate` +Under `runneros`, a flow's own `logger` level (`DEBUG` included) and format apply to the collector's lines without `--verbose`; the runner's own `DEBUG` lines still need `--verbose`. A custom `handler` in the flow receives an `originalHandler` that does nothing under `runneros`, because the host prints every line once, masked and in the flow's format. A web flow has no output format, so `walkeros validate` rejects `logger.format` there (`WEB_LOGGER_FORMAT`). Secrets are masked in both -formats, and a JSON line is capped at 32 KiB. +formats, and a JSON line is capped at 32 KiB: a longer line truncates its context and, if still too long, its message (`messageTruncated: true`). ## Scoped loggers diff --git a/website/docs/destinations/create-your-own.mdx b/website/docs/destinations/create-your-own.mdx index 829d25db5..b1a9bdc6d 100644 --- a/website/docs/destinations/create-your-own.mdx +++ b/website/docs/destinations/create-your-own.mdx @@ -84,20 +84,20 @@ export const destinationWebhook: Destination.Instance = { if (!settings?.url) throw new StepError('Webhook needs settings.url', { code: 'config' }); - // A rule with silent: true sends nothing - if (rule?.silent) return; - - // 4. Send the event and await it; fetch resolves on an error status - const response = await fetch(settings.url, { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify(event), - }); - if (!response.ok) - throw new StepError(\`HTTP \${response.status}\`, { - code: 'transport', - status: response.status, + // A rule with silent: true sends nothing; the push still returns normally + if (!rule?.silent) { + // 4. Send the event and await it; fetch resolves on an error status + const response = await fetch(settings.url, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(event), }); + if (!response.ok) + throw new StepError(\`HTTP \${response.status}\`, { + code: 'transport', + status: response.status, + }); + } }, };`} language="typescript" @@ -142,8 +142,11 @@ with the `timeout` code. ### Rule options every destination honours A destination honours `rule.silent`: a rule with `silent: true` sends no event -of its own and only runs the side effects the rule maps, such as an identify. A -silent rule that has nothing to send returns normally. +of its own and only runs the side effects the rule maps, such as an identify. +Skip the send with a condition rather than an early `return`, as in the example +above. A silent rule whose side effects resolve to nothing sends nothing, counts +as delivered and writes one DEBUG line that names the call and the mapping key, +never the value. ### Vendor scripts diff --git a/website/docs/destinations/index.mdx b/website/docs/destinations/index.mdx index b078baf94..89a73cfff 100644 --- a/website/docs/destinations/index.mdx +++ b/website/docs/destinations/index.mdx @@ -60,7 +60,7 @@ for the codes. Two things are not failures. Unmet consent only waits (or, on a server, skips) the event and is a `DEBUG` line. A rule with `ignore: true` or `silent: true` -is a decision, not an error, and every destination honours both. Events a +is a decision, not an error, and every destination honours both. A silent rule whose side-effect mapping (identify, group and the like) resolves to nothing sends nothing, counts as delivered and writes one DEBUG line that names the call and the mapping key, never the value. Events a destination batches answer `queued` until the flush. ## Analytics tools @@ -121,6 +121,14 @@ destination batches answer `queued` until the flush. | **Redis Streams** | - | Yes | | **SQLite** | - | Yes | +## Server SDK destinations + +The server destinations for Segment, RudderStack, Amplitude and PostHog batch by default (15, 20, 200 and 20 events, each with a 10 second wait) and send a batch through `pushBatch`. Each event keeps its own outcome, so a failed delivery fails that event and lands in the dead-letter queue. `config.batch: { size: 1 }` sends each event at once; there is no JSON switch that turns the default batch off. Options are in [Batch scheduling](/docs/mapping/rule#batch-scheduling). + +- **Acknowledgement:** a batched event answers `queued` with the reason `batch`, so a server source acknowledges the producer when the event is queued. A failed flush is reported (one ERROR line with the status, the failed counter, the dead-letter queue) but the message is not redelivered. +- **Retry budget:** the SDK retries inside the destination's `config.timeout`: a quarter of it per attempt, at most two retries. A value set in the settings wins. RudderStack has a `timeout` setting. Amplitude's requests time out through walkerOS's own transport, and a vendor that never answers fails after the budget. +- **Deduplication:** each destination passes `event.id` as the vendor's dedup id (`messageId` for Segment and RudderStack, `insert_id` for Amplitude, a UUID derived from `event.id` for PostHog; other call types use `-`). Delivery is at least once, and the vendor drops duplicates. + ## Create your own Build a custom destination for any endpoint. See diff --git a/website/docs/destinations/server/amplitude.mdx b/website/docs/destinations/server/amplitude.mdx index 9ee17cac3..260b0ef40 100644 --- a/website/docs/destinations/server/amplitude.mdx +++ b/website/docs/destinations/server/amplitude.mdx @@ -82,17 +82,8 @@ Add to your `flow.json` destinations: ## Batching and delivery -This destination batches by default: events are collected and sent together once -`config.batch` reaches 200 events or 10 seconds, whichever comes -first. Each event keeps its own outcome, so a failed delivery fails that event -and lands in the dead-letter queue instead of counting as delivered. Set -`config.batch: { size: 1 }` to send each event at once; there is no JSON switch -that turns the default batch off. - -A batched event answers `queued` with the reason `batch`. A server source -acknowledges the producer when the event is queued, and a failed flush is -reported (one ERROR line with the status, the failed counter, the dead-letter -queue) but not redelivered. Options are in -[Batch scheduling](/docs/mapping/rule#batch-scheduling). +This destination batches by default: `config.batch` sends after 200 events or 10 seconds, and each event keeps its own outcome. Set `config.batch: { size: 1 }` to send each event at once. How batching, retries, deduplication and acknowledgement work for the server SDK destinations is described in [Server SDK destinations](/docs/destinations#server-sdk-destinations). A delivery that Amplitude does not answer with `2xx` fails with its status. + +A group name list that holds a non-string is skipped with a DEBUG line, not sent. diff --git a/website/docs/destinations/server/gcp.mdx b/website/docs/destinations/server/gcp.mdx index af16c07b8..80a8c81ad 100644 --- a/website/docs/destinations/server/gcp.mdx +++ b/website/docs/destinations/server/gcp.mdx @@ -244,6 +244,7 @@ ingestion. This replaces the legacy `tabledata.insertAll` path. - **Cost**: $25/TB after the 2 TiB/month free tier (vs ~$50/TB for the legacy path). Most low-volume deployments fit entirely in the free tier. +- **Rejected appends**: an append the server rejects (quota, permission, a dropped table) fails the push. Pub/Sub retries only the error codes its SDK retries. - **First use**: the writer opens at the first push, not at startup; a missing dataset or table fails that push with the setup hint. BigQuery and Pub/Sub failures are reported once by the collector, and the messages name codes, not row data. - **Batching**: `pushBatch` is implemented. Set `config.batch` on the destination (a bare number is the debounce `wait` in ms) to flush all events diff --git a/website/docs/destinations/server/kafka.mdx b/website/docs/destinations/server/kafka.mdx index 3890c51ca..ea17b4f49 100644 --- a/website/docs/destinations/server/kafka.mdx +++ b/website/docs/destinations/server/kafka.mdx @@ -86,7 +86,7 @@ Add to your `flow.json` destinations: -The destination creates a single long-lived `kafkajs` producer and connects it at the first push, so a flow starts without a reachable broker. A failed send fails the push (dead-letter queue, one ERROR line) instead of being logged and counted delivered, and an unknown topic is a `config` error with the setup hint. On flow hot-swap or server shutdown, `destroy()` calls `producer.disconnect()` to flush in-flight messages and close TCP connections. +The destination creates a single long-lived `kafkajs` producer and connects it at the first push, so a flow starts without a reachable broker. A failed send fails the push (dead-letter queue, one ERROR line) instead of being logged and counted delivered, and it is retried only when kafkajs marks the error as retriable, and an unknown topic is a `config` error with the setup hint. On flow hot-swap or server shutdown, `destroy()` calls `producer.disconnect()` to flush in-flight messages and close TCP connections. ## Message format diff --git a/website/docs/destinations/server/klaviyo.mdx b/website/docs/destinations/server/klaviyo.mdx index 825a89407..9e3b8adc5 100644 --- a/website/docs/destinations/server/klaviyo.mdx +++ b/website/docs/destinations/server/klaviyo.mdx @@ -79,7 +79,7 @@ Add to your `flow.json` destinations: -Identity is resolved automatically from each event: `email` defaults to `user.email` and `externalId` defaults to `user.id`. Klaviyo requires at least one identifier (`email`, `phoneNumber`, or `externalId`) per event. Events without any identifier are skipped with a warning. +Identity is resolved automatically from each event: `email` defaults to `user.email` and `externalId` defaults to `user.id`. Klaviyo requires at least one identifier (`email`, `phoneNumber`, or `externalId`) per event. An event without any identifier fails with `missing-input`. ## Revenue tracking diff --git a/website/docs/destinations/server/linkedin.mdx b/website/docs/destinations/server/linkedin.mdx index f59fdd6a6..ff0e82a70 100644 --- a/website/docs/destinations/server/linkedin.mdx +++ b/website/docs/destinations/server/linkedin.mdx @@ -84,7 +84,7 @@ Add to your `flow.json` destinations: ## User identification -The destination builds a `userIds` array from the event data. At least one identifier is required or the event is silently skipped. +The destination builds a `userIds` array from the event data. At least one identifier is required; an event without one fails with `missing-input`. **SHA256_EMAIL**: resolved from `event.user.email` or a `user_data.email` mapping. The email is normalized (trimmed, lowercased) and SHA-256 hashed before sending. Set `doNotHash: ['email']` if you are pre-hashing upstream. diff --git a/website/docs/destinations/server/piwikpro.mdx b/website/docs/destinations/server/piwikpro.mdx index 8355db8ba..214e345b7 100644 --- a/website/docs/destinations/server/piwikpro.mdx +++ b/website/docs/destinations/server/piwikpro.mdx @@ -122,7 +122,7 @@ Without it, Piwik PRO receives no client IP, user agent or language, and locates | `ip` | `["ingest.ip", "event.user.ip"]` | `cip`, the visitor's IP address. Kept on anonymous hits, for country-level location. | | `userAgent` | `["ingest.userAgent", "event.user.userAgent"]` | `ua` | | `language` | `["ingest.language", "event.user.language"]` | `lang` | -| `pageUrl` | `"event.source.url"` | `url`. The Tracking API requires it, so an event without a page URL is skipped. | +| `pageUrl` | `"event.source.url"` | `url`. The Tracking API requires it, so an event without a page URL fails with `missing-input`. | | `referrer` | `"event.source.referrer"` | `urlref` | | `visitorId` | `"event.user.device"` | `_id` on identified hits. A 16 character lowercase hex value passes through. Anything else is hashed to 16 hex characters, so walkerOS device ids are always hashed. | | `userId` | `"event.user.id"` | `uid` on identified hits. | @@ -137,7 +137,7 @@ For cookieless tracking, pair `visitorId` with the [fingerprint transformer](/do `rule.name` selects a Piwik PRO JavaScript method, and `data` resolves to its arguments in order. Use `set` to pass several arguments; any other value is a single argument. A `page view` without `rule.name` becomes `trackPageView`, with `rule.data` as its arguments, or else `data.title` as the title. -The server builds hits for 13 methods. A required argument is marked with `*`. When it is missing, the event is invalid and skipped. Every hit also carries the common parameters, the custom dimensions and the request context from the settings above. +The server builds hits for 13 methods. A required argument is marked with `*`. When it is missing, the event is invalid and fails with `missing-input`. Every hit also carries the common parameters, the custom dimensions and the request context from the settings above. | Method (JavaScript signature) | What the server sends | |---|---| @@ -228,7 +228,7 @@ A rule without `name` on an event that is not a page view sends only the goal wh language="json" /> -When the goal value resolves to something other than a number or a numeric string, the goal hit is invalid and the whole event is skipped. The method hit is not sent either. The same applies to any invalid hit: an event is sent completely or not at all. A goal value that resolves to nothing only leaves out `revenue`. +When the goal value resolves to something other than a number or a numeric string, the goal hit is invalid and the whole event fails with `missing-input`. The method hit is not sent either. The same applies to any invalid hit: an event is sent completely or not at all. A goal value that resolves to nothing only leaves out `revenue`. ## Custom dimensions @@ -293,7 +293,7 @@ The body of a bulk request holds one request string per hit. For a page view and language="json" /> -A goal adds its hit to the same list. Skipped events are left out of the request and count as delivered. If every event of a batch is skipped, no request is sent. +A goal adds its hit to the same list. Invalid events fail individually and are left out of the request; a `silent` rule without a goal is left out and counts as delivered. If every event of a batch is left out, no request is sent. A failed request (a 4xx or 5xx status, a timeout or a network error) throws, so the whole batch goes to the destination's dead-letter queue. A 429 status is logged as rate limiting first. The destination does not retry. @@ -315,14 +315,13 @@ The page view id links an event to the page view it happened on. By default it i ## Unmapped and invalid events -These events are skipped: +These events fail with `missing-input`: - An event that is not a page view and whose rule has neither `name` nor `goalId`. - An event whose rule names a method outside the table, such as `setUserId`. -- A `silent` rule without a goal. - An invalid event: a missing required argument, no page URL, a value that must be a number but is not (such as `e_v` or `revenue`), or more than 100 products. -Skips never throw. The destination logs a warning once per event name and reason, and repeats at debug level. To keep unmapped events out of the destination entirely, ignore everything with a wildcard and map what you need: +A `silent` rule without a goal sends nothing and counts as delivered. To keep unmapped events out of the destination without a failure, ignore everything with a wildcard and map what you need: ` already exists in the DOM, no second injection happens. diff --git a/website/docs/destinations/web/matomo.mdx b/website/docs/destinations/web/matomo.mdx index 3fbae6d68..841215224 100644 --- a/website/docs/destinations/web/matomo.mdx +++ b/website/docs/destinations/web/matomo.mdx @@ -94,7 +94,7 @@ Add to your `flow.json` destinations: A destination with `config.consent` tells Matomo about a revocation: `forgetConsentGiven` stops link tracking too, and a grant sends -`setConsentGiven`. An event no rule maps fails with `missing-input`, and an +`setConsentGiven`. An event no rule maps fails with `missing-input` (add an `ignore: true` rule for events you do not send), and an invalid `goalId` sends the hit without the goal and logs one ERROR line per cause. A rule with `silent: true` sends only its goal. @@ -103,12 +103,12 @@ cause. A rule with `silent: true` sends only its goal. An event without `rule.name` that is not a page view and sets no tracking flag (`siteSearch`, `contentImpression` or `contentInteraction`) sends only its goal when the rule has `settings.goalId`, so a goal-only rule needs no name. Without -a goal, such an event is skipped: the destination logs a warning once per event -name, then repeats at debug level. +a goal, such an event fails with `missing-input`; add an `ignore: true` rule for +events you do not send. A `goalId` must be a Matomo goal ID, a positive integer like `1` or `"1"`. The -destination skips any other goal with a warning and still sends the event's own -hit, if it has one. +destination sends the event's own hit without the goal and logs one ERROR line +per cause. ## Ecommerce diff --git a/website/docs/destinations/web/optimizely.mdx b/website/docs/destinations/web/optimizely.mdx index 679218917..7ae5b29a9 100644 --- a/website/docs/destinations/web/optimizely.mdx +++ b/website/docs/destinations/web/optimizely.mdx @@ -103,10 +103,12 @@ Two layers protect event delivery: 1. **`config.consent`**: walkerOS gates delivery. Events are queued until required consent keys resolve to `true`. -2. **`on('consent')`**: the destination closes the Optimizely client - (flushing queued events and stopping datafile polling) when any - required key flips to `false`. On re-grant, the next push re-initializes - the client. +2. **`on('consent')`**: the destination decides on the whole consent state + against its own `config.consent` and acts only when a change touches a + required key. It keeps its client across a revoke and a grant. + +An event from a visitor without the configured `userId` fails with +`missing-input`. `); the event is still delivered. A rule-level `condition` that throws is different: it fails the event (see [Rule](./rule.mdx)). diff --git a/website/docs/sources/server/aws.mdx b/website/docs/sources/server/aws.mdx index eecfc5ff4..5267813e0 100644 --- a/website/docs/sources/server/aws.mdx +++ b/website/docs/sources/server/aws.mdx @@ -129,15 +129,7 @@ rides through instead of being dropped. | 400 | Rejected client input: the pipeline declared the event invalid. The body echoes the validation message, for example `Event name is required` | | 500 | The pipeline failed to process a valid event for a reason a resend could fix (the body is `Event was not processed`), or an unexpected server fault (`Internal server error`) | -A failed event is acknowledged or retried by one rule on every source, from the -push result: a failure a resend could fix (`retryable`, such as a timeout or an -unavailable vendor) answers 500, or 207 for the failed events of a batch, with -the fixed text `Event was not processed`. A failure a resend cannot fix (a -missing setting, a `4xx` from the vendor, a destination that is not ready) is -acknowledged with 200: it is in the dead-letter queue and in the log, and -sending the same event again would fail the same way. The body never carries a -destination id or the text of a step's error; the collector's ERROR line does. -An unexpected throw answers 500 `Internal server error`. +A failed event is acknowledged or retried by one rule on every server source: see [Producer acknowledgement](/docs/sources/server#producer-acknowledgement). Invalid input is counted on `collector.status.sources..rejected` rather than inflating `status.failed`, and is logged at warn with the reason instead of as an diff --git a/website/docs/sources/server/express.mdx b/website/docs/sources/server/express.mdx index 38d6ac18c..ab6295194 100644 --- a/website/docs/sources/server/express.mdx +++ b/website/docs/sources/server/express.mdx @@ -202,15 +202,9 @@ push. | 500 | On synchronous POST: the pipeline failed to process a valid event for a reason a resend could fix; the body is `Event was not processed`. Otherwise: unexpected server fault (`Internal server error`) | | 207 | On synchronous batch POST: some events failed and a resend could fix it; the body lists the failed indices | -A failed event is acknowledged or retried by one rule on every source, from the -push result: a failure a resend could fix (`retryable`, such as a timeout or an -unavailable vendor) answers 500, or 207 for the failed events of a batch, with -the fixed text `Event was not processed`. A failure a resend cannot fix (a -missing setting, a `4xx` from the vendor, a destination that is not ready) is -acknowledged with 200: it is in the dead-letter queue and in the log, and -sending the same event again would fail the same way. The body never carries a -destination id or the text of a step's error; the collector's ERROR line does. -An unexpected throw answers 500 `Internal server error`. +A failed event is acknowledged or retried by one rule on every server source: see [Producer acknowledgement](/docs/sources/server#producer-acknowledgement). + +In respond-first mode a request whose event later fails is already answered; the source writes a DEBUG line and the collector's ERROR line is the signal. GET pixel requests answer 200 with the GIF in every mode, unless a step responded first, because an image endpoint cannot carry a useful 400. Rejected pixel input is visible through the diff --git a/website/docs/sources/server/fetch.mdx b/website/docs/sources/server/fetch.mdx index 0a7c28c8b..e4c33aae2 100644 --- a/website/docs/sources/server/fetch.mdx +++ b/website/docs/sources/server/fetch.mdx @@ -108,15 +108,9 @@ Server sources require platform-specific handlers. For containerized deployments | 500 | The pipeline failed to process a valid event for a reason a resend could fix (the body is `Event was not processed`), or an unexpected server fault (`Internal server error`) | | 207 | Batch request where some events succeeded and others failed in a way a resend could fix. The body carries per-index errors | -A failed event is acknowledged or retried by one rule on every source, from the -push result: a failure a resend could fix (`retryable`, such as a timeout or an -unavailable vendor) answers 500, or 207 for the failed events of a batch, with -the fixed text `Event was not processed`. A failure a resend cannot fix (a -missing setting, a `4xx` from the vendor, a destination that is not ready) is -acknowledged with 200: it is in the dead-letter queue and in the log, and -sending the same event again would fail the same way. The body never carries a -destination id or the text of a step's error; the collector's ERROR line does. -An unexpected throw answers 500 `Internal server error`. +A failed event is acknowledged or retried by one rule on every server source: see [Producer acknowledgement](/docs/sources/server#producer-acknowledgement). + +A per-event "not processed" in a batch logs at DEBUG, because the collector's ERROR line is the signal. A push that throws inside a batch logs an ERROR and answers `Internal server error`. Invalid input is counted on `collector.status.sources..rejected` rather than inflating `status.failed`, and is logged at warn with the reason instead of as an diff --git a/website/docs/sources/server/gcp.mdx b/website/docs/sources/server/gcp.mdx index d9b90b7e1..c53c5ab12 100644 --- a/website/docs/sources/server/gcp.mdx +++ b/website/docs/sources/server/gcp.mdx @@ -136,15 +136,7 @@ Send several events in one request with the shared envelope. See | 400 | Rejected client input: the pipeline declared the event invalid. The body echoes the validation message, for example `Event name is required` | | 500 | The pipeline failed to process a valid event for a reason a resend could fix (the body is `Event was not processed`), or an unexpected server fault (`Internal server error`) | -A failed event is acknowledged or retried by one rule on every source, from the -push result: a failure a resend could fix (`retryable`, such as a timeout or an -unavailable vendor) answers 500, or 207 for the failed events of a batch, with -the fixed text `Event was not processed`. A failure a resend cannot fix (a -missing setting, a `4xx` from the vendor, a destination that is not ready) is -acknowledged with 200: it is in the dead-letter queue and in the log, and -sending the same event again would fail the same way. The body never carries a -destination id or the text of a step's error; the collector's ERROR line does. -An unexpected throw answers 500 `Internal server error`. +A failed event is acknowledged or retried by one rule on every server source: see [Producer acknowledgement](/docs/sources/server#producer-acknowledgement). Invalid input is counted on `collector.status.sources..rejected` rather than inflating `status.failed`, and is logged at warn with the reason instead of as an diff --git a/website/docs/stores/index.mdx b/website/docs/stores/index.mdx index eab553960..5de2f3e05 100644 --- a/website/docs/stores/index.mdx +++ b/website/docs/stores/index.mdx @@ -189,7 +189,7 @@ const value = await collector.stores.assets.get('key');`} ## Failed reads -A read, write or delete that fails (a 403, a 5xx, a network error) throws instead of looking like a missing key, and a stored value that cannot be decoded throws too. A key that is not stored is a normal miss and returns `undefined`. The collector treats a failed read as a miss for that event and writes one ERROR line per cause, so a failing store is visible without stopping the flow. +A read, write or delete that fails (a 403, a 5xx, a network error) throws instead of looking like a missing key, and a stored value that cannot be decoded throws too. A key that is not stored is a normal miss and returns `undefined`. A read through `$store` or a cache treats a failed read as a miss for that event and writes one ERROR line per cause, so a failing store is visible without stopping the flow. The file transformer does not catch a failed read: its request fails. A path that is a directory is a miss in the filesystem store. ## Setup lifecycle diff --git a/website/docs/transformers/fingerprint.mdx b/website/docs/transformers/fingerprint.mdx index f272adfba..6ce68fa89 100644 --- a/website/docs/transformers/fingerprint.mdx +++ b/website/docs/transformers/fingerprint.mdx @@ -93,7 +93,7 @@ The source must extract `ip` and `userAgent` into `ingest`: Behind a load balancer, `ip` is the balancer's address; extract the client from `headers.x-forwarded-for` instead. An IPv4 client on a dual-stack server (`::ffff:203.0.113.7`) is treated as its IPv4 address. -Set an input to another [mapping value](/docs/mapping/value) to read it from elsewhere, or to `false` to leave it out. When one input resolves empty, the transformer hashes without it; when all of them do, the event fails with `missing-input`. +Set an input to another [mapping value](/docs/mapping/value) to read it from elsewhere, or to `false` to leave it out. When one input resolves empty, the transformer hashes without it; when the IP, the user agent and every field resolve empty (the site does not count), the event fails with `missing-input`. ## Salt diff --git a/website/docs/transformers/ga4.mdx b/website/docs/transformers/ga4.mdx index 8b4017252..a459916ff 100644 --- a/website/docs/transformers/ga4.mdx +++ b/website/docs/transformers/ga4.mdx @@ -437,7 +437,7 @@ The string is compiled to a `RegExp` at init time. ## Batch size limit -Every decoded event continues through the pipeline as its own push, and a source's batch limit (such as the express `maxBatchSize`) only covers JSON batches, not a raw `text/plain` body. The transformer therefore caps the events in one request with `settings.maxEvents` (default `100`). It counts the non-empty lines of the POST body before decoding, and a request with more lines than the cap is dropped whole: no event from it is decoded, and one `warn` log line records the line count and the cap. The source still answers the client with success, so that log line is the only sign of the drop. Raise the cap only if your own server-to-server calls send larger batches: +Every decoded event continues through the pipeline as its own push, and a source's batch limit (such as the express `maxBatchSize`) only covers JSON batches, not a raw `text/plain` body. The transformer therefore caps the events in one request with `settings.maxEvents` (default `100`). It counts the non-empty lines of the POST body before decoding, and a request with more lines than the cap is dropped whole: no event from it is decoded, and the collector writes one ERROR line with the line count and the cap. The source still answers the client with success, so that log line is the only sign of the drop. Raise the cap only if your own server-to-server calls send larger batches: ```json "settings": { diff --git a/website/docs/transformers/index.mdx b/website/docs/transformers/index.mdx index 8ec34c735..adf508367 100644 --- a/website/docs/transformers/index.mdx +++ b/website/docs/transformers/index.mdx @@ -102,7 +102,14 @@ transformer steps, but the semantic differs by position. On a destination, `mapping` shapes the vendor payload. On a transformer step, it mutates the event itself. Only `policy` and a rule's `condition`, `policy`, `name` and `ignore` apply there; every other field does nothing and is named in a -one-time warning at init. +one ERROR line at registration, and `walkeros validate` rejects them +(`MAPPING_NO_OP`). +::: + +:::note Timeout +`config.timeout` (milliseconds, default 10000) bounds a transformer's `init` and +each `push`. A push that does not settle fails the event with the code +`timeout`. ::: :::note Closed schema From acf757def9b5d5beac7a47baa3da1a2bade1f1e4 Mon Sep 17 00:00:00 2001 From: alexanderkirtzel Date: Wed, 21 Oct 2026 09:35:33 +0000 Subject: [PATCH 35/41] docs minors --- .changeset/server-destinations-declared-skips.md | 13 +++++++++++++ .changeset/server-destinations-retry-dedup.md | 4 ++-- .changeset/web-destinations-declared-skips.md | 8 +++----- packages/web/destinations/piwikpro/src/hints.ts | 2 +- website/docs/collector/logger.mdx | 2 +- website/docs/destinations/index.mdx | 2 +- 6 files changed, 21 insertions(+), 10 deletions(-) create mode 100644 .changeset/server-destinations-declared-skips.md diff --git a/.changeset/server-destinations-declared-skips.md b/.changeset/server-destinations-declared-skips.md new file mode 100644 index 000000000..5a56c8164 --- /dev/null +++ b/.changeset/server-destinations-declared-skips.md @@ -0,0 +1,13 @@ +--- +'@walkeros/server-destination-segment': patch +'@walkeros/server-destination-rudderstack': patch +'@walkeros/server-destination-amplitude': patch +'@walkeros/server-destination-posthog': patch +'@walkeros/server-destination-hubspot': patch +'@walkeros/server-destination-mixpanel': patch +--- + +A silent rule whose identify, group or similar mapping resolves to nothing sends +nothing, counts as delivered and writes one DEBUG line naming the call and +mapping key, never the value. HubSpot's warning and Mixpanel's error for this +case are now DEBUG, because nothing was lost. diff --git a/.changeset/server-destinations-retry-dedup.md b/.changeset/server-destinations-retry-dedup.md index e0f753261..4fa47213f 100644 --- a/.changeset/server-destinations-retry-dedup.md +++ b/.changeset/server-destinations-retry-dedup.md @@ -7,5 +7,5 @@ Retries stay inside the destination's `config.timeout`: a quarter of it per attempt, at most two retries, and a value set in the settings wins. RudderStack -gains a `timeout` setting. Each event carries `event.id` as the vendor's dedup -id, so at-least-once delivery drops duplicates. +gains a `timeout` setting. Events carry `event.id` as the vendor's dedup id +(PostHog on `capture` only), so at-least-once delivery drops duplicates. diff --git a/.changeset/web-destinations-declared-skips.md b/.changeset/web-destinations-declared-skips.md index 0170720a9..e9d65a4d0 100644 --- a/.changeset/web-destinations-declared-skips.md +++ b/.changeset/web-destinations-declared-skips.md @@ -1,15 +1,13 @@ --- '@walkeros/web-destination-amplitude': patch '@walkeros/web-destination-segment': patch +'@walkeros/web-destination-clarity': patch +'@walkeros/web-destination-fullstory': patch +'@walkeros/web-destination-hotjar': patch '@walkeros/web-destination-mixpanel': patch '@walkeros/web-destination-posthog': patch -'@walkeros/web-destination-fullstory': patch -'@walkeros/web-destination-heap': patch -'@walkeros/web-destination-piwikpro': patch -'@walkeros/web-destination-optimizely': patch '@walkeros/web-destination-pinterest': patch '@walkeros/web-destination-tiktok': patch -'@walkeros/web-destination-snowplow': patch --- A silent rule whose identify, group or similar mapping resolves to nothing sends diff --git a/packages/web/destinations/piwikpro/src/hints.ts b/packages/web/destinations/piwikpro/src/hints.ts index 3943cd942..52a4c222f 100644 --- a/packages/web/destinations/piwikpro/src/hints.ts +++ b/packages/web/destinations/piwikpro/src/hints.ts @@ -29,7 +29,7 @@ export const hints: Hint.Hints = { ], }, 'unmapped-events': { - text: 'An event without a rule name that is not a page view sends only its goal when the rule has settings.goalId, and is otherwise skipped with a warning (once per event name, then at debug level). A named rule with silent: true also sends only its goal. To keep other events out of the destination entirely, ignore everything with a wildcard and keep the page view explicitly.', + text: 'An event without a rule name that is not a page view sends only its goal when the rule has settings.goalId, and otherwise fails with missing-input; add an ignore: true rule for events you do not send. A named rule with silent: true also sends only its goal. To keep other events out of the destination entirely, ignore everything with a wildcard and keep the page view explicitly.', code: [ { lang: 'json', diff --git a/website/docs/collector/logger.mdx b/website/docs/collector/logger.mdx index 5d2e52152..66f41348b 100644 --- a/website/docs/collector/logger.mdx +++ b/website/docs/collector/logger.mdx @@ -218,7 +218,7 @@ see each line. | `beacon-rejected` | The browser refused a `sendBeacon` | only up to 64 KiB | Send smaller batches | | `consent-not-applied` | A destination's `on('consent')` handler failed | n/a | Fix the handler; the destination's events wait for one window of `config.timeout`, then fail with `blocked` | | `dlq-overflow` | A destination's dead-letter queue was full and evicted its oldest entry | n/a | Raise `dlqMax` or fix the failing destination; counted in `status.dropped` | -| `on-overflow` | A destination's queue of waiting `on()` deliveries was full and dropped the oldest | n/a | Raise `queueMax`; counted in `status.dropped` | +| `on-overflow` | A destination's queue of waiting `on()` deliveries was full and dropped the oldest | n/a | Raise the destination's own `config.queueMax`; counted in `status.dropped` | | `decode` | A store holds a value that cannot be decoded | no | Repair or delete the stored value; the read counts as a miss for that event | | `invalid` | An event was rejected as invalid input (a `WARN`) | no | Fix the producer | diff --git a/website/docs/destinations/index.mdx b/website/docs/destinations/index.mdx index 89a73cfff..04801243a 100644 --- a/website/docs/destinations/index.mdx +++ b/website/docs/destinations/index.mdx @@ -127,7 +127,7 @@ The server destinations for Segment, RudderStack, Amplitude and PostHog batch by - **Acknowledgement:** a batched event answers `queued` with the reason `batch`, so a server source acknowledges the producer when the event is queued. A failed flush is reported (one ERROR line with the status, the failed counter, the dead-letter queue) but the message is not redelivered. - **Retry budget:** the SDK retries inside the destination's `config.timeout`: a quarter of it per attempt, at most two retries. A value set in the settings wins. RudderStack has a `timeout` setting. Amplitude's requests time out through walkerOS's own transport, and a vendor that never answers fails after the budget. -- **Deduplication:** each destination passes `event.id` as the vendor's dedup id (`messageId` for Segment and RudderStack, `insert_id` for Amplitude, a UUID derived from `event.id` for PostHog; other call types use `-`). Delivery is at least once, and the vendor drops duplicates. +- **Deduplication:** each destination passes `event.id` as the vendor's dedup id (`messageId` for Segment and RudderStack, `insert_id` for Amplitude, a UUID derived from `event.id` on PostHog's `capture` only; other call types of Segment, RudderStack and Amplitude use `-`). Delivery is at least once, and the vendor drops duplicates. ## Create your own From 2634b39b922cf0719d70ad1b6f738c3ed9b5fc1c Mon Sep 17 00:00:00 2001 From: alexanderkirtzel Date: Wed, 21 Oct 2026 09:45:41 +0000 Subject: [PATCH 36/41] amplitude lazy plugins --- .../web/destinations/amplitude/src/index.ts | 40 +++++++++++++------ .../web/destinations/amplitude/tsconfig.json | 3 +- 2 files changed, 29 insertions(+), 14 deletions(-) diff --git a/packages/web/destinations/amplitude/src/index.ts b/packages/web/destinations/amplitude/src/index.ts index 5e736cc09..5e5db282a 100644 --- a/packages/web/destinations/amplitude/src/index.ts +++ b/packages/web/destinations/amplitude/src/index.ts @@ -11,9 +11,6 @@ import { stepCall, } from '@walkeros/core'; import * as amplitudeSDK from '@amplitude/unified'; -import { sessionReplayPlugin } from '@amplitude/plugin-session-replay-browser'; -import { experimentPlugin } from '@amplitude/plugin-experiment-browser'; -import { plugin as engagementPlugin } from '@amplitude/engagement-browser'; import type { AmplitudePlugin, AmplitudeSDK, @@ -302,35 +299,52 @@ async function start( // Session Replay, Experiment and Guides & Surveys are opt-in, each on its // own: only the ones the settings enable start, on the analytics server - // zone and instance. + // zone and instance. Each package loads only then, with a dynamic import: + // a static one of @amplitude/engagement-browser (an ESM entry without + // "type": "module") can come in as CommonJS without its named exports + // under Node loaders such as tsx, and failed every import of this module. const { serverZone, instanceName } = analyticsOptions; const shared = { ...(serverZone ? { serverZone } : {}), ...(instanceName ? { instanceName } : {}), }; - const plugins: Array<[string, AmplitudePlugin]> = []; + const plugins: Array<[string, () => Promise]> = []; if (sessionReplay) plugins.push([ 'Session Replay', - sessionReplayPlugin({ ...sessionReplay, ...shared }), + async () => { + const { sessionReplayPlugin } = + await import('@amplitude/plugin-session-replay-browser'); + return sessionReplayPlugin({ ...sessionReplay, ...shared }); + }, ]); if (experiment?.deploymentKey) plugins.push([ 'Experiment', - experimentPlugin({ ...experiment, ...shared }), + async () => { + const { experimentPlugin } = + await import('@amplitude/plugin-experiment-browser'); + return experimentPlugin({ ...experiment, ...shared }); + }, ]); if (engagement) plugins.push([ 'Guides & Surveys', - engagementPlugin({ - ...(engagement === true ? {} : engagement), - ...shared, - }), + async () => { + const { plugin } = await import('@amplitude/engagement-browser'); + return plugin({ + ...(engagement === true ? {} : engagement), + ...shared, + }); + }, ]); - for (const [name, plugin] of plugins) + for (const [name, create] of plugins) await stepCall( - () => settled(() => amp.add(plugin)), + async () => { + const plugin = await create(); + await settled(() => amp.add(plugin)); + }, 'vendor-not-ready', `Amplitude ${name} failed to start`, ); diff --git a/packages/web/destinations/amplitude/tsconfig.json b/packages/web/destinations/amplitude/tsconfig.json index e9f0a4037..5e5bc0e71 100644 --- a/packages/web/destinations/amplitude/tsconfig.json +++ b/packages/web/destinations/amplitude/tsconfig.json @@ -1,7 +1,8 @@ { "extends": "@walkeros/config/tsconfig/web.json", "compilerOptions": { - "rootDir": "src" + "rootDir": "src", + "module": "es2020" }, "include": ["src/**/*"], "exclude": ["node_modules", "dist"] From 5fad2d9c07780eb86516e4c64542fe96d4522f5b Mon Sep 17 00:00:00 2001 From: alexanderkirtzel Date: Wed, 21 Oct 2026 10:13:34 +0000 Subject: [PATCH 37/41] vendor env document --- .../src/dev/__tests__/conformance.test.ts | 10 ++++ .../__tests__/fixture-package/src/vendor.ts | 28 ++++++---- packages/collector/src/dev/conformance.ts | 4 +- packages/web/core/package.json | 5 ++ .../web/core/src/__tests__/loadVendor.test.ts | 37 +++++++++++++ packages/web/core/src/dev.ts | 2 + packages/web/core/src/loadVendor.ts | 46 +++++++++++----- packages/web/core/src/mockScriptDocument.ts | 52 +++++++++++++++++++ packages/web/core/tsup.config.ts | 9 +++- .../web/destinations/clarity/src/index.ts | 21 +++++--- .../web/destinations/gtag/src/ads/config.ts | 2 +- .../web/destinations/gtag/src/ga4/config.ts | 2 +- .../web/destinations/gtag/src/gtm/config.ts | 9 ++-- packages/web/destinations/gtag/src/index.ts | 10 ++-- .../web/destinations/gtag/src/shared/gtag.ts | 36 +++++++++---- packages/web/destinations/heap/src/index.ts | 5 +- .../web/destinations/linkedin/src/index.ts | 2 +- packages/web/destinations/matomo/src/index.ts | 2 +- packages/web/destinations/meta/src/index.ts | 2 +- packages/web/destinations/piano/src/index.ts | 4 +- .../web/destinations/pinterest/src/index.ts | 2 +- .../web/destinations/piwikpro/src/index.ts | 2 +- .../plausible/src/examples/env.ts | 23 ++------ .../web/destinations/plausible/src/index.ts | 1 + .../destinations/snowplow/src/examples/env.ts | 21 ++------ .../web/destinations/snowplow/src/index.ts | 2 +- .../destinations/tiktok/src/examples/env.ts | 4 +- packages/web/destinations/tiktok/src/index.ts | 2 +- .../templates/simple/index.ts | 2 +- 29 files changed, 239 insertions(+), 108 deletions(-) create mode 100644 packages/web/core/src/dev.ts create mode 100644 packages/web/core/src/mockScriptDocument.ts diff --git a/packages/collector/src/dev/__tests__/conformance.test.ts b/packages/collector/src/dev/__tests__/conformance.test.ts index 0c2e21c5b..a821c8abf 100644 --- a/packages/collector/src/dev/__tests__/conformance.test.ts +++ b/packages/collector/src/dev/__tests__/conformance.test.ts @@ -646,6 +646,16 @@ describe('conformance kit, vendor-global profile', () => { ).resolves.toBeUndefined(); }); + it('reaches the script error when the examples env carries its own document', async () => { + const own = document.implementation.createHTMLDocument('env'); + await expect( + conformance('destination', pageScriptVendor, { + ...scriptOptions, + env: { ...scriptOptions.env, push: { vendor, document: own } }, + }), + ).resolves.toBeUndefined(); + }); + it('fails item 2 when the push ignores the script error', async () => { await expect( conformance('destination', pageScriptIgnoring, scriptOptions), diff --git a/packages/collector/src/dev/__tests__/fixture-package/src/vendor.ts b/packages/collector/src/dev/__tests__/fixture-package/src/vendor.ts index ec99a1212..a125abf5d 100644 --- a/packages/collector/src/dev/__tests__/fixture-package/src/vendor.ts +++ b/packages/collector/src/dev/__tests__/fixture-package/src/vendor.ts @@ -8,7 +8,11 @@ export interface Vendor { consent(granted: boolean): void; } -type Types = Destination.Types<{ site?: string }, unknown, { vendor?: Vendor }>; +type Types = Destination.Types< + { site?: string }, + unknown, + { vendor?: Vendor; document?: Document } +>; function isVendor(value: unknown): value is Vendor { return ( @@ -158,30 +162,32 @@ const failedScripts = new WeakSet(); /** * Injects its marked script once per page, as loadVendor does (a marked - * script with its src already on the page is the injected one), with - * `loadScript` on by default. A push calls the stub synchronously and, - * with `reports`, fails `vendor-not-loaded` once the script fired `error`. + * script with its src already on the page is the injected one), into the + * env's document where it has one, with `loadScript` on by default. A push + * calls the stub synchronously and, with `reports`, fails + * `vendor-not-loaded` once the script fired `error`. */ function pageScript(reports: boolean): Destination.Instance { const src = 'https://vendor.example/page.js'; - const scriptOnPage = () => - Array.from(document.querySelectorAll('script[data-walkeros-vendor]')).find( + const scriptOnPage = (doc: Document) => + Array.from(doc.querySelectorAll('script[data-walkeros-vendor]')).find( (script) => script.getAttribute('src') === src, ); return { ...vendorGlobal, type: reports ? 'page-script' : 'page-script-ignoring', config: { loadScript: true }, - init: ({ config }) => { - if (!config.loadScript || scriptOnPage()) return; - const script = document.createElement('script'); + init: ({ config, env }) => { + const doc = env.document ?? document; + if (!config.loadScript || scriptOnPage(doc)) return; + const script = doc.createElement('script'); script.src = src; script.setAttribute('data-walkeros-vendor', ''); script.addEventListener('error', () => failedScripts.add(script)); - document.head.appendChild(script); + doc.head.appendChild(script); }, push: async (event, context) => { - const script = scriptOnPage(); + const script = scriptOnPage(context.env.document ?? document); if (reports && script && failedScripts.has(script)) throw new StepError('Vendor script failed to load', { code: 'vendor-not-loaded', diff --git a/packages/collector/src/dev/conformance.ts b/packages/collector/src/dev/conformance.ts index 01f639479..4de7c8194 100644 --- a/packages/collector/src/dev/conformance.ts +++ b/packages/collector/src/dev/conformance.ts @@ -745,7 +745,9 @@ async function vendorFailureProblem( const mapping = mappingOf(primary.example, { event: primary.event }); const session = await openDestination(step, { settings: ctx.options.settings, - env: ctx.env, + // The page's document replaces an env one, so the script lands where + // the kit fires its `error`. + env: ctx.env && { ...ctx.env, push: { ...ctx.env.push, document } }, config: { timeout: RUN_TIMEOUT, loadScript: true, diff --git a/packages/web/core/package.json b/packages/web/core/package.json index e78dec179..33b7645c0 100644 --- a/packages/web/core/package.json +++ b/packages/web/core/package.json @@ -11,6 +11,11 @@ "types": "./dist/index.d.ts", "import": "./dist/index.mjs", "require": "./dist/index.js" + }, + "./dev": { + "types": "./dist/dev.d.ts", + "import": "./dist/dev.mjs", + "require": "./dist/dev.js" } }, "files": [ diff --git a/packages/web/core/src/__tests__/loadVendor.test.ts b/packages/web/core/src/__tests__/loadVendor.test.ts index 0aeb19b1a..2943c7552 100644 --- a/packages/web/core/src/__tests__/loadVendor.test.ts +++ b/packages/web/core/src/__tests__/loadVendor.test.ts @@ -1,3 +1,5 @@ +import { execFileSync } from 'child_process'; +import path from 'path'; import { DEFAULT_DESTINATION_TIMEOUT_MS } from '@walkeros/core'; import { loadVendor, VENDOR_SCRIPT_MARKER } from '../loadVendor'; @@ -74,6 +76,41 @@ describe('loadVendor', () => { expect(scriptsOf(src)).toHaveLength(1); }); + it('injects into the document it is given, which is also the registry', () => { + const src = nextSrc(); + const other = document.implementation.createHTMLDocument('env'); + loadVendor(src, get, { document: other }); + loadVendor(src, get, { document: other }); + + const injectedThere = Array.from(other.querySelectorAll('script')).filter( + (script) => script.getAttribute('src') === src, + ); + expect(injectedThere).toHaveLength(1); + expect(injectedThere[0].hasAttribute(VENDOR_SCRIPT_MARKER)).toBe(true); + expect(scriptsOf(src)).toEqual([]); + }); + + it('throws vendor-not-ready naming the src where there is no document at all', () => { + // jsdom's global document cannot be removed: plain Node runs the source. + const source = path.join(__dirname, '..', 'loadVendor.ts'); + const script = ` + const mod = await import(${JSON.stringify(source)}); + const { loadVendor } = mod.default ?? mod; + try { loadVendor('https://vendor.example/sdk.js', () => undefined); } + catch (error) { console.log(JSON.stringify({ code: error.code, message: error.message })); } + `; + const output = execFileSync( + process.execPath, + ['--import', 'tsx', '--input-type=module', '--eval', script], + { cwd: __dirname, encoding: 'utf8' }, + ); + expect(JSON.parse(output)).toEqual({ + code: 'vendor-not-ready', + message: + 'Vendor script https://vendor.example/sdk.js has no document to go into', + }); + }); + it("adopts another bundle's script that already loaded: ready at once", async () => { const src = nextSrc(); // Another bundle injected and loaded it before this one asks. diff --git a/packages/web/core/src/dev.ts b/packages/web/core/src/dev.ts new file mode 100644 index 000000000..f3edf444f --- /dev/null +++ b/packages/web/core/src/dev.ts @@ -0,0 +1,2 @@ +export { mockScriptDocument } from './mockScriptDocument'; +export type { MockScript, MockScriptDocument } from './mockScriptDocument'; diff --git a/packages/web/core/src/loadVendor.ts b/packages/web/core/src/loadVendor.ts index 67cea814c..5c09a76e1 100644 --- a/packages/web/core/src/loadVendor.ts +++ b/packages/web/core/src/loadVendor.ts @@ -11,6 +11,12 @@ export interface LoadVendorOptions { * script is inserted. `src` and the marker cannot be overwritten. */ attributes?: Record; + /** + * The document the script goes into, and whose scripts are the registry: + * the destination's env one (`getEnv(env).document`). The global document + * without it. + */ + document?: Document; } export interface Vendor { @@ -47,8 +53,8 @@ const injected = new WeakMap(); * marked script with that src means the src was injected (once per page, a * failure is never re-injected), and another document has its own. */ -function scriptOf(src: string): HTMLScriptElement | undefined { - const marked = document.querySelectorAll( +function scriptOf(doc: Document, src: string): HTMLScriptElement | undefined { + const marked = doc.querySelectorAll( `script[${VENDOR_SCRIPT_MARKER}]`, ); return Array.from(marked).find( @@ -57,12 +63,13 @@ function scriptOf(src: string): HTMLScriptElement | undefined { } function inject( + doc: Document, src: string, timeout: number, attributes: Record, available: () => boolean, ): Injected { - const existing = scriptOf(src); + const existing = scriptOf(doc, src); const known = existing && injected.get(existing); if (known) return known; @@ -81,7 +88,7 @@ function inject( injected.set(existing, entry); return entry; } - const script = existing ?? document.createElement('script'); + const script = existing ?? doc.createElement('script'); const settle = (outcome: Outcome) => { const waiting = entry.waiting; @@ -113,7 +120,7 @@ function inject( script.src = src; script.async = true; script.setAttribute(VENDOR_SCRIPT_MARKER, ''); - document.head.appendChild(script); + doc.head.appendChild(script); } return entry; @@ -138,7 +145,9 @@ const timedOut = (timeout: number) => * its readiness. The deadline runs from the injection and equals the * destination's resolved timeout, so an awaited `ready()` settles before the * push's own timeout. Without `src` (the page loads the vendor) nothing is - * injected and there is no deadline. + * injected and there is no deadline. A `src` with no document to inject + * into (neither `options.document` nor a global one) throws + * `vendor-not-ready`. */ export function loadVendor( src: string | undefined, @@ -150,14 +159,23 @@ export function loadVendor( return vendor !== undefined && vendor !== null; }; - const script = src - ? inject( - src, - resolveDestinationTimeout(options.timeout), - options.attributes ?? {}, - available, - ) - : undefined; + let script: Injected | undefined; + if (src) { + const doc = + options.document ?? + (typeof document !== 'undefined' ? document : undefined); + if (!doc) + throw new StepError(`Vendor script ${src} has no document to go into`, { + code: 'vendor-not-ready', + }); + script = inject( + doc, + src, + resolveDestinationTimeout(options.timeout), + options.attributes ?? {}, + available, + ); + } const present = (): Promise => { const vendor = get(); diff --git a/packages/web/core/src/mockScriptDocument.ts b/packages/web/core/src/mockScriptDocument.ts new file mode 100644 index 000000000..a44a21b77 --- /dev/null +++ b/packages/web/core/src/mockScriptDocument.ts @@ -0,0 +1,52 @@ +/** A script element, as far as `loadVendor` uses one. */ +export type MockScript = { + src: string; + async: boolean; + getAttribute(name: string): string | null; + setAttribute(name: string, value: string): void; + addEventListener(type: string, listener: () => void): void; +}; + +/** A document, as far as `loadVendor` injects into one. */ +export type MockScriptDocument = { + querySelectorAll(selector: string): MockScript[]; + createElement(tagName: string): MockScript; + head: { appendChild(script: MockScript): void }; +}; + +/** + * A document without a DOM for `loadVendor` to inject into, for an examples + * env (tests, simulation in Node). It keeps the scripts appended to its head, + * so each src goes in once, and an appended script loads at once. + */ +export function mockScriptDocument(): MockScriptDocument { + const scripts: MockScript[] = []; + const loads = new WeakMap void>>(); + + return { + querySelectorAll: () => [...scripts], + createElement: () => { + const attributes = new Map(); + const script: MockScript = { + src: '', + async: false, + getAttribute: (name) => + name === 'src' ? script.src : (attributes.get(name) ?? null), + setAttribute: (name, value) => { + attributes.set(name, value); + }, + addEventListener: (type, listener) => { + if (type === 'load') + loads.set(script, [...(loads.get(script) ?? []), listener]); + }, + }; + return script; + }, + head: { + appendChild: (script) => { + scripts.push(script); + (loads.get(script) ?? []).forEach((listener) => listener()); + }, + }, + }; +} diff --git a/packages/web/core/tsup.config.ts b/packages/web/core/tsup.config.ts index d5c3d722a..db969c386 100644 --- a/packages/web/core/tsup.config.ts +++ b/packages/web/core/tsup.config.ts @@ -1,3 +1,10 @@ import { defineConfig, buildModules } from '@walkeros/config/tsup'; -export default defineConfig([buildModules()]); +export default defineConfig([ + buildModules(), + + buildModules({ + entry: ['src/dev.ts'], + outDir: 'dist', + }), +]); diff --git a/packages/web/destinations/clarity/src/index.ts b/packages/web/destinations/clarity/src/index.ts index 7bd3d78c4..2d4d10d69 100644 --- a/packages/web/destinations/clarity/src/index.ts +++ b/packages/web/destinations/clarity/src/index.ts @@ -14,6 +14,7 @@ import type { ClaritySDK, ConsentV2, Destination, + Env, Mapping, Settings, } from './types'; @@ -45,11 +46,11 @@ function installStub(): ClarityQueue { } /** The tag script `loadVendor` injects; it replays the stub's queue. */ -function vendor(apiKey: string, timeout?: number) { +function vendor(apiKey: string, env: Env | undefined, timeout?: number) { return loadVendor( `${SCRIPT_SRC}${apiKey}?ref=npm`, () => getEnv().window.clarity, - { timeout }, + { timeout, document: getEnv(env).document }, ); } @@ -58,12 +59,16 @@ function vendor(apiKey: string, timeout?: number) { * through `loadVendor`, so a script that fails to load fails every later * call with `vendor-not-loaded`. */ -function realClarity(apiKey: string, timeout?: number): ClaritySDK { - const loaded = () => vendor(apiKey, timeout).check(); +function realClarity( + apiKey: string, + env: Env | undefined, + timeout?: number, +): ClaritySDK { + const loaded = () => vendor(apiKey, env, timeout).check(); return { init: () => { installStub(); - vendor(apiKey, timeout); + vendor(apiKey, env, timeout); }, identify: (...args) => { loaded(); @@ -101,12 +106,12 @@ function isClaritySDK(value: unknown): value is ClaritySDK { * otherwise the real one. */ function getClarity( - env: unknown, + env: Env | undefined, apiKey: string, timeout?: number, ): ClaritySDK { - const mock = isObject(env) ? env.clarity : undefined; - if (mock === undefined) return realClarity(apiKey, timeout); + const mock = env?.clarity; + if (mock === undefined) return realClarity(apiKey, env, timeout); if (isClaritySDK(mock)) return mock; throw new StepError('env.clarity is not a Clarity SDK', { code: 'config' }); } diff --git a/packages/web/destinations/gtag/src/ads/config.ts b/packages/web/destinations/gtag/src/ads/config.ts index 2e2c91fc4..8e8e5083b 100644 --- a/packages/web/destinations/gtag/src/ads/config.ts +++ b/packages/web/destinations/gtag/src/ads/config.ts @@ -24,7 +24,7 @@ export function initAds( // Default currency value if (!settings.currency) settings.currency = 'EUR'; - if (loadScript) loadGtag(conversionId, undefined, timeout); + if (loadScript) loadGtag(conversionId, undefined, timeout, env); // Initialize gtag infrastructure initializeGtag(window); diff --git a/packages/web/destinations/gtag/src/ga4/config.ts b/packages/web/destinations/gtag/src/ga4/config.ts index 3289d64ed..fabc6ee44 100644 --- a/packages/web/destinations/gtag/src/ga4/config.ts +++ b/packages/web/destinations/gtag/src/ga4/config.ts @@ -42,7 +42,7 @@ export function initGA4( // Load the gtag script. `server_container_url` routes measurement data and // says nothing about script delivery, so first-party serving needs its own // `scriptSrc` pointing at the container's configured tag serving path. - if (loadScript) loadGtag(measurementId, scriptSrc, timeout); + if (loadScript) loadGtag(measurementId, scriptSrc, timeout, env); const gtagSettings: WalkerOS.AnyObject = {}; diff --git a/packages/web/destinations/gtag/src/gtm/config.ts b/packages/web/destinations/gtag/src/gtm/config.ts index 01273b1ce..6601eff6f 100644 --- a/packages/web/destinations/gtag/src/gtm/config.ts +++ b/packages/web/destinations/gtag/src/gtm/config.ts @@ -10,14 +10,15 @@ const defaultDomain = 'https://www.googletagmanager.com/gtm.js?id='; * The GTM container script `loadVendor` injects (once per src and page). The * container replays its dataLayer, so it is only watched for a failure. */ -export function gtmVendor(settings: GTMSettings, timeout?: number) { +export function gtmVendor(settings: GTMSettings, timeout?: number, env?: Env) { const { containerId = '', dataLayer, domain } = settings; const dataLayerName = dataLayer || defaultDataLayer; const dl = dataLayerName != defaultDataLayer ? '&l=' + dataLayerName : ''; + const { window, document } = getEnv(env); return loadVendor( (domain || defaultDomain) + containerId + dl, - () => getEnv().window[dataLayerName], - { timeout }, + () => window[dataLayerName], + { timeout, document }, ); } @@ -50,6 +51,6 @@ export function initGTM( event: 'gtm.js', }), ]; - gtmVendor(settings, timeout); + gtmVendor(settings, timeout, env); return calls; } diff --git a/packages/web/destinations/gtag/src/index.ts b/packages/web/destinations/gtag/src/index.ts index e609fc2bb..9dd25695b 100644 --- a/packages/web/destinations/gtag/src/index.ts +++ b/packages/web/destinations/gtag/src/index.ts @@ -56,14 +56,14 @@ function setupConsentDefault(config: Config, env: Env | undefined): unknown[] { * Throws `vendor-not-loaded` once a script the destination loaded failed: * gtag replays its stub, so events before that go out as usual. */ -function checkScripts(config: Config): void { +function checkScripts(config: Config, env?: Env): void { const { settings = {}, loadScript, timeout } = config; const { ga4, ads, gtm } = settings; if (!loadScript) return; if (ga4?.measurementId && ga4.init !== false) - checkGtag(ga4.measurementId, ga4.scriptSrc, timeout); - if (ads?.conversionId) checkGtag(ads.conversionId, undefined, timeout); - if (gtm?.containerId) gtmVendor(gtm, timeout).check(); + checkGtag(ga4.measurementId, ga4.scriptSrc, timeout, env); + if (ads?.conversionId) checkGtag(ads.conversionId, undefined, timeout, env); + if (gtm?.containerId) gtmVendor(gtm, timeout, env).check(); } export const destinationGtag: Destination = { @@ -121,7 +121,7 @@ export const destinationGtag: Destination = { }, async push(event, { config, rule = {}, data, env, collector, logger, id }) { - checkScripts(config); + checkScripts(config, env); const { settings = {} } = config; const { ga4, ads, gtm } = settings; const eventMapping = rule.settings || {}; diff --git a/packages/web/destinations/gtag/src/shared/gtag.ts b/packages/web/destinations/gtag/src/shared/gtag.ts index 7e249e3e7..0c289c62b 100644 --- a/packages/web/destinations/gtag/src/shared/gtag.ts +++ b/packages/web/destinations/gtag/src/shared/gtag.ts @@ -1,13 +1,19 @@ // Gtag script loading and initialization utilities -import type { WindowWithDataLayer } from '../types'; +import type { Env, WindowWithDataLayer } from '../types'; import { StepError } from '@walkeros/core'; import { getEnv, loadVendor } from '@walkeros/web-core'; export const gtagScriptSrc = 'https://www.googletagmanager.com/gtag/js?id='; -/** gtag replays its dataLayer stub, so loadVendor only watches the script. */ -const gtagGlobal = () => getEnv().window.gtag; +/** + * gtag replays its dataLayer stub, so loadVendor only watches the script, in + * the env's document. + */ +function gtagVendor(src: string, timeout?: number, env?: Env) { + const { window, document } = getEnv(env); + return loadVendor(src, () => window.gtag, { timeout, document }); +} /** * Loads gtag.js through `loadVendor` (once per src and page). @@ -17,14 +23,17 @@ const gtagGlobal = () => getEnv().window.gtag; * If it fails to load, the script falls back to googletagmanager.com so a * misconfigured container degrades to working measurement instead of none. */ -export function loadGtag(id: string, src?: string, timeout?: number): void { - const primary = loadVendor(src ?? gtagScriptSrc + id, gtagGlobal, { - timeout, - }); +export function loadGtag( + id: string, + src?: string, + timeout?: number, + env?: Env, +): void { + const primary = gtagVendor(src ?? gtagScriptSrc + id, timeout, env); if (src) primary.ready().catch((error: unknown) => { if (error instanceof StepError && error.code === 'vendor-not-loaded') - loadVendor(gtagScriptSrc + id, gtagGlobal, { timeout }); + gtagVendor(gtagScriptSrc + id, timeout, env); }); } @@ -32,12 +41,17 @@ export function loadGtag(id: string, src?: string, timeout?: number): void { * Throws `vendor-not-loaded` once gtag.js for `id` failed to load, its * googletagmanager.com fallback too where a first-party `src` was set. */ -export function checkGtag(id: string, src?: string, timeout?: number): void { +export function checkGtag( + id: string, + src?: string, + timeout?: number, + env?: Env, +): void { try { - loadVendor(src ?? gtagScriptSrc + id, gtagGlobal, { timeout }).check(); + gtagVendor(src ?? gtagScriptSrc + id, timeout, env).check(); } catch (error) { if (!src) throw error; - loadVendor(gtagScriptSrc + id, gtagGlobal, { timeout }).check(); + gtagVendor(gtagScriptSrc + id, timeout, env).check(); } } diff --git a/packages/web/destinations/heap/src/index.ts b/packages/web/destinations/heap/src/index.ts index 1d73cc651..6f79f3fd3 100644 --- a/packages/web/destinations/heap/src/index.ts +++ b/packages/web/destinations/heap/src/index.ts @@ -6,6 +6,7 @@ import { StepError, stepCall, } from '@walkeros/core'; +import type { DestinationWeb } from '@walkeros/web-core'; import { getEnv, loadVendor } from '@walkeros/web-core'; import type { Destination, @@ -58,13 +59,13 @@ function getHeap(env: unknown): HeapSDK | undefined { function vendor( appId: string | undefined, loadScript: boolean | undefined, - env: unknown, + env: DestinationWeb.Env | undefined, timeout?: number, ) { return loadVendor( loadScript && appId ? scriptSrc(appId) : undefined, () => getHeap(env), - { timeout }, + { timeout, document: getEnv(env).document }, ); } diff --git a/packages/web/destinations/linkedin/src/index.ts b/packages/web/destinations/linkedin/src/index.ts index 27164d609..9a1f0108a 100644 --- a/packages/web/destinations/linkedin/src/index.ts +++ b/packages/web/destinations/linkedin/src/index.ts @@ -68,7 +68,7 @@ function vendor( return loadVendor( config.loadScript ? LINKEDIN_SCRIPT_SRC : undefined, () => getEnv(env).window.lintrk, - { timeout: config.timeout }, + { timeout: config.timeout, document: getEnv(env).document }, ); } diff --git a/packages/web/destinations/matomo/src/index.ts b/packages/web/destinations/matomo/src/index.ts index 9d8214977..fc8fd80ea 100644 --- a/packages/web/destinations/matomo/src/index.ts +++ b/packages/web/destinations/matomo/src/index.ts @@ -41,7 +41,7 @@ function vendor(url: string | undefined, env?: Env, timeout?: number) { return loadVendor( url ? `${normalizeUrl(url)}matomo.js` : undefined, () => getEnv(env).window._paq, - { timeout }, + { timeout, document: getEnv(env).document }, ); } diff --git a/packages/web/destinations/meta/src/index.ts b/packages/web/destinations/meta/src/index.ts index e7e56185d..e019d2494 100644 --- a/packages/web/destinations/meta/src/index.ts +++ b/packages/web/destinations/meta/src/index.ts @@ -19,7 +19,7 @@ function vendor(config: Config, env?: Env) { return loadVendor( config.loadScript ? SCRIPT_SRC : undefined, () => getEnv(env).window.fbq, - { timeout: config.timeout }, + { timeout: config.timeout, document: getEnv(env).document }, ); } diff --git a/packages/web/destinations/piano/src/index.ts b/packages/web/destinations/piano/src/index.ts index 0f947231a..e6028eeba 100644 --- a/packages/web/destinations/piano/src/index.ts +++ b/packages/web/destinations/piano/src/index.ts @@ -1,6 +1,6 @@ import type { Config, Env, Destination, PianoAnalytics } from './types'; import { isObject, StepError, stepCall } from '@walkeros/core'; -import { loadVendor } from '@walkeros/web-core'; +import { getEnv, loadVendor } from '@walkeros/web-core'; // Types export * as DestinationPiano from './types'; @@ -15,7 +15,7 @@ function vendor(config: Config, env?: Env) { return loadVendor( config.loadScript ? SCRIPT_SRC : undefined, () => resolvePa(env), - { timeout: config.timeout }, + { timeout: config.timeout, document: getEnv(env).document }, ); } diff --git a/packages/web/destinations/pinterest/src/index.ts b/packages/web/destinations/pinterest/src/index.ts index 7dc512064..0cd2ece4c 100644 --- a/packages/web/destinations/pinterest/src/index.ts +++ b/packages/web/destinations/pinterest/src/index.ts @@ -61,7 +61,7 @@ function vendor( return loadVendor( config.loadScript !== false ? PINTEREST_SCRIPT_SRC : undefined, () => getEnv(env).window.pintrk, - { timeout: config.timeout }, + { timeout: config.timeout, document: getEnv(env).document }, ); } diff --git a/packages/web/destinations/piwikpro/src/index.ts b/packages/web/destinations/piwikpro/src/index.ts index d33a14012..00a6a1d34 100644 --- a/packages/web/destinations/piwikpro/src/index.ts +++ b/packages/web/destinations/piwikpro/src/index.ts @@ -54,7 +54,7 @@ function vendor(url: string | undefined, env?: Env, timeout?: number) { return loadVendor( url ? `${normalizeUrl(url)}ppms.js` : undefined, () => getEnv(env).window._paq, - { timeout }, + { timeout, document: getEnv(env).document }, ); } diff --git a/packages/web/destinations/plausible/src/examples/env.ts b/packages/web/destinations/plausible/src/examples/env.ts index d1db346e4..4b2cab525 100644 --- a/packages/web/destinations/plausible/src/examples/env.ts +++ b/packages/web/destinations/plausible/src/examples/env.ts @@ -1,3 +1,4 @@ +import { mockScriptDocument } from '@walkeros/web-core/dev'; import type { Env } from '../types'; /** @@ -14,16 +15,7 @@ export const init: Env | undefined = { window: { // plausible absent: init() is expected to install the stub. }, - document: { - createElement: () => ({ - src: '', - dataset: {}, - setAttribute: () => {}, - removeAttribute: () => {}, - }), - head: { appendChild: () => {} }, - querySelector: () => null, - }, + document: mockScriptDocument(), }; export const push: Env = { @@ -33,16 +25,7 @@ export const push: Env = { q: [] as IArguments[], }), }, - document: { - createElement: () => ({ - src: '', - dataset: {}, - setAttribute: () => {}, - removeAttribute: () => {}, - }), - head: { appendChild: () => {} }, - querySelector: () => null, - }, + document: mockScriptDocument(), }; /** diff --git a/packages/web/destinations/plausible/src/index.ts b/packages/web/destinations/plausible/src/index.ts index 0bb3fe859..513606328 100644 --- a/packages/web/destinations/plausible/src/index.ts +++ b/packages/web/destinations/plausible/src/index.ts @@ -64,6 +64,7 @@ function vendor( { timeout: config.timeout, attributes: domain ? { 'data-domain': domain } : undefined, + document: getEnv(env).document, }, ); } diff --git a/packages/web/destinations/snowplow/src/examples/env.ts b/packages/web/destinations/snowplow/src/examples/env.ts index f407e861e..5f4b19130 100644 --- a/packages/web/destinations/snowplow/src/examples/env.ts +++ b/packages/web/destinations/snowplow/src/examples/env.ts @@ -1,3 +1,4 @@ +import { mockScriptDocument } from '@walkeros/web-core/dev'; import type { Env } from '../types'; /** @@ -14,15 +15,7 @@ export const init: Env | undefined = { window: { // snowplow absent: setup() is expected to install the queue stub. }, - document: { - createElement: () => ({ - src: '', - async: false, - setAttribute: () => {}, - removeAttribute: () => {}, - }), - head: { appendChild: () => {} }, - }, + document: mockScriptDocument(), }; export const push: Env = { @@ -31,15 +24,7 @@ export const push: Env = { q: [], }), }, - document: { - createElement: () => ({ - src: '', - async: false, - setAttribute: () => {}, - removeAttribute: () => {}, - }), - head: { appendChild: () => {} }, - }, + document: mockScriptDocument(), }; export const simulation = [ diff --git a/packages/web/destinations/snowplow/src/index.ts b/packages/web/destinations/snowplow/src/index.ts index 8a61eee39..27596fb58 100644 --- a/packages/web/destinations/snowplow/src/index.ts +++ b/packages/web/destinations/snowplow/src/index.ts @@ -51,7 +51,7 @@ function vendor( ? config.settings?.scriptUrl || DEFAULT_SCRIPT_URL : undefined, () => getEnv(env).window.snowplow, - { timeout: config.timeout }, + { timeout: config.timeout, document: getEnv(env).document }, ); } diff --git a/packages/web/destinations/tiktok/src/examples/env.ts b/packages/web/destinations/tiktok/src/examples/env.ts index 18ebc11e8..f89c52aeb 100644 --- a/packages/web/destinations/tiktok/src/examples/env.ts +++ b/packages/web/destinations/tiktok/src/examples/env.ts @@ -1,3 +1,4 @@ +import { mockScriptDocument } from '@walkeros/web-core/dev'; import type { Env, TTQ } from '../types'; const noop = () => {}; @@ -31,12 +32,13 @@ function makeMockTTQ(): TTQ { }); } -/** Minimal web env: window has a mock ttq. */ +/** Minimal web env: window has a mock ttq, document takes the Pixel script. */ function makeEnv(): Env { return { window: { ttq: makeMockTTQ(), }, + document: mockScriptDocument(), }; } diff --git a/packages/web/destinations/tiktok/src/index.ts b/packages/web/destinations/tiktok/src/index.ts index 061b08155..59e07b96d 100644 --- a/packages/web/destinations/tiktok/src/index.ts +++ b/packages/web/destinations/tiktok/src/index.ts @@ -64,7 +64,7 @@ function vendor( return loadVendor( config.loadScript !== false ? SCRIPT_SRC : undefined, () => getEnv(env).window.ttq, - { timeout: config.timeout }, + { timeout: config.timeout, document: getEnv(env).document }, ); } diff --git a/skills/walkeros-create-destination/templates/simple/index.ts b/skills/walkeros-create-destination/templates/simple/index.ts index d2ba7d924..b47828924 100644 --- a/skills/walkeros-create-destination/templates/simple/index.ts +++ b/skills/walkeros-create-destination/templates/simple/index.ts @@ -30,7 +30,7 @@ function vendor(config: Config, env?: Env) { return loadVendor( config.loadScript ? SCRIPT_SRC : undefined, () => getEnv(env).window.vendorSdk, - { timeout: config.timeout }, + { timeout: config.timeout, document: getEnv(env).document }, ); } From b1d65e28af0985a018fb660c9f65632d16d635f9 Mon Sep 17 00:00:00 2001 From: alexanderkirtzel Date: Wed, 21 Oct 2026 10:15:18 +0000 Subject: [PATCH 38/41] loader changesets --- .changeset/web-core-load-vendor-document.md | 8 ++++++++ .../web-destination-amplitude-lazy-plugins.md | 7 +++++++ .../web-destinations-script-document.md | 19 +++++++++++++++++++ skills/walkeros-create-destination/SKILL.md | 10 ++++++---- website/docs/destinations/create-your-own.mdx | 2 +- 5 files changed, 41 insertions(+), 5 deletions(-) create mode 100644 .changeset/web-core-load-vendor-document.md create mode 100644 .changeset/web-destination-amplitude-lazy-plugins.md create mode 100644 .changeset/web-destinations-script-document.md diff --git a/.changeset/web-core-load-vendor-document.md b/.changeset/web-core-load-vendor-document.md new file mode 100644 index 000000000..f9157e481 --- /dev/null +++ b/.changeset/web-core-load-vendor-document.md @@ -0,0 +1,8 @@ +--- +'@walkeros/web-core': patch +--- + +`loadVendor` injects the script into the document that `env` provides, falling +back to the page's own, and fails with `vendor-not-ready` when there is no +document at all. The new `@walkeros/web-core/dev` entry exports +`mockScriptDocument()` for examples and tests that run without a browser. diff --git a/.changeset/web-destination-amplitude-lazy-plugins.md b/.changeset/web-destination-amplitude-lazy-plugins.md new file mode 100644 index 000000000..de8206895 --- /dev/null +++ b/.changeset/web-destination-amplitude-lazy-plugins.md @@ -0,0 +1,7 @@ +--- +'@walkeros/web-destination-amplitude': patch +--- + +Amplitude loads each plugin package only when that plugin is enabled. The +package now imports cleanly under Node and tsx, so simulation and tooling no +longer fail on a browser-only plugin that the flow never uses. diff --git a/.changeset/web-destinations-script-document.md b/.changeset/web-destinations-script-document.md new file mode 100644 index 000000000..622e932aa --- /dev/null +++ b/.changeset/web-destinations-script-document.md @@ -0,0 +1,19 @@ +--- +'@walkeros/web-destination-meta': patch +'@walkeros/web-destination-gtag': patch +'@walkeros/web-destination-piano': patch +'@walkeros/web-destination-linkedin': patch +'@walkeros/web-destination-plausible': patch +'@walkeros/web-destination-heap': patch +'@walkeros/web-destination-snowplow': patch +'@walkeros/web-destination-tiktok': patch +'@walkeros/web-destination-pinterest': patch +'@walkeros/web-destination-piwikpro': patch +'@walkeros/web-destination-matomo': patch +'@walkeros/web-destination-clarity': patch +--- + +Web destinations that load a vendor script inject it through the document of +their `env`, so simulation and server-side rendering work without a browser +page. A page still gets the script in its own document by default, and a missing +document fails the push with `vendor-not-ready`. diff --git a/skills/walkeros-create-destination/SKILL.md b/skills/walkeros-create-destination/SKILL.md index 7fc89ed96..9bdb14c46 100644 --- a/skills/walkeros-create-destination/SKILL.md +++ b/skills/walkeros-create-destination/SKILL.md @@ -664,10 +664,12 @@ Use these templates as your starting point: sends the event and reports once with `reportCause`. Only a wholly unsendable event throws `missing-input`. 9. **Scripts through `loadVendor`** (`@walkeros/web-core`). It injects the - script once per page and reports `vendor-not-loaded`, `vendor-not-ready` and - `vendor-timeout`. For a vendor that replays its own stub, call `check()` in - `push`; for any other, `await ready()`. Pass attributes the tag needs - (`data-domain`) through `options.attributes`. + script once per page into the document `env` provides (the page's own by + default) and reports `vendor-not-loaded`, `vendor-not-ready` and + `vendor-timeout`. For tests and examples, `mockScriptDocument()` from + `@walkeros/web-core/dev` provides a document. For a vendor that replays its + own stub, call `check()` in `push`; for any other, `await ready()`. Pass + attributes the tag needs (`data-domain`) through `options.attributes`. 10. **Honour `rule.silent`.** A rule with `silent: true` sends no event of its own and runs only the side effects it maps (identify, group). Do not `return` early for it (`walkeros/no-silent-return` is an error): write diff --git a/website/docs/destinations/create-your-own.mdx b/website/docs/destinations/create-your-own.mdx index b1a9bdc6d..5b8060d90 100644 --- a/website/docs/destinations/create-your-own.mdx +++ b/website/docs/destinations/create-your-own.mdx @@ -151,7 +151,7 @@ never the value. ### Vendor scripts A web destination that loads a vendor script uses `loadVendor` from -`@walkeros/web-core`. It injects the script once per page and throws +`@walkeros/web-core`. It injects the script once per page, into the document the destination's `env` provides (the page's own by default), so simulation and server-side rendering work without a browser page. It throws `vendor-not-loaded`, `vendor-not-ready` or `vendor-timeout`, so a blocked script shows as failed pushes instead of lost events. From faa5c77fe79660bd43e04201483c8af957738ee6 Mon Sep 17 00:00:00 2001 From: alexanderkirtzel Date: Wed, 21 Oct 2026 12:06:00 +0000 Subject: [PATCH 39/41] loader budget --- .changeset/collector-walkerjs-size.md | 4 ++-- apps/walkerjs/src/__tests__/integration/dist.test.ts | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.changeset/collector-walkerjs-size.md b/.changeset/collector-walkerjs-size.md index 2c0b78f41..9e43cccae 100644 --- a/.changeset/collector-walkerjs-size.md +++ b/.changeset/collector-walkerjs-size.md @@ -3,5 +3,5 @@ --- The web bundle `walker.js` carries the full failure contract, the circuit -breaker included. It grows from 35,181 bytes to 39,334 bytes, measured as gzip --9, and its size is not gated by a budget. +breaker included. It grows from 35,181 bytes to 39,411 bytes, measured as gzip +-9. diff --git a/apps/walkerjs/src/__tests__/integration/dist.test.ts b/apps/walkerjs/src/__tests__/integration/dist.test.ts index 742b99b2f..f9278eea3 100644 --- a/apps/walkerjs/src/__tests__/integration/dist.test.ts +++ b/apps/walkerjs/src/__tests__/integration/dist.test.ts @@ -14,12 +14,12 @@ const builds: [string, string][] = [ ]), ]; -// The Tag Mode loader budget: dist/walker.js measured before the loader -// (2026-10-09, gzip level 9) plus at most 700 bytes for the loader. -// In effect this caps walker.js's total size, with ~168 B headroom after the -// loader: a PR that grows walker.js for another reason raises -// BEFORE_LOADER_GZIP_BYTES by its measured delta and says so. -const BEFORE_LOADER_GZIP_BYTES = 34_627; +// The Tag Mode loader budget: dist/walker.js without the loader (gzip level +// 9) plus at most 700 bytes for the loader. +// In effect this caps walker.js's total size: a PR that grows walker.js for +// another reason raises BEFORE_LOADER_GZIP_BYTES by its measured delta and +// says so. +const BEFORE_LOADER_GZIP_BYTES = 38_857; const LOADER_BUDGET_BYTES = 700; it('keeps the Tag Mode loader within its budget', () => { From 964590ac95fd1cd7b2a9414bfdc8b35c3b7e160b Mon Sep 17 00:00:00 2001 From: alexanderkirtzel Date: Wed, 21 Oct 2026 21:44:04 +0200 Subject: [PATCH 40/41] platform boundaries --- .changeset/cli-web-core-entries.md | 6 + .changeset/collector-web-observe-slot.md | 8 + .changeset/config-platform-boundary.md | 8 + .changeset/core-platform-boundary.md | 10 + .changeset/datamanager-unreadable-answer.md | 6 + .changeset/runner-auth-header-undefined.md | 5 + .changeset/web-core-platform-boundary.md | 7 + .github/workflows/test.yml | 5 + apps/cli/eslint.config.mjs | 4 + apps/demos/react/src/setupTests.ts | 2 +- apps/explorer/eslint.config.mjs | 19 + apps/storybook-addon/eslint.config.mjs | 3 + apps/walkerjs/package.json | 1 + apps/walkerjs/src/index.ts | 2 +- package-lock.json | 5 + package.json | 1 + packages/cli/eslint.config.js | 2 + packages/cli/package.json | 1 + .../integration/bundle/cdn-jsdom.test.ts | 39 ++ .../integration/bundle/size-budget.test.ts | 13 +- .../unit/bundle/bundler-codegen.test.ts | 65 +++- .../unit/bundle/preview-codegen.test.ts | 18 +- .../src/__tests__/unit/bundle/wrap.test.ts | 31 +- .../bundle/__tests__/observe-codegen.test.ts | 12 + packages/cli/src/commands/bundle/bundler.ts | 78 ++-- packages/cli/src/commands/bundle/wrap.ts | 21 +- packages/collector/eslint.config.mjs | 14 + .../src/__tests__/build-flags-bundle.test.ts | 2 +- .../__tests__/cache-codec-roundtrip.test.ts | 6 +- .../src/__tests__/observe-wiring.test.ts | 216 +++++++---- .../src/__tests__/store-cache-wrapper.test.ts | 5 +- .../src/dev/__tests__/conformance.test.ts | 84 +---- .../__tests__/fixture-package/src/vendor.ts | 94 +---- .../collector/src/dev/__tests__/fixtures.ts | 14 + packages/collector/src/dev/boundary.ts | 121 ++---- packages/collector/src/dev/conformance.ts | 106 +++--- packages/collector/src/dev/runExamples.ts | 103 +++--- packages/collector/src/flow.ts | 172 +++------ .../config/eslint/__tests__/platform.test.mjs | 71 ++++ packages/config/eslint/index.mjs | 34 ++ packages/config/eslint/neutral.mjs | 51 +++ packages/config/eslint/node.mjs | 30 +- packages/config/eslint/web.mjs | 25 +- packages/config/package.json | 2 + packages/config/scripts/check-platform.mjs | 346 ++++++++++++++++++ .../config/scripts/check-platform.test.mjs | 221 +++++++++++ packages/config/tsconfig/base.json | 2 + packages/config/tsconfig/web.json | 2 +- packages/core/eslint.config.mjs | 13 + packages/core/src/__tests__/cache.test.ts | 2 +- packages/core/src/__tests__/clone.test.ts | 12 +- packages/core/src/__tests__/flow.test.ts | 130 +++---- packages/core/src/__tests__/preview.test.ts | 4 + packages/core/src/__tests__/state.test.ts | 2 +- packages/core/src/__tests__/stepError.test.ts | 56 ++- packages/core/src/batchedPoster.ts | 5 +- packages/core/src/errorCause.ts | 12 + packages/core/src/fatalError.ts | 9 +- packages/core/src/flow.ts | 19 +- packages/core/src/getId.ts | 14 +- packages/core/src/index.ts | 9 +- packages/core/src/is.ts | 12 - packages/core/src/logger.ts | 2 +- packages/core/src/moin.ts | 91 ----- packages/core/src/node.ts | 25 +- .../{ => node}/__tests__/scrubJson.test.ts | 2 +- .../__tests__/storeCodec.test.ts} | 2 +- .../{ => node}/__tests__/toPrintable.test.ts | 0 packages/core/src/{ => node}/scrubJson.ts | 2 +- packages/core/src/node/storeCodec.ts | 142 +++++++ packages/core/src/{ => node}/toPrintable.ts | 0 packages/core/src/observeConnect.ts | 10 + packages/core/src/preview.ts | 268 +------------- packages/core/src/scope.ts | 6 +- packages/core/src/stepError.ts | 12 +- packages/core/src/store/codec.ts | 152 +------- packages/core/src/types/collector.ts | 11 +- packages/core/src/types/flow.ts | 38 ++ packages/destinations/eslint.config.mjs | 3 + packages/mcps/eslint.config.mjs | 4 + packages/mcps/mcp/eslint.config.js | 2 + packages/runner/eslint.config.js | 2 + packages/runner/src/auth-headers.ts | 12 +- packages/server/core/tsconfig.json | 2 +- .../datamanager/src/__tests__/index.test.ts | 39 ++ .../destinations/datamanager/src/push.ts | 32 +- .../src/__tests__/input-hygiene.test.ts | 16 +- .../express/src/__tests__/text-body.test.ts | 4 +- packages/server/sources/fetch/tsconfig.json | 2 +- .../fs/src/__tests__/conformance.test.ts | 2 +- packages/server/stores/fs/src/store.ts | 4 +- packages/server/stores/gcs/src/store.ts | 4 +- packages/server/stores/s3/src/store.ts | 4 +- packages/transformers/eslint.config.mjs | 3 + packages/transformers/ga4/tsconfig.json | 2 +- packages/transformers/validate/tsconfig.json | 2 +- packages/web/core/package.json | 1 + .../core/src/__tests__/fixtures/src/sender.ts | 27 ++ .../core/src/__tests__/fixtures/src/vendor.ts | 147 ++++++++ .../core/src/__tests__/injectScript.test.ts} | 2 +- .../{ => web}/core/src/__tests__/moin.test.ts | 92 ++--- .../core/src/__tests__/observeSlot.test.ts | 300 +++++++++++++++ packages/web/core/src/__tests__/page.test.ts | 287 +++++++++++++++ .../src/__tests__/previewActivator.test.ts} | 14 +- .../web/core/src/__tests__/sendWeb.test.ts | 4 +- packages/web/core/src/dev.ts | 1 + packages/web/core/src/dev/page.ts | 138 +++++++ packages/web/core/src/index.ts | 4 + .../core/src/injectScript.ts} | 38 +- packages/web/core/src/moin.ts | 67 ++++ packages/web/core/src/observeSlot.ts | 109 ++++++ packages/web/core/src/previewActivator.ts | 235 ++++++++++++ packages/web/core/tsconfig.json | 2 +- .../src/__tests__/conformance.test.ts | 2 + .../api/src/__tests__/conformance.test.ts | 2 + .../clarity/src/__tests__/conformance.test.ts | 2 + .../d8a/src/__tests__/conformance.test.ts | 2 + .../src/__tests__/conformance.test.ts | 2 + packages/web/destinations/gtag/package.json | 1 + .../gtag/src/__tests__/conformance.test.ts | 2 + packages/web/destinations/gtag/tsconfig.json | 3 +- .../heap/src/__tests__/conformance.test.ts | 2 + .../hotjar/src/__tests__/conformance.test.ts | 2 + .../src/__tests__/conformance.test.ts | 2 + .../matomo/src/__tests__/conformance.test.ts | 2 + .../meta/src/__tests__/conformance.test.ts | 2 + packages/web/destinations/meta/tsconfig.json | 3 +- .../src/__tests__/conformance.test.ts | 2 + .../src/__tests__/conformance.test.ts | 2 + .../piano/src/__tests__/conformance.test.ts | 2 + .../src/__tests__/conformance.test.ts | 2 + .../src/__tests__/conformance.test.ts | 2 + .../src/__tests__/conformance.test.ts | 2 + .../posthog/src/__tests__/conformance.test.ts | 2 + .../segment/src/__tests__/conformance.test.ts | 2 + .../src/__tests__/conformance.test.ts | 2 + .../tiktok/src/__tests__/conformance.test.ts | 2 + packages/web/sources/demo/eslint.config.mjs | 3 + skills/walkeros-create-destination/SKILL.md | 10 +- .../templates/simple/conformance.test.ts | 6 + skills/walkeros-create-source/SKILL.md | 3 +- skills/walkeros-create-transformer/SKILL.md | 6 +- skills/walkeros-testing-strategy/SKILL.md | 5 +- .../SKILL.md | 3 +- .../SKILL.md | 34 ++ skills/walkeros-understanding-stores/SKILL.md | 5 +- skills/walkeros-using-cli/SKILL.md | 5 +- skills/walkeros-using-step-examples/SKILL.md | 5 +- turbo.json | 12 +- website/docs/core/index.mdx | 28 -- website/docs/core/web.mdx | 28 ++ website/docs/destinations/create-your-own.mdx | 4 +- .../docs/getting-started/modes/integrated.mdx | 15 +- website/docs/getting-started/observe.mdx | 10 +- website/docs/guides/reference-syntax.mdx | 11 +- 155 files changed, 3570 insertions(+), 1434 deletions(-) create mode 100644 .changeset/cli-web-core-entries.md create mode 100644 .changeset/collector-web-observe-slot.md create mode 100644 .changeset/config-platform-boundary.md create mode 100644 .changeset/core-platform-boundary.md create mode 100644 .changeset/datamanager-unreadable-answer.md create mode 100644 .changeset/runner-auth-header-undefined.md create mode 100644 .changeset/web-core-platform-boundary.md create mode 100644 apps/cli/eslint.config.mjs create mode 100644 apps/storybook-addon/eslint.config.mjs create mode 100644 packages/collector/eslint.config.mjs create mode 100644 packages/config/eslint/__tests__/platform.test.mjs create mode 100644 packages/config/eslint/neutral.mjs create mode 100644 packages/config/scripts/check-platform.mjs create mode 100644 packages/config/scripts/check-platform.test.mjs create mode 100644 packages/core/eslint.config.mjs create mode 100644 packages/core/src/errorCause.ts rename packages/core/src/{ => node}/__tests__/scrubJson.test.ts (97%) rename packages/core/src/{store/__tests__/codec.test.ts => node/__tests__/storeCodec.test.ts} (98%) rename packages/core/src/{ => node}/__tests__/toPrintable.test.ts (100%) rename packages/core/src/{ => node}/scrubJson.ts (99%) create mode 100644 packages/core/src/node/storeCodec.ts rename packages/core/src/{ => node}/toPrintable.ts (100%) create mode 100644 packages/destinations/eslint.config.mjs create mode 100644 packages/mcps/eslint.config.mjs create mode 100644 packages/transformers/eslint.config.mjs create mode 100644 packages/web/core/src/__tests__/fixtures/src/sender.ts create mode 100644 packages/web/core/src/__tests__/fixtures/src/vendor.ts rename packages/{core/src/__tests__/inject-script.test.ts => web/core/src/__tests__/injectScript.test.ts} (93%) rename packages/{ => web}/core/src/__tests__/moin.test.ts (82%) create mode 100644 packages/web/core/src/__tests__/observeSlot.test.ts create mode 100644 packages/web/core/src/__tests__/page.test.ts rename packages/{core/src/__tests__/preview-activator.test.ts => web/core/src/__tests__/previewActivator.test.ts} (97%) create mode 100644 packages/web/core/src/dev/page.ts rename packages/{core/src/inject-script.ts => web/core/src/injectScript.ts} (51%) create mode 100644 packages/web/core/src/moin.ts create mode 100644 packages/web/core/src/observeSlot.ts create mode 100644 packages/web/core/src/previewActivator.ts create mode 100644 packages/web/sources/demo/eslint.config.mjs diff --git a/.changeset/cli-web-core-entries.md b/.changeset/cli-web-core-entries.md new file mode 100644 index 000000000..87d51e6b9 --- /dev/null +++ b/.changeset/cli-web-core-entries.md @@ -0,0 +1,6 @@ +--- +'@walkeros/cli': patch +--- + +Browser and wrap bundles import from `@walkeros/web-core`, which ships with the +CLI, so no extra package is downloaded. diff --git a/.changeset/collector-web-observe-slot.md b/.changeset/collector-web-observe-slot.md new file mode 100644 index 000000000..6aa193d81 --- /dev/null +++ b/.changeset/collector-web-observe-slot.md @@ -0,0 +1,8 @@ +--- +'@walkeros/collector': patch +--- + +A web Observe config takes `slot: pageObserveSlot()` from `@walkeros/web-core`; +without it one error line says so and Observe stays off. The conformance kit's +browser profiles take `page` from `@walkeros/web-core/dev`. Observed web bundles +need this CLI version. diff --git a/.changeset/config-platform-boundary.md b/.changeset/config-platform-boundary.md new file mode 100644 index 000000000..c4ccf6a42 --- /dev/null +++ b/.changeset/config-platform-boundary.md @@ -0,0 +1,8 @@ +--- +'@walkeros/config': patch +--- + +The presets set `lib` and `types` and enforce the platform: web and neutral code +are held to ES2020 APIs, and the new `@walkeros/config/eslint/neutral` and the +platform bans in `eslint/web` and `eslint/node` keep each package on its own +platform. `npm run check:platform` checks every package. diff --git a/.changeset/core-platform-boundary.md b/.changeset/core-platform-boundary.md new file mode 100644 index 000000000..a9b64a695 --- /dev/null +++ b/.changeset/core-platform-boundary.md @@ -0,0 +1,10 @@ +--- +'@walkeros/core': patch +--- + +Tag Mode's `moin` and the preview activator `browserSwapActivator` (with +`SwapActivatorConfig` and `SESSION_STORAGE_KEY`) moved to `@walkeros/web-core`, +and `isElementOrDocument` is removed. `serializeStoreValue`, +`deserializeStoreValue` and `StoreCodecError` moved to `@walkeros/core/node`. +`getFlowSettings` reads `$env` only from the `env` you pass it, never from +`process.env`. Errors keep `cause` in every browser. diff --git a/.changeset/datamanager-unreadable-answer.md b/.changeset/datamanager-unreadable-answer.md new file mode 100644 index 000000000..e5c58e823 --- /dev/null +++ b/.changeset/datamanager-unreadable-answer.md @@ -0,0 +1,6 @@ +--- +'@walkeros/server-destination-datamanager': patch +--- + +A 2xx Data Manager answer whose body is not a JSON object is reported as a +transport failure ("Data Manager answered with an unreadable body"). diff --git a/.changeset/runner-auth-header-undefined.md b/.changeset/runner-auth-header-undefined.md new file mode 100644 index 000000000..e754955f2 --- /dev/null +++ b/.changeset/runner-auth-header-undefined.md @@ -0,0 +1,5 @@ +--- +'@walkeros/runner': patch +--- + +Auth header records skip undefined values. diff --git a/.changeset/web-core-platform-boundary.md b/.changeset/web-core-platform-boundary.md new file mode 100644 index 000000000..a3d01ac1b --- /dev/null +++ b/.changeset/web-core-platform-boundary.md @@ -0,0 +1,7 @@ +--- +'@walkeros/web-core': patch +--- + +`@walkeros/web-core` exports `moin`, `browserSwapActivator` and +`pageObserveSlot`, and `@walkeros/web-core/dev` exports `page` for the +conformance tests of web steps. diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4a27c0ec9..ad3c980e5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -69,6 +69,11 @@ jobs: turbo-${{ runner.os }}- - name: Validate documentation run: npm run validate:links + # Every package is one platform (web, server or neutral) and its tsconfig + # and eslint config enforce it. Outside turbo, so no cache can replay an + # old pass. + - name: Check platform boundaries + run: npm run check:platform - name: Verify (PR, affected only) if: github.event_name == 'pull_request' run: | diff --git a/apps/cli/eslint.config.mjs b/apps/cli/eslint.config.mjs new file mode 100644 index 000000000..a9ecf85f8 --- /dev/null +++ b/apps/cli/eslint.config.mjs @@ -0,0 +1,4 @@ +import baseConfig from '@walkeros/config/eslint'; +import { nodePlatform } from '@walkeros/config/eslint/node'; + +export default [...baseConfig, ...nodePlatform]; diff --git a/apps/demos/react/src/setupTests.ts b/apps/demos/react/src/setupTests.ts index bbc90116f..e44fcf0ac 100644 --- a/apps/demos/react/src/setupTests.ts +++ b/apps/demos/react/src/setupTests.ts @@ -11,4 +11,4 @@ const localStorageMock = { removeItem: jest.fn(), clear: jest.fn(), }; -global.localStorage = localStorageMock as unknown as Storage; +globalThis.localStorage = localStorageMock as unknown as Storage; diff --git a/apps/explorer/eslint.config.mjs b/apps/explorer/eslint.config.mjs index 6c466eab7..1b7588bcd 100644 --- a/apps/explorer/eslint.config.mjs +++ b/apps/explorer/eslint.config.mjs @@ -1,7 +1,26 @@ import baseConfig from '@walkeros/config/eslint/web'; +import { nodeGlobals } from '@walkeros/config/eslint'; import storybook from 'eslint-plugin-storybook'; export default [ ...baseConfig, ...storybook.configs['flat/recommended'], + { + // walkeros-design-check is a Node bin, not web code. + files: ['src/design/check/**', 'src/design/generate.ts'], + rules: { + 'no-restricted-globals': 'off', + 'no-restricted-imports': 'off', + }, + }, + { + // process.env.NODE_ENV is the define every React bundler sets. + files: ['src/components/atoms/code.tsx'], + rules: { + 'no-restricted-globals': [ + 'error', + ...nodeGlobals.filter((name) => name !== 'process'), + ], + }, + }, ]; diff --git a/apps/storybook-addon/eslint.config.mjs b/apps/storybook-addon/eslint.config.mjs new file mode 100644 index 000000000..5e6790c96 --- /dev/null +++ b/apps/storybook-addon/eslint.config.mjs @@ -0,0 +1,3 @@ +import webConfig from '@walkeros/config/eslint/web'; + +export default [...webConfig]; diff --git a/apps/walkerjs/package.json b/apps/walkerjs/package.json index 7942419f0..95aed49dc 100644 --- a/apps/walkerjs/package.json +++ b/apps/walkerjs/package.json @@ -28,6 +28,7 @@ "devDependencies": { "@walkeros/collector": "4.8.1", "@walkeros/core": "4.8.1", + "@walkeros/web-core": "4.8.1", "@walkeros/web-source-browser": "4.8.1", "@walkeros/web-source-session": "4.8.1", "@swc/jest": "^0.2.39", diff --git a/apps/walkerjs/src/index.ts b/apps/walkerjs/src/index.ts index 34980330b..8c0b0ed93 100644 --- a/apps/walkerjs/src/index.ts +++ b/apps/walkerjs/src/index.ts @@ -1,5 +1,5 @@ import { startFlow } from '@walkeros/collector'; -import { moin } from '@walkeros/core'; +import { moin } from '@walkeros/web-core'; import { sourceBrowser } from '@walkeros/web-source-browser'; import { sourceSession } from '@walkeros/web-source-session'; import { dataLayerDestination } from './destination'; diff --git a/package-lock.json b/package-lock.json index ba73e1b93..63517b64f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -950,6 +950,7 @@ "@types/jest": "^30.0.0", "@walkeros/collector": "4.8.1", "@walkeros/core": "4.8.1", + "@walkeros/web-core": "4.8.1", "@walkeros/web-source-browser": "4.8.1", "@walkeros/web-source-session": "4.8.1", "jest": "^30.2.0", @@ -47455,6 +47456,7 @@ "@walkeros/server-core": "4.8.1", "@walkeros/server-destination-api": "4.8.1", "@walkeros/transformer-validate": "4.8.1", + "@walkeros/web-core": "4.8.1", "ajv": "^8.17.1", "chalk": "^5.6.2", "ci-info": "^4.4.0", @@ -48423,6 +48425,7 @@ "@swc/jest": "^0.2.37", "@testing-library/jest-dom": "^6.6.3", "@types/jest": "^30.0.0", + "@types/node": "^25.9.1", "@typescript-eslint/eslint-plugin": "^8.28.0", "@typescript-eslint/parser": "^8.28.0", "eslint": "^9.39.4", @@ -50362,6 +50365,7 @@ "@walkeros/core": "4.8.1" }, "devDependencies": { + "@walkeros/collector": "4.8.1", "@walkeros/core": "4.8.1" } }, @@ -50477,6 +50481,7 @@ "@walkeros/web-core": "4.8.1" }, "devDependencies": { + "@types/gtag.js": "^0.0.20", "@walkeros/collector": "4.8.1" } }, diff --git a/package.json b/package.json index a66c19c92..03a7cc599 100644 --- a/package.json +++ b/package.json @@ -47,6 +47,7 @@ "test:integration": "turbo run test:integration --filter=@walkeros/cli --concurrency=1", "test:e2e": "turbo run test:e2e --filter=@walkeros/cli --concurrency=1", "test:smoke": "turbo run typecheck lint test test:integration --filter=@walkeros/core --filter=@walkeros/collector --filter=@walkeros/cli --affected --summarize", + "check:platform": "node --test packages/config/scripts/check-platform.test.mjs && node packages/config/scripts/check-platform.mjs", "verify:touched": "bash scripts/verify-touched.sh", "verify:affected": "turbo run typecheck lint test --affected --summarize", "verify:force": "turbo run test --affected --force --summarize", diff --git a/packages/cli/eslint.config.js b/packages/cli/eslint.config.js index 130636a4e..c08665536 100644 --- a/packages/cli/eslint.config.js +++ b/packages/cli/eslint.config.js @@ -1,4 +1,5 @@ import baseConfig from '@walkeros/config/eslint'; +import { nodePlatform } from '@walkeros/config/eslint/node'; // The base config's own no-restricted-syntax selectors. A flat config replaces // a rule set twice for one file rather than merging it, so every block below @@ -29,6 +30,7 @@ const devZones = ['src/dev.ts', 'src/examples/**', 'src/schemas/**']; export default [ ...baseConfig, + ...nodePlatform, { files: ['src/**/*.ts'], ignores: [...tests, ...devZones, 'src/core/app-paths.ts'], diff --git a/packages/cli/package.json b/packages/cli/package.json index a5f340a44..8faff6b4d 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -58,6 +58,7 @@ "@walkeros/server-core": "4.8.1", "@walkeros/server-destination-api": "4.8.1", "@walkeros/transformer-validate": "4.8.1", + "@walkeros/web-core": "4.8.1", "ajv": "^8.17.1", "chalk": "^5.6.2", "ci-info": "^4.4.0", diff --git a/packages/cli/src/__tests__/integration/bundle/cdn-jsdom.test.ts b/packages/cli/src/__tests__/integration/bundle/cdn-jsdom.test.ts index 86557f40f..3eaaaacc8 100644 --- a/packages/cli/src/__tests__/integration/bundle/cdn-jsdom.test.ts +++ b/packages/cli/src/__tests__/integration/bundle/cdn-jsdom.test.ts @@ -394,6 +394,45 @@ describe('CDN bundle without a browser source — window.elb', () => { }, 120000); }); +/** + * A browser flow whose only step is neutral brings no web package into its + * build, yet its entry imports the Tag Mode loader from `@walkeros/web-core`: + * the CLI resolves it from its own tree. + */ +describe('CDN bundle whose only step is neutral', () => { + const NEUTRAL_FLOW = withLocalPackages({ + version: 4, + flows: { + default: { + config: { platform: 'web' }, + sources: {}, + destinations: { demo: { package: '@walkeros/destination-demo' } }, + }, + }, + }); + + let tmpDir: string; + + beforeAll(async () => { + tmpDir = await mkdtemp(join(tmpdir(), 'walkeros-neutral-')); + }); + + afterAll(async () => { + await rm(tmpDir, { recursive: true, force: true }).catch(() => {}); + }); + + it('takes the Tag Mode loader from the CLI web-core', async () => { + const out = join(tmpDir, 'walker.js'); + await bundle(NEUTRAL_FLOW, { + target: 'cdn', + silent: true, + cache: false, + buildOverrides: { output: out }, + }); + expect(await readFile(out, 'utf8')).toContain('walkeros.moin'); + }, 120000); +}); + /** * The flow's `config.settings.windowCollector` names the one collector global, * whether the flow is built with `walkeros bundle` or from a build manifest. diff --git a/packages/cli/src/__tests__/integration/bundle/size-budget.test.ts b/packages/cli/src/__tests__/integration/bundle/size-budget.test.ts index 2de423d7a..829734c2b 100644 --- a/packages/cli/src/__tests__/integration/bundle/size-budget.test.ts +++ b/packages/cli/src/__tests__/integration/bundle/size-budget.test.ts @@ -35,7 +35,14 @@ const RAW_BUDGET_BYTES = 114_000; const GZIP_BUDGET_BYTES = 40_000; /** Strings only the observe recorder (connect, poster, call capture) carries. */ -const RECORDER_MARKERS = ['elbObserve', '/ingest/', 'X-Walkeros-Binding']; +const RECORDER_MARKERS = [ + 'credential binding mismatch', + '/ingest/', + 'X-Walkeros-Binding', +]; + +/** The page's Observe slot (web-core), imported only by an observed entry. */ +const SLOT_MARKER = 'elbObserve'; /** * One string per other build flag that only its guarded code carries: store @@ -103,6 +110,7 @@ describe('CDN bundle size budget', () => { it('cdn target ships no unused feature and no unresolved build flag', () => { for (const marker of [...RECORDER_MARKERS, ...FEATURE_MARKERS]) expect(cdn).not.toContain(marker); + expect(cdn).not.toContain(SLOT_MARKER); expect(cdn).not.toMatch(UNRESOLVED_FLAG); }); @@ -112,6 +120,7 @@ describe('CDN bundle size budget', () => { 'utf8', ); for (const marker of RECORDER_MARKERS) expect(text).toContain(marker); + expect(text).toContain(SLOT_MARKER); for (const marker of FEATURE_MARKERS) expect(text).not.toContain(marker); expect(text).not.toMatch(UNRESOLVED_FLAG); }, 120000); @@ -178,7 +187,7 @@ describe('CDN bundle size budget', () => { const plain = await readFile(plainPath, 'utf8'); const observed = await readFile(observedPath, 'utf8'); - for (const marker of RECORDER_MARKERS) { + for (const marker of [...RECORDER_MARKERS, SLOT_MARKER]) { expect(plain).not.toContain(marker); expect(observed).toContain(marker); } diff --git a/packages/cli/src/__tests__/unit/bundle/bundler-codegen.test.ts b/packages/cli/src/__tests__/unit/bundle/bundler-codegen.test.ts index d62016c67..b2878749d 100644 --- a/packages/cli/src/__tests__/unit/bundle/bundler-codegen.test.ts +++ b/packages/cli/src/__tests__/unit/bundle/bundler-codegen.test.ts @@ -9,11 +9,16 @@ jest.mock('../../../commands/bundle/bundler.js', () => { }; }); +import * as path from 'path'; +import * as os from 'os'; +import * as fs from 'fs-extra'; +import * as esbuild from 'esbuild'; import { applyCollectorDefaults, applyCollectorProvenance, buildDataPayload, buildSplitConfigObject, + cliPackagesPlugin, createEntryPoint, detectStepPackages, detectNamedImports, @@ -1137,7 +1142,7 @@ describe('generateWrapEntry platform gating', () => { describe('the Tag Mode loader call', () => { const DATA_PAYLOAD = '{"sources":{},"destinations":{}}'; - const LOADER_IMPORT = "import { moin } from '@walkeros/core';\n"; + const LOADER_IMPORT = "import { moin } from '@walkeros/web-core';\n"; const STAGE = { app: 'https://stage.app.walkeros.io', base: 'https://stage.cdn.walkeros.io/tag-mode/', @@ -1176,6 +1181,14 @@ describe('the Tag Mode loader call', () => { ); }); + it('generateWrapEntry imports the preview activator from web-core', () => { + const output = generateWrapEntry('./skeleton.mjs', { preview }); + expect(output).toContain( + "import { browserSwapActivator } from '@walkeros/web-core';\n", + ); + expect(output).not.toContain("from '@walkeros/core'"); + }); + it('generateWrapEntry calls the loader in a preview artifact', () => { const output = generateWrapEntry('./skeleton.mjs', { previewGrantTargets: ['api'], @@ -1223,6 +1236,56 @@ describe('the Tag Mode loader call', () => { }); }); +describe('cliPackagesPlugin', () => { + let tmpDir: string; + + beforeEach(async () => { + tmpDir = fs.realpathSync( + await fs.mkdtemp(path.join(os.tmpdir(), 'cli-packages-plugin-')), + ); + // A web-core beside the entry, where esbuild would look first. + const shadow = path.join(tmpDir, 'node_modules', '@walkeros', 'web-core'); + await fs.outputJson(path.join(shadow, 'package.json'), { + name: '@walkeros/web-core', + version: '0.0.1', + type: 'module', + exports: { '.': './index.mjs' }, + }); + await fs.writeFile( + path.join(shadow, 'index.mjs'), + 'export function moin() {}\n', + ); + }); + + afterEach(async () => { + await fs.remove(tmpDir).catch(() => {}); + }); + + it("resolves @walkeros/web-core from the CLI's own dir, not the entry's", async () => { + const result = await esbuild.build({ + stdin: { + contents: "import { moin } from '@walkeros/web-core';\nmoin();\n", + resolveDir: tmpDir, + }, + absWorkingDir: tmpDir, + bundle: true, + write: false, + metafile: true, + format: 'esm', + platform: 'browser', + logLevel: 'silent', + plugins: [cliPackagesPlugin()], + }); + const inputs = Object.keys(result.metafile.inputs).map((input) => + path.resolve(tmpDir, input), + ); + const cliWebCore = localPackageDir('@walkeros/web-core') + path.sep; + const besideEntry = path.join(tmpDir, 'node_modules') + path.sep; + expect(inputs.some((input) => input.startsWith(cliWebCore))).toBe(true); + expect(inputs.filter((input) => input.startsWith(besideEntry))).toEqual([]); + }); +}); + describe('generateServerEntry observer wiring', () => { it('iterates context.observers and installs them on collector.observers after startFlow', () => { const out = generateServerEntry('./skel.mjs', '{}'); diff --git a/packages/cli/src/__tests__/unit/bundle/preview-codegen.test.ts b/packages/cli/src/__tests__/unit/bundle/preview-codegen.test.ts index 801df3aa6..c4b11009b 100644 --- a/packages/cli/src/__tests__/unit/bundle/preview-codegen.test.ts +++ b/packages/cli/src/__tests__/unit/bundle/preview-codegen.test.ts @@ -23,9 +23,11 @@ describe('generateWrapEntry preview codegen', () => { expect(code).not.toContain('browserSwapActivator'); }); - it('imports the activator from core and short-circuits the production boot', () => { + it('imports the activator from web-core and short-circuits the production boot', () => { const code = generateWrapEntry('/tmp/stage1.js', { preview }); - expect(code).toContain("from '@walkeros/core'"); + expect(code).toContain( + "import { browserSwapActivator } from '@walkeros/web-core';", + ); expect(code).toContain('browserSwapActivator'); expect(code).toMatch( /if \(await browserSwapActivator\([\s\S]*?\)\) return;/, @@ -163,15 +165,19 @@ describe('generateWrapEntry preview-artifact observe connect', () => { observe, }); expect(code).not.toContain('browserSwapActivator'); - // The connect module is pure config; the artifact entry's only core - // import is the Tag Mode loader. - expect(code.match(/from '@walkeros\/core'/g)).toHaveLength(1); - expect(code).toContain("import { moin } from '@walkeros/core';"); + // The connect module is pure config; the artifact entry's only walkerOS + // imports are the Tag Mode loader and the page's Observe slot. + expect(code.match(/from '@walkeros\//g)).toHaveLength(2); + expect(code).toContain("import { moin } from '@walkeros/web-core';"); + expect(code).toContain( + "import { pageObserveSlot } from '@walkeros/web-core';", + ); }); it('bakes the connect module for a web-only artifact (no grant targets)', () => { const code = generateWrapEntry('/tmp/stage1.js', { observe }); expect(code).toContain('config.observe'); + expect(code).toContain('slot: pageObserveSlot() };'); expect(code).not.toContain('X-Walkeros-Preview'); expect(code).not.toMatch(/obsw_/); expect(code).not.toMatch(/Bearer\s+[A-Za-z0-9]/); diff --git a/packages/cli/src/__tests__/unit/bundle/wrap.test.ts b/packages/cli/src/__tests__/unit/bundle/wrap.test.ts index 8379f41e8..66872db3a 100644 --- a/packages/cli/src/__tests__/unit/bundle/wrap.test.ts +++ b/packages/cli/src/__tests__/unit/bundle/wrap.test.ts @@ -208,7 +208,7 @@ export const __configData = { test: true }; app: 'https://stage.app.walkeros.io', base: 'https://stage.cdn.walkeros.io/tag-mode/', }); - // The call keeps its literals; the minified core renames `moin`. + // The call keeps its literals; the minified web-core renames `moin`. expect(stage).toMatch( /\w+\(\{\s*app: "https:\/\/stage\.app\.walkeros\.io",\s*base: "https:\/\/stage\.cdn\.walkeros\.io\/tag-mode\/"\s*\}\)/, ); @@ -251,18 +251,21 @@ describe('wrapSkeleton core resolution', () => { beforeEach(async () => { tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'wrap-shadow-test-')); - // An older core above the wrap's temp dir, as a shared /tmp can hold one. - const shadow = path.join(tmpDir, 'node_modules', '@walkeros', 'core'); - await fs.outputJson(path.join(shadow, 'package.json'), { - name: '@walkeros/core', - version: '0.0.1', - type: 'module', - exports: { '.': './index.mjs' }, - }); - await fs.writeFile( - path.join(shadow, 'index.mjs'), - 'export const shadowCore = true;\n', - ); + // An older core and web-core above the wrap's temp dir, as a shared /tmp + // can hold them. + for (const name of ['core', 'web-core']) { + const shadow = path.join(tmpDir, 'node_modules', '@walkeros', name); + await fs.outputJson(path.join(shadow, 'package.json'), { + name: `@walkeros/${name}`, + version: '0.0.1', + type: 'module', + exports: { '.': './index.mjs' }, + }); + await fs.writeFile( + path.join(shadow, 'index.mjs'), + 'export const shadowCore = true;\n', + ); + } // The CLI's temp root is read once at load: reload it under tmpDir. jest .spyOn(jest.requireActual('os'), 'tmpdir') @@ -274,7 +277,7 @@ describe('wrapSkeleton core resolution', () => { await fs.remove(tmpDir).catch(() => {}); }); - it('imports the loader from the CLI core, never from a core above its temp dir', async () => { + it('imports the loader from the CLI web-core, never from a copy above its temp dir', async () => { const { wrapSkeleton: wrap } = await import('../../../commands/bundle/wrap.js'); const skeletonPath = path.join(tmpDir, 'skeleton.mjs'); diff --git a/packages/cli/src/commands/bundle/__tests__/observe-codegen.test.ts b/packages/cli/src/commands/bundle/__tests__/observe-codegen.test.ts index 19231c3fa..ea468e4cd 100644 --- a/packages/cli/src/commands/bundle/__tests__/observe-codegen.test.ts +++ b/packages/cli/src/commands/bundle/__tests__/observe-codegen.test.ts @@ -44,6 +44,18 @@ describe('generateWebEntry observe connect codegen', () => { expect(startFlowIdx).toBeGreaterThan(observeIdx); }); + it('passes the page slot beside the public pair, imported from web-core', () => { + const out = wrapWebEntryFor(flowWithObserve); + // The web arm reads its credential through the slot port; without it the + // collector logs an ERROR and Observe stays off. + expect(out).toContain( + "import { pageObserveSlot } from '@walkeros/web-core';", + ); + expect(out).toContain( + 'config.observe = { ...{"url":"https://obs.example","binding":"pb_x"}, slot: pageObserveSlot() };', + ); + }); + it('never emits a token or obsw_ secret literal into the bundle', () => { const out = wrapWebEntryFor(flowWithObserve); expect(out).not.toMatch(/obsw_/); diff --git a/packages/cli/src/commands/bundle/bundler.ts b/packages/cli/src/commands/bundle/bundler.ts index 997cbac51..2e523136d 100644 --- a/packages/cli/src/commands/bundle/bundler.ts +++ b/packages/cli/src/commands/bundle/bundler.ts @@ -699,9 +699,9 @@ export async function bundleCore( // `window` explicitly, so they still run. The browser entry has zero // exports, so no `globalName` is needed (one would add a window var). stage2Options.format = 'iife'; - // The entry's Tag Mode loader comes from the CLI's own core, never - // from the flow's core in TEMP_DIR, which may predate it. - stage2Options.plugins = [cliCorePlugin()]; + // The entry's Tag Mode loader comes from the CLI's own web-core, + // never from a copy in TEMP_DIR, which may predate it. + stage2Options.plugins = [cliPackagesPlugin()]; // Every build flag is defined from the flow's needs, so a feature the // flow does not use folds out (see build-flags.ts). stage2Options.define = { @@ -991,30 +991,34 @@ export function getNodeExternals(): string[] { } // Marks the plugin's own lookup, so it is not intercepted again. -const CLI_CORE = 'walkeros-cli-core'; +const CLI_PACKAGES = 'walkeros-cli-packages'; /** - * Resolves every `@walkeros/core` import of a generated entry (the Tag Mode - * loader, the preview activator) from this module's own location: the core - * this CLI was installed with. Resolving from the entry's temp dir would walk - * up through every `node_modules` above it, where another core (a flow's - * pinned one, or a stray install in a shared temp root) can shadow it. + * Resolves every `@walkeros/core` and `@walkeros/web-core` import of a + * generated entry (the Tag Mode loader, the preview activator) from this + * module's own location: the CLI's own walkerOS packages, the ones it was + * installed with. Resolving from the entry's temp dir would walk up through + * every `node_modules` above it, where another copy (a flow's pinned one, or a + * stray install in a shared temp root) can shadow it, or predate the loader. */ -export function cliCorePlugin(): esbuild.Plugin { +export function cliPackagesPlugin(): esbuild.Plugin { const resolveDir = path.dirname(fileURLToPath(import.meta.url)); return { - name: CLI_CORE, + name: CLI_PACKAGES, setup(build) { - build.onResolve({ filter: /^@walkeros\/core$/ }, async (args) => { - if (args.pluginData === CLI_CORE) return undefined; - const resolved = await build.resolve(args.path, { - kind: args.kind, - resolveDir, - pluginData: CLI_CORE, - }); - if (resolved.errors.length > 0) return { errors: resolved.errors }; - return { path: resolved.path, sideEffects: resolved.sideEffects }; - }); + build.onResolve( + { filter: /^@walkeros\/(core|web-core)$/ }, + async (args) => { + if (args.pluginData === CLI_PACKAGES) return undefined; + const resolved = await build.resolve(args.path, { + kind: args.kind, + resolveDir, + pluginData: CLI_PACKAGES, + }); + if (resolved.errors.length > 0) return { errors: resolved.errors }; + return { path: resolved.path, sideEffects: resolved.sideEffects }; + }, + ); }, }; } @@ -1999,13 +2003,19 @@ export function readObserveConnect( return undefined; } +/** Imports the page's Observe credential slot into a browser entry. */ +const OBSERVE_SLOT_IMPORT = + "import { pageObserveSlot } from '@walkeros/web-core';\n"; + /** * Emit the STATIC observe connect assignment for a generated entry: bake * ONLY the public connect values (`url` + `binding`, plus the optional - * `flowId`/`level`/`sample` scoping) onto the startFlow config. The runtime - * connect module inside startFlow does the credential/slot work at boot - * (the per-session secret arrives out-of-band via the `elbObserve` - * companion); no emitted byte here is ever secret. + * `flowId`/`level`/`sample` scoping) onto the startFlow config, beside the + * page slot the web arm reads its credential through. The runtime connect + * module inside startFlow does the credential work at boot (the per-session + * secret arrives out-of-band via the `elbObserve` companion); no emitted + * byte here is ever secret. The entry imports the slot with + * `OBSERVE_SLOT_IMPORT`. */ function emitObserveAssignment(observe: ObserveWeb): string { const literal: ObserveWeb = { @@ -2016,11 +2026,11 @@ function emitObserveAssignment(observe: ObserveWeb): string { ...(observe.sample !== undefined ? { sample: observe.sample } : {}), }; return ` - config.observe = ${JSON.stringify(literal)};`; + config.observe = { ...${JSON.stringify(literal)}, slot: pageObserveSlot() };`; } /** Imports the Tag Mode loader into a browser entry. */ -const MOIN_IMPORT = "import { moin } from '@walkeros/core';\n"; +const MOIN_IMPORT = "import { moin } from '@walkeros/web-core';\n"; /** * The Tag Mode loader call, the first statement of a browser entry: the @@ -2118,6 +2128,7 @@ export function generateWebEntry( const stage1Specifier = toFileImportSpecifier(stage1Path); + const observeImport = options.observe ? OBSERVE_SLOT_IMPORT : ''; const observeBlock = options.observe ? emitObserveAssignment(options.observe) : ''; @@ -2125,7 +2136,7 @@ export function generateWebEntry( const loaderImport = platform === 'browser' ? MOIN_IMPORT : ''; const loaderCall = platform === 'browser' ? emitMoinCall() : ''; - return `${loaderImport}import { startFlow, wireConfig } from '${stage1Specifier}'; + return `${loaderImport}${observeImport}import { startFlow, wireConfig } from '${stage1Specifier}'; const __configData = ${dataPayload}; @@ -2146,9 +2157,9 @@ const __configData = ${dataPayload}; */ /** * Preview activation wiring. When present and enabled, the generated entry - * imports the activator from core and lets it decide whether a preview bundle - * boots in place of this flow. The bundle carries only PUBLIC values: a keyring, - * an issuer, and opaque bindings. Never a project id, never a secret. + * imports the activator from web-core and lets it decide whether a preview + * bundle boots in place of this flow. The bundle carries only PUBLIC values: a + * keyring, an issuer, and opaque bindings. Never a project id, never a secret. */ export interface WrapEntryPreview { enabled: boolean; @@ -2238,7 +2249,7 @@ export function generateWrapEntry( // The activator is imported (not inlined) so its logic is tested as a module // and DCE keeps it out of every bundle that does not enable preview. const previewImport = previewEnabled - ? `import { browserSwapActivator } from '@walkeros/core';\n` + ? `import { browserSwapActivator } from '@walkeros/web-core';\n` : ''; // Preview-artifact variant: read the SESSION-FORWARDING grant (the @@ -2317,6 +2328,7 @@ ${grantTargets // STATIC observe connect wiring (see emitObserveAssignment): baked inside // the IIFE AFTER the activator's early return, so an activated page (the // artifact swapped in) never also installs the host's connect module. + const observeImport = options.observe ? OBSERVE_SLOT_IMPORT : ''; const observeBlock = options.observe ? emitObserveAssignment(options.observe) : ''; @@ -2326,7 +2338,7 @@ ${grantTargets const loaderImport = platform === 'browser' ? MOIN_IMPORT : ''; const loaderCall = platform === 'browser' ? emitMoinCall(options.moin) : ''; - return `${loaderImport}${previewImport}import { startFlow, wireConfig, __configData } from '${stage1Specifier}'; + return `${loaderImport}${previewImport}${observeImport}import { startFlow, wireConfig, __configData } from '${stage1Specifier}'; (async () => {${loaderCall}${previewBlock} const config = wireConfig(__configData);${envBlock}${previewGrantBlock}${observeBlock} diff --git a/packages/cli/src/commands/bundle/wrap.ts b/packages/cli/src/commands/bundle/wrap.ts index 397dde488..e3df50a96 100644 --- a/packages/cli/src/commands/bundle/wrap.ts +++ b/packages/cli/src/commands/bundle/wrap.ts @@ -20,7 +20,7 @@ import { fileURLToPath } from 'url'; import fs from 'fs-extra'; import * as esbuild from 'esbuild'; import { - cliCorePlugin, + cliPackagesPlugin, generateWrapEntry, generateWrapEntryServer, getNodeExternals, @@ -230,11 +230,12 @@ export async function wrapSkeleton( format: 'esm', platform, outfile: outputPath, - // Resolve `@walkeros/core` (used by the preview activator) from the CLI - // package's own dependency tree and the workspace root. Without this, - // esbuild looks under the temp entryDir which has no node_modules. - // The wrap step always runs from inside the CLI process; both the - // CLI's local node_modules and the workspace root are safe anchors. + // Resolve `@walkeros/web-core` (the Tag Mode loader and the preview + // activator) from the CLI package's own dependency tree and the + // workspace root. Without this, esbuild looks under the temp entryDir + // which has no node_modules. The wrap step always runs from inside the + // CLI process; both the CLI's local node_modules and the workspace root + // are safe anchors. nodePaths: getNodeResolutionPaths(), treeShaking: true, logLevel: 'error', @@ -258,10 +259,10 @@ export async function wrapSkeleton( // `window` explicitly, so they still run. The entry has zero exports, so // no `globalName` is needed. esbuildOptions.format = 'iife'; - // The entry's core imports (the Tag Mode loader, the preview - // activator) come from the CLI's own core, never from a core above the - // temp dir. - esbuildOptions.plugins = [cliCorePlugin()]; + // The entry's web-core imports (the Tag Mode loader, the preview + // activator) come from the CLI's own web-core, never from a copy above + // the temp dir. + esbuildOptions.plugins = [cliPackagesPlugin()]; // Build flags from the needs the skeleton carries. Only a baked observe // config adds to them: the recorder runs only for a collector that has // one. A skeleton without needs (an older CLI) keeps every flag on. diff --git a/packages/collector/eslint.config.mjs b/packages/collector/eslint.config.mjs new file mode 100644 index 000000000..ca1bbc80e --- /dev/null +++ b/packages/collector/eslint.config.mjs @@ -0,0 +1,14 @@ +import neutralConfig from '@walkeros/config/eslint/neutral'; +import { browserGlobals, nodeGlobals } from '@walkeros/config/eslint'; + +export default [ + ...neutralConfig, + { + // The kit's one replace(globalThis, 'fetch', ...): globalThis is allowed + // here, every other platform global stays banned. + files: ['src/dev/boundary.ts'], + rules: { + 'no-restricted-globals': ['error', ...browserGlobals, ...nodeGlobals], + }, + }, +]; diff --git a/packages/collector/src/__tests__/build-flags-bundle.test.ts b/packages/collector/src/__tests__/build-flags-bundle.test.ts index 4cbecfa63..0d0fb72c8 100644 --- a/packages/collector/src/__tests__/build-flags-bundle.test.ts +++ b/packages/collector/src/__tests__/build-flags-bundle.test.ts @@ -28,7 +28,7 @@ const FLAGS: BuildFlag[] = [ /** Strings only the code behind each flag carries. */ const MARKERS: Record = { __WALKEROS_OBSERVE__: [ - 'elbObserve', + 'credential binding mismatch', '/ingest/', 'X-Walkeros-Binding', 'new Proxy', diff --git a/packages/collector/src/__tests__/cache-codec-roundtrip.test.ts b/packages/collector/src/__tests__/cache-codec-roundtrip.test.ts index 48c2f47d4..aba4ddb1d 100644 --- a/packages/collector/src/__tests__/cache-codec-roundtrip.test.ts +++ b/packages/collector/src/__tests__/cache-codec-roundtrip.test.ts @@ -3,12 +3,14 @@ import { createMappingRoot, checkCache, compileCache, - serializeStoreValue, - deserializeStoreValue, storeCache, wrapCacheEnvelope, readCacheEnvelope, } from '@walkeros/core'; +import { + serializeStoreValue, + deserializeStoreValue, +} from '@walkeros/core/node'; import { createCacheStore } from '../cache-store'; import type { Store, Transformer, WalkerOS } from '@walkeros/core'; diff --git a/packages/collector/src/__tests__/observe-wiring.test.ts b/packages/collector/src/__tests__/observe-wiring.test.ts index 1957893fd..192734ac0 100644 --- a/packages/collector/src/__tests__/observe-wiring.test.ts +++ b/packages/collector/src/__tests__/observe-wiring.test.ts @@ -1,15 +1,22 @@ /** * startFlow observe wiring: server-arm direct attach, web-arm credential - * handshake (URL param -> localStorage slot -> zero-work), the observers - * escape hatch, and the advisory no-crash invariant. + * handshake through the `ObserveSlot` port (keep, self-heal, zero-work), the + * observers escape hatch, and the advisory no-crash invariant. The page slot + * itself is pinned in @walkeros/web-core's observeSlot tests. * * Env-injection only: the poster's global fetch is swapped via * Object.defineProperty (and restored), warns are captured via an injected - * logger handler, and the web credential arrives through the real jsdom URL - * (history.replaceState) or the localStorage slot. No jest.mock anywhere. + * logger handler, and the web credential arrives through a fake slot that + * records every keep and clear. No jest.mock anywhere. */ import { Level } from '@walkeros/core'; -import type { Logger, ObserverFn, PosterFetch } from '@walkeros/core'; +import type { + Collector, + Logger, + ObserveSlot, + ObserverFn, + PosterFetch, +} from '@walkeros/core'; import { startFlow } from '../flow'; const OBSERVER_URL = 'https://obs.example'; @@ -50,16 +57,29 @@ function warnCapture(): { warns: string[]; logger: Logger.Config } { return { warns, logger: { level: 'WARN', handler } }; } -function setUrl(search: string): void { - window.history.replaceState({}, '', `https://example.com/${search}`); +interface FakeSlot extends ObserveSlot { + kept: Array<[string, boolean]>; + cleared: string[]; } -describe('startFlow observe wiring', () => { - beforeEach(() => { - localStorage.clear(); - setUrl(''); - }); +/** An in-memory slot holding `found`, recording every keep and clear. */ +function fakeSlot(found: { raw: string; fromUrl: boolean } | null): FakeSlot { + const kept: Array<[string, boolean]> = []; + const cleared: string[] = []; + return { + kept, + cleared, + read: () => found, + keep: (raw, fromUrl) => { + kept.push([raw, fromUrl]); + }, + clear: (raw) => { + cleared.push(raw); + }, + }; +} +describe('startFlow observe wiring', () => { describe('server form', () => { it('installs an observer that posts the v1 envelope to url/ingest/sessionId', async () => { jest.useRealTimers(); @@ -135,29 +155,28 @@ describe('startFlow observe wiring', () => { }); describe('web form', () => { - it('installs nothing with no URL param and no stored slot (zero-work)', async () => { - const setItem = jest.spyOn(Storage.prototype, 'setItem'); + it('installs nothing and never touches the slot when it holds no credential (zero-work)', async () => { + const slot = fakeSlot(null); const { collector } = await startFlow({ - observe: { url: OBSERVER_URL, binding: 'pb_x' }, + observe: { url: OBSERVER_URL, binding: 'pb_x', slot }, }); expect(collector.observers.size).toBe(0); - expect(setItem).not.toHaveBeenCalled(); - setItem.mockRestore(); + expect(slot.kept).toEqual([]); + expect(slot.cleared).toEqual([]); }); - it('reads the URL param, persists the slot, and posts with binding header + credential secret', async () => { + it('keeps a URL credential as URL-borne and posts with binding header + credential secret', async () => { jest.useRealTimers(); - setUrl('?elbObserve=obsw_pb_x.ses_9.sec_1'); + const slot = fakeSlot({ raw: 'obsw_pb_x.ses_9.sec_1', fromUrl: true }); const posts: CapturedPost[] = []; const restore = stubFetch(posts); try { const { collector, elb } = await startFlow({ - observe: { url: OBSERVER_URL, binding: 'pb_x' }, + observe: { url: OBSERVER_URL, binding: 'pb_x', slot }, }); expect(collector.observers.size).toBe(1); - expect(localStorage.getItem('elbObserve')).toBe( - 'obsw_pb_x.ses_9.sec_1', - ); + // keep(raw, true): the slot persists it and strips the address bar. + expect(slot.kept).toEqual([['obsw_pb_x.ses_9.sec_1', true]]); await elb('page view'); await new Promise((resolve) => setTimeout(resolve, 120)); @@ -172,36 +191,23 @@ describe('startFlow observe wiring', () => { } }); - it('attaches from the stored slot when no URL param is present', async () => { - setUrl('?keep=1'); - localStorage.setItem('elbObserve', 'obsw_pb_x.ses_9.sec_1'); + it('keeps a stored credential as stored, so the address bar is never touched', async () => { + const slot = fakeSlot({ raw: 'obsw_pb_x.ses_9.sec_1', fromUrl: false }); const { collector } = await startFlow({ - observe: { url: OBSERVER_URL, binding: 'pb_x' }, + observe: { url: OBSERVER_URL, binding: 'pb_x', slot }, }); expect(collector.observers.size).toBe(1); - // Slot-sourced attach never touches the address bar. - expect(window.location.search).toBe('?keep=1'); - }); - - it('strips elbObserve from the address bar after a URL-param attach, keeping innocent params', async () => { - setUrl('?foo=1&elbObserve=obsw_pb_x.ses_9.sec_1&bar=2'); - const { collector } = await startFlow({ - observe: { url: OBSERVER_URL, binding: 'pb_x' }, - }); - expect(collector.observers.size).toBe(1); - expect(window.location.search).not.toContain('elbObserve'); - expect(window.location.search).toContain('foo=1'); - expect(window.location.search).toContain('bar=2'); + expect(slot.kept).toEqual([['obsw_pb_x.ses_9.sec_1', false]]); }); it('clears the slot and stops posting when the observer answers 401', async () => { jest.useRealTimers(); - localStorage.setItem('elbObserve', 'obsw_pb_x.ses_9.sec_1'); + const slot = fakeSlot({ raw: 'obsw_pb_x.ses_9.sec_1', fromUrl: false }); const posts: CapturedPost[] = []; const restore = stubFetch(posts, { ok: false, status: 401 }); try { const { collector, elb } = await startFlow({ - observe: { url: OBSERVER_URL, binding: 'pb_x' }, + observe: { url: OBSERVER_URL, binding: 'pb_x', slot }, }); expect(collector.observers.size).toBe(1); @@ -209,7 +215,7 @@ describe('startFlow observe wiring', () => { await new Promise((resolve) => setTimeout(resolve, 120)); expect(posts.length).toBeGreaterThan(0); - expect(localStorage.getItem('elbObserve')).toBeNull(); + expect(slot.cleared).toEqual(['obsw_pb_x.ses_9.sec_1']); expect(collector.observers.size).toBe(0); // Detached: further events never reach the poster. @@ -224,53 +230,73 @@ describe('startFlow observe wiring', () => { it('keeps the slot and keeps observing on a non-401 status', async () => { jest.useRealTimers(); - localStorage.setItem('elbObserve', 'obsw_pb_x.ses_9.sec_1'); + const slot = fakeSlot({ raw: 'obsw_pb_x.ses_9.sec_1', fromUrl: false }); const posts: CapturedPost[] = []; const restore = stubFetch(posts, { ok: false, status: 500 }); try { const { collector, elb } = await startFlow({ - observe: { url: OBSERVER_URL, binding: 'pb_x' }, + observe: { url: OBSERVER_URL, binding: 'pb_x', slot }, }); await elb('page view'); await new Promise((resolve) => setTimeout(resolve, 120)); expect(posts.length).toBeGreaterThan(0); - expect(localStorage.getItem('elbObserve')).toBe( - 'obsw_pb_x.ses_9.sec_1', - ); + expect(slot.cleared).toEqual([]); expect(collector.observers.size).toBe(1); } finally { restore(); } }); - it('warns and installs nothing when the credential pb mismatches the binding', async () => { - setUrl('?elbObserve=obsw_pb_other.ses_9.sec_1'); + it('warns, installs nothing and never touches the slot when the credential pb mismatches the binding', async () => { + const slot = fakeSlot({ + raw: 'obsw_pb_other.ses_9.sec_1', + fromUrl: true, + }); const { warns, logger } = warnCapture(); const { collector } = await startFlow({ logger, - observe: { url: OBSERVER_URL, binding: 'pb_x' }, + observe: { url: OBSERVER_URL, binding: 'pb_x', slot }, }); expect(collector.observers.size).toBe(0); expect(warns.some((m) => m.includes('binding'))).toBe(true); - expect(localStorage.getItem('elbObserve')).toBeNull(); + expect(slot.kept).toEqual([]); + expect(slot.cleared).toEqual([]); }); - it('warns and installs nothing on a malformed credential', async () => { - setUrl('?elbObserve=garbage'); + it('warns and never touches the slot on a malformed URL credential', async () => { + // URL-sourced garbage never touches stored state (anti-griefing). + const slot = fakeSlot({ raw: 'garbage', fromUrl: true }); const { warns, logger } = warnCapture(); const { collector } = await startFlow({ logger, - observe: { url: OBSERVER_URL, binding: 'pb_x' }, + observe: { url: OBSERVER_URL, binding: 'pb_x', slot }, }); expect(collector.observers.size).toBe(0); - expect(warns.length).toBeGreaterThan(0); + expect(warns).toEqual(['observe: malformed credential ignored']); + expect(slot.kept).toEqual([]); + expect(slot.cleared).toEqual([]); + }); + + it('warns and self-heals a malformed STORED credential', async () => { + const slot = fakeSlot({ raw: 'garbage', fromUrl: false }); + const { warns, logger } = warnCapture(); + const { collector } = await startFlow({ + logger, + observe: { url: OBSERVER_URL, binding: 'pb_x', slot }, + }); + expect(collector.observers.size).toBe(0); + expect(warns).toEqual(['observe: malformed stored credential cleared']); + // Stored state clears on its own failure, exactly that value: the next + // pageview is back on the silent zero-work path. + expect(slot.cleared).toEqual(['garbage']); + expect(slot.kept).toEqual([]); }); it('honors a baked flowId and trace level from the connect config', async () => { jest.useRealTimers(); - localStorage.setItem('elbObserve', 'obsw_pb_x.ses_9.sec_1'); + const slot = fakeSlot({ raw: 'obsw_pb_x.ses_9.sec_1', fromUrl: false }); const posts: CapturedPost[] = []; const restore = stubFetch(posts); try { @@ -281,6 +307,7 @@ describe('startFlow observe wiring', () => { binding: 'pb_x', flowId: 'flow_db_1', level: 'trace', + slot, }, }); // The baked level reaches the collector-wide supplier so destination @@ -310,31 +337,82 @@ describe('startFlow observe wiring', () => { }); it('defaults to the flow name, standard level, and no observeLevel supplier', async () => { - localStorage.setItem('elbObserve', 'obsw_pb_x.ses_9.sec_1'); + const slot = fakeSlot({ raw: 'obsw_pb_x.ses_9.sec_1', fromUrl: false }); const { collector } = await startFlow({ name: 'myflow', - observe: { url: OBSERVER_URL, binding: 'pb_x' }, + observe: { url: OBSERVER_URL, binding: 'pb_x', slot }, }); expect(collector.observers.size).toBe(1); // No baked level: the collector-wide capture supplier stays unset. expect(collector.observeLevel).toBeUndefined(); }); - it('warns and self-heals a malformed STORED slot value', async () => { - localStorage.setItem('elbObserve', 'garbage'); - const { warns, logger } = warnCapture(); - const { collector } = await startFlow({ - logger, - observe: { url: OBSERVER_URL, binding: 'pb_x' }, + it('a web observe without slot logs one ERROR line, and startFlow resolves', async () => { + // A JSON caller cannot carry the slot: the config arrives as parsed + // JSON, the shape a JS or JSON caller hands over. + const config: Collector.InitConfig = JSON.parse( + JSON.stringify({ observe: { url: OBSERVER_URL, binding: 'pb_x' } }), + ); + const logs: Array<{ level: Level; message: string }> = []; + // No level: the line must be visible at the default level. + const handler: Logger.Handler = (level, message) => { + logs.push({ level, message }); + }; + const { collector, elb } = await startFlow({ + ...config, + logger: { handler }, }); + expect(logs).toEqual([ + { + level: Level.ERROR, + message: + 'observe: a web observe config needs slot: pageObserveSlot() from @walkeros/web-core', + }, + ]); + // Observe stays off; the flow runs. expect(collector.observers.size).toBe(0); - expect(warns.length).toBeGreaterThan(0); - // Stored state clears on its own failure -- the next pageview is - // back on the silent zero-work path. - expect(localStorage.getItem('elbObserve')).toBeNull(); + await expect(elb('page view')).resolves.toBeDefined(); }); }); + describe('a throwing slot', () => { + const failing = (): never => { + throw new Error('boom'); + }; + it.each<[string, ObserveSlot]>([ + ['read', { read: failing, keep: () => {}, clear: () => {} }], + [ + 'keep', + { + read: () => ({ raw: 'obsw_pb_x.ses_9.sec_1', fromUrl: false }), + keep: failing, + clear: () => {}, + }, + ], + ])( + 'a throwing %s logs one ERROR line, and startFlow resolves', + async (_, slot) => { + const logs: Array<{ level: Level; message: string }> = []; + const handler: Logger.Handler = (level, message) => { + logs.push({ level, message }); + }; + const { collector, elb } = await startFlow({ + logger: { handler }, + observe: { url: OBSERVER_URL, binding: 'pb_x', slot }, + }); + expect(logs).toEqual([ + { + level: Level.ERROR, + message: 'observe: the slot failed, Observe is off: boom', + }, + ]); + // Observe stays off; the flow runs. + expect(collector.observers.size).toBe(0); + await expect(elb('page view')).resolves.toBeDefined(); + }, + ); + }); + describe('observers escape hatch', () => { it('installs caller-supplied observers into the advisory set', async () => { const spy: ObserverFn = () => {}; diff --git a/packages/collector/src/__tests__/store-cache-wrapper.test.ts b/packages/collector/src/__tests__/store-cache-wrapper.test.ts index 6a57af71b..f7830dcb3 100644 --- a/packages/collector/src/__tests__/store-cache-wrapper.test.ts +++ b/packages/collector/src/__tests__/store-cache-wrapper.test.ts @@ -1,5 +1,8 @@ import type { Cache, Logger, Store } from '@walkeros/core'; -import { serializeStoreValue, deserializeStoreValue } from '@walkeros/core'; +import { + serializeStoreValue, + deserializeStoreValue, +} from '@walkeros/core/node'; import { createCacheStore } from '../cache-store'; import { wrapStoreWithCache } from '../store-cache-wrapper'; diff --git a/packages/collector/src/dev/__tests__/conformance.test.ts b/packages/collector/src/dev/__tests__/conformance.test.ts index a821c8abf..0cf9398ab 100644 --- a/packages/collector/src/dev/__tests__/conformance.test.ts +++ b/packages/collector/src/dev/__tests__/conformance.test.ts @@ -1,8 +1,8 @@ import type { Flow, Store, Trigger, WalkerOS } from '@walkeros/core'; import { isObject } from '@walkeros/core'; import { startFlow } from '../../flow'; -import { assertProfileEnvironment, conformance } from '../conformance'; -import { fixtures } from './fixtures'; +import { conformance } from '../conformance'; +import { blankPage, fixtures } from './fixtures'; import { batching, batchingByDefault, @@ -32,9 +32,6 @@ import { consentAware, consentIgnoring, initCalling, - pageScriptIgnoring, - pageScriptVendor, - scriptVendor, swallowingSdk, vendorGlobal, } from './fixture-package/src/vendor'; @@ -239,19 +236,6 @@ describe('conformance kit', () => { }); }); - it('refuses a browser profile without a document, and nothing else', () => { - expect(() => assertProfileEnvironment('web-sdk', false)).toThrow( - /profile web-sdk is for web steps and needs a document/, - ); - expect(() => assertProfileEnvironment('vendor-global', false)).toThrow( - /profile vendor-global is for web steps and needs a document/, - ); - expect(() => assertProfileEnvironment('sdk-env', false)).not.toThrow(); - expect(() => assertProfileEnvironment('http', false)).not.toThrow(); - expect(() => assertProfileEnvironment('web-sdk', true)).not.toThrow(); - expect(() => assertProfileEnvironment('vendor-global')).not.toThrow(); - }); - it('refuses fake timers with a clear message', async () => { jest.useFakeTimers(); await expect( @@ -522,8 +506,26 @@ describe('conformance kit, vendor-global profile', () => { simulation: ['call:vendor.track', 'call:vendor.consent'], }, examples, + page: blankPage, }; + it('refuses a browser profile without a page', async () => { + const { page, ...withoutPage } = options; + await expect( + conformance('destination', vendorGlobal, withoutPage), + ).rejects.toThrow( + 'profile vendor-global runs on a page: pass page from @walkeros/web-core/dev', + ); + await expect( + conformance('destination', vendorGlobal, { + ...withoutPage, + profile: 'web-sdk', + }), + ).rejects.toThrow( + 'profile web-sdk runs on a page: pass page from @walkeros/web-core/dev', + ); + }); + it('passes a vendor destination that passes consent to the vendor', async () => { await expect( conformance('destination', vendorGlobal, options), @@ -620,52 +622,6 @@ describe('conformance kit, vendor-global profile', () => { }); }); - it('counts item 2 per cause: a vendor-timeout line before the script error does not count', async () => { - await expect( - conformance('destination', scriptVendor, { - ...options, - schema: { type: 'object', properties: {} }, - settings: {}, - examples: { - pageView: { in: event }, - silent: examples.silent, - revoke: examples.revoke, - }, - }), - ).resolves.toBeUndefined(); - }); - - describe('a script injected once per page, loadScript on by default (item 2)', () => { - // No declared failure example: item 2 must reach the script error itself. - const { unreachable, ...scriptExamples } = examples; - const scriptOptions = { ...options, examples: scriptExamples }; - - it('reaches the script error although item 1 injected the src first', async () => { - await expect( - conformance('destination', pageScriptVendor, scriptOptions), - ).resolves.toBeUndefined(); - }); - - it('reaches the script error when the examples env carries its own document', async () => { - const own = document.implementation.createHTMLDocument('env'); - await expect( - conformance('destination', pageScriptVendor, { - ...scriptOptions, - env: { ...scriptOptions.env, push: { vendor, document: own } }, - }), - ).resolves.toBeUndefined(); - }); - - it('fails item 2 when the push ignores the script error', async () => { - await expect( - conformance('destination', pageScriptIgnoring, scriptOptions), - ).rejects.toMatchObject({ - items: [2], - message: expect.stringContaining('is not reported vendor-not-loaded'), - }); - }); - }); - describe('item 8 runs a revoke gated on its keys, granted first (I1)', () => { it('passes a destination that passes the revocation to its vendor', async () => { await expect( diff --git a/packages/collector/src/dev/__tests__/fixture-package/src/vendor.ts b/packages/collector/src/dev/__tests__/fixture-package/src/vendor.ts index a125abf5d..743007cba 100644 --- a/packages/collector/src/dev/__tests__/fixture-package/src/vendor.ts +++ b/packages/collector/src/dev/__tests__/fixture-package/src/vendor.ts @@ -1,5 +1,4 @@ import { isObject, StepError } from '@walkeros/core'; -import type { Collector } from '@walkeros/core'; import type { Destination } from '@walkeros/core'; export interface Vendor { @@ -8,11 +7,7 @@ export interface Vendor { consent(granted: boolean): void; } -type Types = Destination.Types< - { site?: string }, - unknown, - { vendor?: Vendor; document?: Document } ->; +type Types = Destination.Types<{ site?: string }, unknown, { vendor?: Vendor }>; function isVendor(value: unknown): value is Vendor { return ( @@ -113,93 +108,6 @@ export const consentIgnoring: Destination.Instance = { on: undefined, }; -/** Whether a page's vendor script fired `error`, per collector (one page). */ -const scriptFailed = new WeakMap(); - -/** - * Loads its vendor script as loadVendor does (marked, awaited): a push - * waits for the script and fails `vendor-timeout` at a short deadline (jsdom - * never loads it), and once the script fired `error` every push fails - * `vendor-not-loaded`: two causes, one line each. - */ -export const scriptVendor: Destination.Instance = { - type: 'script-vendor', - config: {}, - init: ({ collector }) => { - const script = document.createElement('script'); - script.src = 'https://vendor.example/sdk.js'; - script.setAttribute('data-walkeros-vendor', ''); - script.addEventListener('error', () => scriptFailed.set(collector, true)); - document.head.appendChild(script); - }, - push: async (_, { collector, rule }) => { - if (rule?.silent) return; - if (scriptFailed.get(collector)) - throw new StepError('Vendor script failed to load', { - code: 'vendor-not-loaded', - }); - await new Promise((__, reject) => - setTimeout( - () => - reject( - new StepError('Vendor script did not load within 50ms', { - code: 'vendor-timeout', - }), - ), - 50, - ), - ); - }, - on: (type, { env, data }) => { - const vendor = env.vendor; - if (type !== 'consent' || !isVendor(vendor) || !isObject(data)) return; - vendor.consent(data.marketing === true); - }, -}; - -/** A script's error, kept with its element (the page is the registry). */ -const failedScripts = new WeakSet(); - -/** - * Injects its marked script once per page, as loadVendor does (a marked - * script with its src already on the page is the injected one), into the - * env's document where it has one, with `loadScript` on by default. A push - * calls the stub synchronously and, with `reports`, fails - * `vendor-not-loaded` once the script fired `error`. - */ -function pageScript(reports: boolean): Destination.Instance { - const src = 'https://vendor.example/page.js'; - const scriptOnPage = (doc: Document) => - Array.from(doc.querySelectorAll('script[data-walkeros-vendor]')).find( - (script) => script.getAttribute('src') === src, - ); - return { - ...vendorGlobal, - type: reports ? 'page-script' : 'page-script-ignoring', - config: { loadScript: true }, - init: ({ config, env }) => { - const doc = env.document ?? document; - if (!config.loadScript || scriptOnPage(doc)) return; - const script = doc.createElement('script'); - script.src = src; - script.setAttribute('data-walkeros-vendor', ''); - script.addEventListener('error', () => failedScripts.add(script)); - doc.head.appendChild(script); - }, - push: async (event, context) => { - const script = scriptOnPage(context.env.document ?? document); - if (reports && script && failedScripts.has(script)) - throw new StepError('Vendor script failed to load', { - code: 'vendor-not-loaded', - }); - await vendorGlobal.push(event, context); - }, - }; -} - -export const pageScriptVendor = pageScript(true); -export const pageScriptIgnoring = pageScript(false); - /** Sends "revoked" to its vendor on every consent change, grant included. */ export const alwaysRevoking: Destination.Instance = { ...vendorGlobal, diff --git a/packages/collector/src/dev/__tests__/fixtures.ts b/packages/collector/src/dev/__tests__/fixtures.ts index 4efb1230c..a08e7ce67 100644 --- a/packages/collector/src/dev/__tests__/fixtures.ts +++ b/packages/collector/src/dev/__tests__/fixtures.ts @@ -29,6 +29,20 @@ export const examples: Flow.StepExamples = { }, }; +/** + * A page without a DOM for the browser profiles: the vendor fixtures call + * their vendor only, never a page transport or script. The real page is + * `page` from `@walkeros/web-core/dev`, whose tests run it. + */ +export const blankPage: Flow.StepPage = { + open: () => ({ + globals: {}, + scripts: () => ({ marked: 0, unmarked: 0 }), + failScripts: () => undefined, + close: () => undefined, + }), +}; + export const fixtures = { schema, examples, diff --git a/packages/collector/src/dev/boundary.ts b/packages/collector/src/dev/boundary.ts index e4d6ca3dc..fe08abcb7 100644 --- a/packages/collector/src/dev/boundary.ts +++ b/packages/collector/src/dev/boundary.ts @@ -3,25 +3,14 @@ import { isObject, isThenable, parseCallPath } from '@walkeros/core'; /** * The network boundary the runner and the conformance kit control: the - * global `fetch`, `navigator.sendBeacon` and `XMLHttpRequest`, for web and - * server alike (`sendServer` and `sendWeb` run on them). Each is replaced for - * the duration of one run and restored after it. Every request is answered - * by the boundary and recorded, so a request that never shows up here - * escaped it (its own `https.request`). + * global `fetch`, for web and server alike (`sendServer` runs on it), and on + * a web step's page (`Flow.StepPage`, from `@walkeros/web-core/dev`) the + * page's own transports, which answer through `record`. Each is replaced + * for the duration of one run and restored after it. Every request is + * answered by the boundary and recorded, so a request that never shows up + * here escaped it (its own `https.request`). */ -/** How the boundary answers one request. */ -export interface Answer { - status?: number; - body?: unknown; - /** The status text, which failure answers use to echo a canary. */ - statusText?: string; - /** A network error, or no answer at all. */ - error?: 'network' | 'timeout'; - /** Answer one macrotask late (the floating-send check). */ - late?: boolean; -} - /** One request the boundary answered. */ export interface Request { effect: Flow.StepEffect; @@ -34,12 +23,14 @@ export interface Boundary { requests: Request[]; /** The phase recorded with each request; the kit sets `init` during init. */ phase: string; + /** The page the step runs on, opened on this boundary's `record`. */ + page?: Flow.StepPageSession; /** Requests whose answer is still outstanding. */ pending(): number; restore(): void; } -type Answers = (index: number) => Answer; +type Answers = (index: number) => Flow.StepAnswer; type BodyMethod = ReturnType< Extract unknown> @@ -49,7 +40,7 @@ type BodyMethod = ReturnType< class KitResponse implements Response { readonly headers = new Headers(); readonly redirected = false; - readonly type: ResponseType = 'basic'; + readonly type: Response['type'] = 'basic'; readonly url = ''; readonly body = null; readonly bodyUsed = false; @@ -131,10 +122,17 @@ function replace(target: object, key: string, value: unknown): () => void { }; } -/** Install the boundary; `answers(n)` answers the n-th request (0-based). */ -export function installBoundary(answers: Answers = () => ({})): Boundary { +/** + * Install the boundary; `answers(n)` answers the n-th request (0-based). With + * a page, the step's page is opened on it and closed at restore. + */ +export function installBoundary( + answers: Answers = () => ({}), + page?: Flow.StepPage, +): Boundary { const requests: Request[] = []; const closers: Array<() => void> = []; + const restores: Array<() => void> = []; const boundary: Boundary = { requests, phase: 'push', @@ -145,19 +143,29 @@ export function installBoundary(answers: Answers = () => ({})): Boundary { }, }; - const record = (effect: Flow.StepEffect): [Request, Answer] => { + const track = (effect: Flow.StepEffect): [Request, Flow.StepAnswer] => { const request: Request = { effect, phase: boundary.phase, settled: false }; const answer = answers(requests.length); requests.push(request); return [request, answer]; }; + const record: Flow.StepRecord = (effect) => { + const [request, answer] = track(effect); + return { + answer, + settle: () => { + request.settled = true; + }, + }; + }; + const kitFetch = ( input: string | URL | { url: string }, init?: RequestInit, ): Promise => { const initRecord = recordedInit(init); - const [request, answer] = record( + const [request, answer] = track( initRecord ? ['fetch', urlOf(input), initRecord] : ['fetch', urlOf(input)], @@ -195,69 +203,14 @@ export function installBoundary(answers: Answers = () => ({})): Boundary { }); }; - const kitBeacon = (url: string, data?: unknown): boolean => { - const [request, answer] = record(['sendBeacon', url, data]); - request.settled = true; - return !answer.error && (answer.status ?? 200) < 400; - }; - - // Synchronous by default, as sendWeb opens it; the async form answers late. - class KitXMLHttpRequest { - status = 0; - statusText = ''; - response: unknown = ''; - responseText = ''; - readyState = 0; - onload: (() => void) | null = null; - onerror: (() => void) | null = null; - onreadystatechange: (() => void) | null = null; - private method = 'GET'; - private url = ''; - private async = true; - - open(method: string, url: string, async = true) { - this.method = method; - this.url = url; - this.async = async; - this.readyState = 1; - } - setRequestHeader() {} - addEventListener(type: string, listener: () => void) { - if (type === 'load') this.onload = listener; - if (type === 'error') this.onerror = listener; - } - send(body?: unknown) { - const [request, answer] = record([ - 'XMLHttpRequest', - this.method, - this.url, - body, - ]); - const settle = () => { - request.settled = true; - this.readyState = 4; - if (answer.error) { - if (!this.async) throw new Error('NetworkError'); - this.onerror?.(); - return; - } - this.status = answer.status ?? 200; - this.statusText = answer.statusText ?? ''; - this.responseText = bodyText(answer.body); - this.response = this.responseText; - this.onreadystatechange?.(); - this.onload?.(); - }; - if (this.async) macrotask().then(settle); - else settle(); - } + // Opened first: a page that cannot open leaves nothing replaced. + const opened = page?.open(record); + restores.push(replace(globalThis, 'fetch', kitFetch)); + if (opened) { + boundary.page = opened; + restores.push(() => opened.close()); } - const restores = [replace(globalThis, 'fetch', kitFetch)]; - restores.push(replace(globalThis, 'XMLHttpRequest', KitXMLHttpRequest)); - if (typeof navigator !== 'undefined') - restores.push(replace(navigator, 'sendBeacon', kitBeacon)); - return boundary; } diff --git a/packages/collector/src/dev/conformance.ts b/packages/collector/src/dev/conformance.ts index 4de7c8194..3a453cdee 100644 --- a/packages/collector/src/dev/conformance.ts +++ b/packages/collector/src/dev/conformance.ts @@ -10,7 +10,7 @@ import type { } from '@walkeros/core'; import type { JSONSchema } from '@walkeros/core/dev'; import { getEvent, isObject, Level, StepError, stepId } from '@walkeros/core'; -import type { Answer, EnvAnswer } from './boundary'; +import type { EnvAnswer } from './boundary'; import type { ExamplesEnv, LogLine, @@ -25,6 +25,7 @@ import { isEvent, isStoreOperation, mappingOf, + onPage, openDestination, openSource, openStore, @@ -39,7 +40,6 @@ import { sourceConfig, STEP_ID, timersAreFake, - VENDOR_SCRIPT_MARKER, } from './runExamples'; /** @@ -77,6 +77,11 @@ export interface ConformanceOptions { passThrough?: string[]; /** Sources: the package's `createTrigger`, which feeds the examples. */ createTrigger?: Trigger.CreateFn; + /** + * The page a web step runs on: `page` from `@walkeros/web-core/dev`. The + * browser profiles (`web-sdk`, `vendor-global`) require it. + */ + page?: Flow.StepPage; } /** Every failed check of one conformance run: `items` lists their numbers. */ @@ -414,7 +419,7 @@ function pureJson(settings: Record): boolean { } /** The failures item 2 answers at the fetch boundary, one run each. */ -const failureAnswers: Array<{ label: string; answer: Answer }> = [ +const failureAnswers: Array<{ label: string; answer: Flow.StepAnswer }> = [ { label: 'HTTP 500', answer: { status: 500, body: { error: 'busy' } } }, { label: 'a network error', answer: { error: 'network' } }, { label: 'a timeout', answer: { error: 'timeout' } }, @@ -456,6 +461,7 @@ async function runDestination( const mapping = mappingOf(target.example, { event: target.event }); const session = await openDestination(step, { env: ctx.runEnv, + page: ctx.options.page, ...setup, config: { timeout: RUN_TIMEOUT, @@ -660,6 +666,7 @@ async function destinationItem( for (const problem of await checkDestinationExample(step, example, { settings: options.settings, env: options.env, + page: options.page, })) problems.push(`example ${name}: ${problem}`); } @@ -684,6 +691,7 @@ async function destinationItem( checkDestinationExample(step, example, { settings: options.settings, env: ctx.runEnv ? options.env : undefined, + page: options.page, }), ); @@ -707,6 +715,7 @@ async function destinationItem( for (const problem of await checkDestinationExample(step, example, { settings: options.settings, env: ctx.runEnv ? options.env : undefined, + page: options.page, })) problems.push(`example ${name}: ${problem}`); } @@ -731,6 +740,7 @@ async function declaredFailures( for (const problem of await checkDestinationExample(step, example, { settings: ctx.options.settings, env: ctx.options.env, + page: ctx.options.page, })) problems.push(`example ${name}: ${problem}`); return problems.length ? problems.join(', ') : undefined; @@ -745,9 +755,11 @@ async function vendorFailureProblem( const mapping = mappingOf(primary.example, { event: primary.event }); const session = await openDestination(step, { settings: ctx.options.settings, + env: ctx.env, + page: ctx.options.page, // The page's document replaces an env one, so the script lands where // the kit fires its `error`. - env: ctx.env && { ...ctx.env, push: { ...ctx.env.push, document } }, + pageGlobals: true, config: { timeout: RUN_TIMEOUT, loadScript: true, @@ -758,9 +770,7 @@ async function vendorFailureProblem( await session.run({ event: primary.event }); if (!session.scripts().marked) return declaredFailures(step, ctx); - document - .querySelectorAll(`script[${VENDOR_SCRIPT_MARKER}]`) - .forEach((script) => script.dispatchEvent(new Event('error'))); + session.boundary.page?.failScripts(); const run = await session.run({ event: primary.event }); const reason = run.result.failed?.[STEP_ID]?.reason; if (reason?.code !== 'vendor-not-loaded') @@ -787,6 +797,7 @@ async function batchProblem( const session = await openDestination(step, { settings: ctx.options.settings, env: ctx.runEnv, + page: ctx.options.page, config: { timeout: RUN_TIMEOUT, batch: { size: 1000, wait: 60_000 }, @@ -1359,6 +1370,7 @@ async function sourceItem( const start = async (setup: Setup) => { const thrown: unknown[] = []; const session = await openSource(step, { + page: options.page, ...setup, initError: (error) => thrown.push(error), }).catch((error: unknown) => { @@ -1402,7 +1414,8 @@ async function sourceItem( } case 7: { - if (!options.createTrigger) + const { createTrigger } = options; + if (!createTrigger) return 'sources run their examples through the package createTrigger: pass it'; const found: string[] = []; let planted = 0; @@ -1411,32 +1424,34 @@ async function sourceItem( if (!hasCanary(input)) continue; planted++; const logs: LogLine[] = []; - const instance = await options.createTrigger( - sourceConfig( - step, - { env: options.env, settings: options.settings }, - [], - logs, - ), - ); - try { - await instance - .trigger( - example.trigger?.type, - example.trigger?.options, - )(input) - .then( - (result) => { - if (hasCanary(isObject(result) ? result.error : undefined)) - found.push('the answered error'); - }, - (error: unknown) => { - if (hasCanary(error)) found.push('the thrown error message'); - }, - ); - } finally { - await instance.flow?.collector.command('shutdown'); - } + await onPage(options.page, async () => { + const instance = await createTrigger( + sourceConfig( + step, + { env: options.env, settings: options.settings }, + [], + logs, + ), + ); + try { + await instance + .trigger( + example.trigger?.type, + example.trigger?.options, + )(input) + .then( + (result) => { + if (hasCanary(isObject(result) ? result.error : undefined)) + found.push('the answered error'); + }, + (error: unknown) => { + if (hasCanary(error)) found.push('the thrown error message'); + }, + ); + } finally { + await instance.flow?.collector.command('shutdown'); + } + }); found.push(...leaks(logs)); } if (!planted) @@ -1459,22 +1474,12 @@ type KitArgs = | ['store', StoreStep] | ['source', SourceStep]; -/** Profiles of steps that run in a browser: they need a document. */ -const BROWSER_PROFILES: Profile[] = ['web-sdk', 'vendor-global']; - /** - * Refuses a browser profile in a test environment without a document (a - * server package on `web-sdk` would skip item 5 by mistake). + * Profiles of steps that run in a browser: they run on the step's page, so + * a server package on `web-sdk` cannot skip item 5 by mistake, and items 2, + * 6 and 8 never run without the page's scripts. */ -export function assertProfileEnvironment( - profile: Profile, - hasDocument = typeof document !== 'undefined', -): void { - if (BROWSER_PROFILES.includes(profile) && !hasDocument) - throw new Error( - `profile ${profile} is for web steps and needs a document: run it in a jsdom test environment, or use a server profile (http, sdk-env, socket)`, - ); -} +const BROWSER_PROFILES: Profile[] = ['web-sdk', 'vendor-global']; const KIND_PROFILES: Record = { destination: ['http', 'sdk-env', 'web-sdk', 'vendor-global', 'socket'], @@ -1537,7 +1542,10 @@ export async function conformance(...given: ConformanceArgs): Promise { throw new Error( `profile ${options.profile} is not a ${kind} profile (${KIND_PROFILES[kind].join(', ')})`, ); - assertProfileEnvironment(options.profile); + if (BROWSER_PROFILES.includes(options.profile) && !options.page) + throw new Error( + `profile ${options.profile} runs on a page: pass page from @walkeros/web-core/dev`, + ); const args = kitArgs(given); const ctx = contextOf(options); diff --git a/packages/collector/src/dev/runExamples.ts b/packages/collector/src/dev/runExamples.ts index 8144a31c1..c947a164e 100644 --- a/packages/collector/src/dev/runExamples.ts +++ b/packages/collector/src/dev/runExamples.ts @@ -21,7 +21,7 @@ import { } from '@walkeros/core'; import { startFlow } from '../flow'; import { wrapEnv } from '../wrapEnv'; -import type { Answer, Boundary, EnvAnswer, EnvBoundary } from './boundary'; +import type { Boundary, EnvAnswer, EnvBoundary } from './boundary'; import { answerEnv, installBoundary } from './boundary'; /** @@ -29,6 +29,7 @@ import { answerEnv, installBoundary } from './boundary'; * (`startFlow`), mocked only at the boundary (the package's examples env, or * the kit's fetch boundary), and its `out` and `outcome` are compared with * what the step did. One implementation with a thin binding per step kind. + * A web step runs on its page (`Flow.StepPage`, from `@walkeros/web-core/dev`). */ /** The id every run registers its step under. */ @@ -105,7 +106,7 @@ export function mappingOf( } /** How the boundary answers an example's `respond`. */ -export function answerOf(respond?: Flow.StepRespond): Answer { +export function answerOf(respond?: Flow.StepRespond): Flow.StepAnswer { if (!respond) return {}; return { status: respond.status, body: respond.body, error: respond.error }; } @@ -143,7 +144,11 @@ export interface Setup { config?: Record; /** Examples env; without it the step runs on its own env, at the boundary. */ env?: ExamplesEnv; - answers?: (index: number) => Answer; + /** The page a web step runs on: its transports, its scripts. */ + page?: Flow.StepPage; + /** Merge the page's globals over the env's `push` (its scripts land there). */ + pageGlobals?: boolean; + answers?: (index: number) => Flow.StepAnswer; envAnswers?: (index: number) => EnvAnswer; /** Records every settings key the step reads (kit item 6). */ reads?: Set; @@ -168,7 +173,7 @@ export interface Session { envBoundary?: EnvBoundary; /** Requests and SDK calls made while the step initialized. */ initCalls(): number; - /** Scripts added to the document (web), with and without the marker. */ + /** Scripts added to the page (web), with and without the marker. */ scripts(): { marked: number; unmarked: number }; run(input: Input): Promise; close(): Promise; @@ -204,18 +209,6 @@ function settingsOf(setup: Setup): unknown { }); } -/** - * The marker web-core's `loadVendor` sets on every script it injects (its - * `VENDOR_SCRIPT_MARKER`; the collector does not depend on web-core). A - * drift fails the vendor-global item 1 of every adopting destination. - */ -export const VENDOR_SCRIPT_MARKER = 'data-walkeros-vendor'; - -function documentScripts(): Element[] { - if (typeof document === 'undefined') return []; - return Array.from(document.querySelectorAll('script')); -} - /** Runs `fn` as the init phase; its errors go to `initError`. */ async function inInit( setPhase: (phase: Phase) => void, @@ -243,15 +236,15 @@ async function open( effectsOf?: () => Flow.StepEffect[], ): Promise { const logs: LogLine[] = []; - const boundary = installBoundary(setup.answers); - const before = new Set(documentScripts()); + const boundary = installBoundary(setup.answers, setup.page); let envBoundary: EnvBoundary | undefined; let env: Record | undefined; let recorded: Array<{ fn: string; args: unknown[] }> = []; if (setup.env) { + const globals = setup.pageGlobals ? boundary.page?.globals : undefined; envBoundary = answerEnv( - setup.env.push, + globals ? { ...setup.env.push, ...globals } : setup.env.push, setup.env.simulation, setup.envAnswers, ); @@ -318,13 +311,7 @@ async function open( initCalls: () => boundary.requests.filter((request) => request.phase === 'init').length + (envBoundary?.calls.filter((call) => call.phase === 'init').length ?? 0), - scripts: () => { - const added = documentScripts().filter((script) => !before.has(script)); - const marked = added.filter((script) => - script.hasAttribute(VENDOR_SCRIPT_MARKER), - ).length; - return { marked, unmarked: added.length - marked }; - }, + scripts: () => boundary.page?.scripts() ?? { marked: 0, unmarked: 0 }, run: async (input) => { const effectsBefore = effects().length; const logsBefore = logs.length; @@ -347,12 +334,9 @@ async function open( try { await collector.command('shutdown'); } finally { + // The page goes with the boundary: the scripts this run added are + // removed, so the next run starts on the page it found. boundary.restore(); - // The page is loadVendor's once-per-src registry: the scripts this - // run added go, so the next run starts on the page it found. - documentScripts() - .filter((script) => !before.has(script)) - .forEach((script) => script.remove()); } }, }; @@ -618,6 +602,8 @@ export interface DestinationRunOptions extends RunOptions { config?: Partial; /** R3: the standard events the destination maps. */ claims?: StandardEventName[]; + /** A web destination's page: `page` from `@walkeros/web-core/dev`. */ + page?: Flow.StepPage; } export interface TransformerRunOptions extends RunOptions { @@ -632,12 +618,14 @@ export interface SourceRunOptions extends RunOptions { config?: Partial; /** The package's `createTrigger`, which feeds an example's `in`. */ createTrigger?: Trigger.CreateFn; + /** A web source's page: `page` from `@walkeros/web-core/dev`. */ + page?: Flow.StepPage; } /** The setup of an example's run. */ function setupOf( example: Flow.StepExample, - options: RunOptions & { config?: object }, + options: RunOptions & { config?: object; page?: Flow.StepPage }, mapping?: Mapping.Rules, ): Setup { const respond = example.respond; @@ -649,6 +637,7 @@ function setupOf( settings: options.settings, config, env: readEnv(options.env), + page: options.page, answers: () => answerOf(respond), envAnswers: () => envAnswerOf(respond), }; @@ -790,27 +779,47 @@ export function sourceConfig( }; } +/** + * Run `fn` on a web step's page, its requests answered at the boundary and + * its scripts removed after it; without a page it runs as is. + */ +export async function onPage( + page: Flow.StepPage | undefined, + fn: () => Promise, +): Promise { + if (!page) return fn(); + const boundary = installBoundary(undefined, page); + try { + return await fn(); + } finally { + boundary.restore(); + } +} + async function checkSourceExample( step: Source.Init, example: Flow.StepExample, options: SourceRunOptions, ): Promise { - if (!options.createTrigger) + const { createTrigger } = options; + if (!createTrigger) return ['sources run through the package createTrigger: pass it']; - const captured: Flow.StepEffect[] = []; - const instance = await options.createTrigger( - sourceConfig(step, options, captured, []), - ); - try { - await instance.trigger( - example.trigger?.type, - example.trigger?.options, - )(example.in); - const out = compareOut(captured, example); - return out ? [out] : []; - } finally { - await instance.flow?.collector.command('shutdown'); - } + return onPage(options.page, async () => { + const captured: Flow.StepEffect[] = []; + const instance = await createTrigger( + sourceConfig(step, options, captured, []), + ); + try { + await instance.trigger( + example.trigger?.type, + example.trigger?.options, + )(example.in); + const out = compareOut(captured, example); + return out ? [out] : []; + } finally { + await instance.flow?.collector.command('shutdown'); + } + }); } /** diff --git a/packages/collector/src/flow.ts b/packages/collector/src/flow.ts index b28cca3c9..7f39f6ec9 100644 --- a/packages/collector/src/flow.ts +++ b/packages/collector/src/flow.ts @@ -3,13 +3,13 @@ import type { Destination, Elb, EmitFn, + ObserveSlot, ObserverFn, Source, Store, Transformer, } from '@walkeros/core'; import { - OBSERVE_STORAGE_KEY, createBatchedPoster, createTelemetryObserver, parseObserveCredential, @@ -18,112 +18,6 @@ import type { StartFlow } from './types'; import { collector } from './collector'; import { initSources } from './source'; -// Structural slices of the browser globals the web-arm credential read -// touches. The collector runs platform-neutral, so window/URLSearchParams -// are read off globalThis behind guards -- the same pattern core's -// preview.ts activator uses. On a server runtime both lookups resolve to -// undefined and the web arm reads "absent" (zero work). -interface ObserveGlobals { - window?: { - location: { href: string; search: string }; - history: { replaceState(data: unknown, title: string, url: string): void }; - localStorage: { - getItem(key: string): string | null; - setItem(key: string, value: string): void; - removeItem(key: string): void; - }; - }; - URLSearchParams?: new (init: string) => { - getAll(name: string): string[]; - delete(name: string): void; - toString(): string; - }; -} - -const G = globalThis as ObserveGlobals; - -/** - * Last `elbObserve` URL param value, or null. The param name and the - * storage slot share the `elbObserve` key by design (the preview activator - * ferries the former into the latter). Never throws. - */ -function readObserveParam(): string | null { - const win = G.window; - const USP = G.URLSearchParams; - if (!win || !USP) return null; - try { - const search = new USP(win.location.search.replace(/^\?/, '')); - const values = search.getAll(OBSERVE_STORAGE_KEY); - return values.length > 0 ? (values[values.length - 1] ?? null) : null; - } catch { - return null; - } -} - -/** Stored `elbObserve` slot, or null. Never throws (Safari private mode). */ -function readObserveSlot(): string | null { - try { - return G.window?.localStorage.getItem(OBSERVE_STORAGE_KEY) ?? null; - } catch { - return null; - } -} - -function writeObserveSlot(value: string): void { - try { - G.window?.localStorage.setItem(OBSERVE_STORAGE_KEY, value); - } catch { - // Slot persistence is best-effort; the session still attaches this page. - } -} - -/** - * CAS-style slot clear: remove the slot only when it still holds exactly - * `expected`, so a credential re-minted by another tab (or a later - * activation) is never destroyed by a stale poster's self-heal. - */ -function clearObserveSlotIf(expected: string): void { - try { - const win = G.window; - if (!win) return; - if (win.localStorage.getItem(OBSERVE_STORAGE_KEY) === expected) { - win.localStorage.removeItem(OBSERVE_STORAGE_KEY); - } - } catch { - // Storage unavailable: nothing to heal. - } -} - -/** - * Remove the `elbObserve` param from the address bar after it has been - * persisted to the slot, keeping every other param and the hash intact. - * Same history.replaceState mechanics as core preview.ts's stripParam, but - * scoped to the one param this wiring owns. - */ -function stripObserveParam(): void { - const win = G.window; - const USP = G.URLSearchParams; - if (!win || !USP) return; - try { - const href = win.location.href; - const hashIdx = href.indexOf('#'); - const hash = hashIdx >= 0 ? href.slice(hashIdx) : ''; - const withoutHash = hashIdx >= 0 ? href.slice(0, hashIdx) : href; - const qIdx = withoutHash.indexOf('?'); - const path = qIdx >= 0 ? withoutHash.slice(0, qIdx) : withoutHash; - const params = new USP(qIdx >= 0 ? withoutHash.slice(qIdx + 1) : ''); - params.delete(OBSERVE_STORAGE_KEY); - const query = params.toString(); - win.history.replaceState( - {}, - '', - `${path}${query ? `?${query}` : ''}${hash}`, - ); - } catch { - // A page that forbids history writes still attaches; the param stays. - } -} - /** * Stamp the runtime's release provenance onto every posted record. Like * `flowId`, `release` is baked by the wiring layer so emitting sites never @@ -145,6 +39,27 @@ function withFlowId(post: EmitFn, flowId: string | undefined): EmitFn { return (state) => post({ ...state, flowId }); } +/** + * Run one call into a caller-supplied `ObserveSlot`. The slot is caller code + * on the advisory path: a throwing or non-conforming one (e.g. + * `slot: pageObserveSlot`, not called) logs one ERROR line and returns + * undefined, so the caller turns Observe off and the flow runs on. + */ +function callSlot( + instance: Collector.Instance, + call: () => T, +): { value: T } | undefined { + try { + return { value: call() }; + } catch (error) { + const reason = error instanceof Error ? error.message : String(error); + instance.logger.error( + `observe: the slot failed, Observe is off: ${reason}`, + ); + return undefined; + } +} + /** * Install the advisory observation channel configured on `initConfig`: an * `observe` connect config attaches a telemetry poster for its arm. Runs @@ -163,25 +78,36 @@ function installObserve( const flowId = instance.name ?? 'default'; if ('binding' in observe) { - // Web arm: the credential arrives out-of-band via the elbObserve URL - // param (or the storage slot the preview activator ferries it into). + // Web arm: the credential arrives out-of-band and the slot reads it + // (`pageObserveSlot()` from @walkeros/web-core: the elbObserve URL param, + // or the storage slot the preview activator ferries it into). The type + // requires the slot; a JS or JSON caller can still omit it, so its + // absence is an ERROR at the default level, never a silent no-op. + const slot: ObserveSlot | undefined = observe.slot; + if (!slot) { + instance.logger.error( + 'observe: a web observe config needs slot: pageObserveSlot() from @walkeros/web-core', + ); + return; + } + // Truly absent means zero work: no parse, no storage write, no observer. - const param = readObserveParam(); - const raw = param ?? readObserveSlot(); - if (!raw) return; + const read = callSlot(instance, () => slot.read()); + if (!read?.value) return; + const { raw, fromUrl } = read.value; // A present-but-broken credential is warned, never silently dropped; // only the truly-absent case above is silent (zero-work contract). const credential = parseObserveCredential(raw); if (!credential) { - if (param === null) { + if (fromUrl) { + // URL-sourced garbage never touches stored state (anti-griefing). + instance.logger.warn('observe: malformed credential ignored'); + } else { // Slot-sourced garbage: stored state clears on its own failure, so // self-heal instead of warning on every pageview. instance.logger.warn('observe: malformed stored credential cleared'); - clearObserveSlotIf(raw); - } else { - // URL-sourced garbage never touches stored state (anti-griefing). - instance.logger.warn('observe: malformed credential ignored'); + callSlot(instance, () => slot.clear(raw)); } return; } @@ -192,10 +118,9 @@ function installObserve( return; } - writeObserveSlot(raw); - // The credential is persisted; a URL-borne copy is now redundant and, - // being a bearer credential, must not linger in the address bar. - if (param !== null) stripObserveParam(); + // Persist the credential; a URL-borne copy is then redundant and, being + // a bearer credential, the slot strips it from the address bar. + if (!callSlot(instance, () => slot.keep(raw, fromUrl))) return; let observer: ObserverFn | undefined; const post = createBatchedPoster({ @@ -207,8 +132,9 @@ function installObserve( // The observer rejected this credential (session ended or token // revoked): self-heal the slot -- CAS, only while it still holds // exactly this credential -- and detach, so posting stops and the - // next pageview is back on the silent zero-work path. - clearObserveSlotIf(raw); + // next pageview is back on the silent zero-work path. A throwing + // clear is reported and never skips the detach. + callSlot(instance, () => slot.clear(raw)); if (observer) instance.observers.delete(observer); }, }); diff --git a/packages/config/eslint/__tests__/platform.test.mjs b/packages/config/eslint/__tests__/platform.test.mjs new file mode 100644 index 000000000..468abd612 --- /dev/null +++ b/packages/config/eslint/__tests__/platform.test.mjs @@ -0,0 +1,71 @@ +import { Linter } from 'eslint'; +import * as tsParser from '@typescript-eslint/parser'; +import neutral from '../neutral.mjs'; +import node from '../node.mjs'; +import web from '../web.mjs'; + +const linter = new Linter({ cwd: '/pkg' }); +const source = 'src/index.ts'; +const test = 'src/__tests__/index.test.ts'; +// The rule reads the comments before the first statement. +const domReference = '/// \nexport {};'; + +// The rules a preset reports for one file. Under jest the presets' default +// import of the parser is empty, so the test names the parser, as the rule +// tests do. +const hits = (config, code, file = source) => + linter + .verify(code, [...config, { languageOptions: { parser: tsParser } }], { + filename: `/pkg/${file}`, + }) + .map((message) => message.ruleId); + +describe('neutral preset', () => { + it.each([ + ["Buffer.from('x');", 'no-restricted-globals'], + ["import fs from 'node:fs';", 'no-restricted-imports'], + ["localStorage.getItem('x');", 'no-restricted-globals'], + ['globalThis.document;', 'no-restricted-globals'], + ["Reflect.get(globalThis, 'history');", 'no-restricted-globals'], + [domReference, '@typescript-eslint/triple-slash-reference'], + ])('flags %s', (code, rule) => { + expect(hits(neutral, code)).toContain(rule); + }); + + it('ignores a local named process', () => { + expect( + hits( + neutral, + 'const process = { env: {} };\nexport const env = process.env;', + ), + ).toEqual([]); + }); + + it('flags jsdom in a test file', () => { + expect(hits(neutral, "import { JSDOM } from 'jsdom';", test)).toContain( + 'no-restricted-imports', + ); + }); +}); + +describe('web preset', () => { + it('flags process.env', () => { + expect(hits(web, 'export const env = process.env;')).toContain( + 'no-restricted-globals', + ); + }); +}); + +describe('node preset', () => { + it('flags localStorage', () => { + expect(hits(node, "localStorage.getItem('x');")).toContain( + 'no-restricted-globals', + ); + }); + + it.each([source, test])('flags a DOM lib reference in %s', (file) => { + expect(hits(node, domReference, file)).toContain( + '@typescript-eslint/triple-slash-reference', + ); + }); +}); diff --git a/packages/config/eslint/index.mjs b/packages/config/eslint/index.mjs index e4272af7a..cee0afc59 100644 --- a/packages/config/eslint/index.mjs +++ b/packages/config/eslint/index.mjs @@ -1,3 +1,4 @@ +import { builtinModules } from 'node:module'; import typescriptEslint from '@typescript-eslint/eslint-plugin'; import jest from 'eslint-plugin-jest'; import globals from 'globals'; @@ -5,6 +6,39 @@ import tsParser from '@typescript-eslint/parser'; import awaitSend from './rules/await-send.mjs'; import noSilentReturn from './rules/no-silent-return.mjs'; +// The platform boundary, checked by `npm run check:platform`. A web package +// uses no Node name, a server package no browser name, a neutral package +// neither. TypeScript cannot hold the line alone: @types/node declares +// localStorage and navigator, and @types/jsdom brings the DOM into any program +// that loads it. +export const browserGlobals = [ + 'window', + 'self', + 'document', + 'location', + 'history', + 'localStorage', + 'sessionStorage', + 'navigator', + 'XMLHttpRequest', +]; +export const nodeGlobals = [ + 'Buffer', + 'process', + 'require', + 'module', + '__dirname', + '__filename', + 'global', + 'setImmediate', + 'clearImmediate', +]; +export const nodeBuiltins = [...builtinModules]; + +// Tests run in Node under jest, so the platform bans on globals and imports +// skip them. +export const testFiles = ['**/__tests__/**', '**/*.test.{js,ts,tsx}']; + // walkerOS rules for the delivery contract. At `warn` while the step packages // migrate onto it; `error` once every package is migrated. const walkeros = { diff --git a/packages/config/eslint/neutral.mjs b/packages/config/eslint/neutral.mjs new file mode 100644 index 000000000..14ae6ab52 --- /dev/null +++ b/packages/config/eslint/neutral.mjs @@ -0,0 +1,51 @@ +import baseConfig, { + browserGlobals, + nodeBuiltins, + nodeGlobals, + testFiles, +} from '@walkeros/config/eslint'; + +const message = + 'A neutral package runs on web and server: it takes what one platform has through a port in @walkeros/core.'; +const jsdom = { + name: 'jsdom', + message: 'jsdom loads the DOM types into the whole neutral program.', +}; + +// The platform boundary: neutral sources use neither platform's globals or +// Node imports, and read nothing through globalThis. No file, tests included, +// references a lib or types, or imports jsdom: either loads the DOM for the +// whole program. +export default [ + ...baseConfig, + { + files: ['**/*.{js,ts,tsx}'], + rules: { + '@typescript-eslint/triple-slash-reference': [ + 'error', + { lib: 'never', types: 'never' }, + ], + 'no-restricted-imports': ['error', { paths: [jsdom] }], + }, + }, + { + files: ['**/src/**/*.{js,ts,tsx}'], + ignores: testFiles, + rules: { + 'no-restricted-globals': [ + 'error', + ...[...browserGlobals, ...nodeGlobals, 'globalThis'].map((name) => ({ + name, + message, + })), + ], + 'no-restricted-imports': [ + 'error', + { + paths: [...nodeBuiltins.map((name) => ({ name, message })), jsdom], + patterns: [{ regex: '^node:', message }], + }, + ], + }, + }, +]; diff --git a/packages/config/eslint/node.mjs b/packages/config/eslint/node.mjs index 016680825..6bb64f98e 100644 --- a/packages/config/eslint/node.mjs +++ b/packages/config/eslint/node.mjs @@ -1,6 +1,33 @@ -import baseConfig from '@walkeros/config/eslint'; +import baseConfig, { browserGlobals, testFiles } from '@walkeros/config/eslint'; import globals from 'globals'; +const message = 'Browser only: server code runs in Node.'; + +// The platform boundary: server sources use no browser global, and no file, +// tests included, references the DOM lib, which would load it for the whole +// program. +export const nodePlatform = [ + { + files: ['**/src/**/*.{js,ts,tsx}'], + ignores: testFiles, + rules: { + 'no-restricted-globals': [ + 'error', + ...browserGlobals.map((name) => ({ name, message })), + ], + }, + }, + { + files: ['**/*.{js,ts,tsx}'], + rules: { + '@typescript-eslint/triple-slash-reference': [ + 'error', + { lib: 'never', types: 'never' }, + ], + }, + }, +]; + export default [ ...baseConfig, { @@ -16,4 +43,5 @@ export default [ 'no-buffer-constructor': 'error', }, }, + ...nodePlatform, ]; diff --git a/packages/config/eslint/web.mjs b/packages/config/eslint/web.mjs index fc43cf782..0ac9b6309 100644 --- a/packages/config/eslint/web.mjs +++ b/packages/config/eslint/web.mjs @@ -1,6 +1,12 @@ -import baseConfig from '@walkeros/config/eslint'; +import baseConfig, { + nodeBuiltins, + nodeGlobals, + testFiles, +} from '@walkeros/config/eslint'; import globals from 'globals'; +const message = 'Node only: web code runs in the browser.'; + export default [ ...baseConfig, { @@ -10,8 +16,23 @@ export default [ ...globals.browser, }, }, + }, + { + // The platform boundary: web sources use no Node API. + files: ['**/src/**/*.{js,ts,tsx}'], + ignores: testFiles, rules: { - // web-specific rules + 'no-restricted-globals': [ + 'error', + ...nodeGlobals.map((name) => ({ name, message })), + ], + 'no-restricted-imports': [ + 'error', + { + paths: nodeBuiltins.map((name) => ({ name, message })), + patterns: [{ regex: '^node:', message }], + }, + ], }, }, ]; diff --git a/packages/config/package.json b/packages/config/package.json index 425d96872..a2d2d8807 100644 --- a/packages/config/package.json +++ b/packages/config/package.json @@ -20,6 +20,7 @@ "./eslint": "./eslint/index.mjs", "./eslint/web": "./eslint/web.mjs", "./eslint/node": "./eslint/node.mjs", + "./eslint/neutral": "./eslint/neutral.mjs", "./jest": "./jest/index.mjs", "./jest/web.config": "./jest/web.config.mjs", "./jest/node.config": "./jest/node.config.mjs", @@ -35,6 +36,7 @@ "@swc/jest": "^0.2.37", "@testing-library/jest-dom": "^6.6.3", "@types/jest": "^30.0.0", + "@types/node": "^25.9.1", "@eslint/js": "^9.39.4", "@typescript-eslint/eslint-plugin": "^8.28.0", "@typescript-eslint/parser": "^8.28.0", diff --git a/packages/config/scripts/check-platform.mjs b/packages/config/scripts/check-platform.mjs new file mode 100644 index 000000000..3ce6d7aba --- /dev/null +++ b/packages/config/scripts/check-platform.mjs @@ -0,0 +1,346 @@ +// `npm run check:platform`, from the walkerOS root. Every package is one +// platform: web, server, or neutral (usable on both). This checks that the +// package's tsconfig and eslint config enforce its platform and prints every +// violation. CI runs it on every run, outside turbo, so no cache can replay an +// old pass. +import { existsSync, readFileSync, readdirSync } from 'node:fs'; +import { join, relative } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { ESLint } from 'eslint'; +import ts from 'typescript'; +import { browserGlobals, nodeBuiltins, nodeGlobals } from '../eslint/index.mjs'; + +// The platform of every package that is not a step package. A step package +// declares it in walkerOS.platform: ["web"], ["server"], or both for neutral. +const declared = { + 'packages/core': 'neutral', + 'packages/collector': 'neutral', + 'packages/web/core': 'web', + 'apps/walkerjs': 'web', + 'apps/explorer': 'web', + 'apps/storybook-addon': 'web', + 'packages/server/core': 'server', + 'packages/cli': 'server', + 'apps/cli': 'server', + 'packages/runner': 'server', + 'packages/mcps/mcp': 'server', + 'packages/mcps/source-browser': 'server', +}; + +const presets = { neutral: 'base.json', web: 'web.json', server: 'node.json' }; +const presetTypes = ['node', 'jest']; + +// Neutral and web code ships in walker.js, so its lib stays at this ES year. +// Only web packages that never ship in walker.js may raise it. +const esFloor = 2020; +const esRaised = ['apps/explorer', 'apps/storybook-addon']; + +// Ambient vendor types a package may add to the preset's types. +const vendorTypes = { + 'packages/web/destinations/gtag': ['gtag.js'], + 'packages/web/destinations/meta': ['facebook-pixel'], +}; + +// What each platform's eslint config bans: globals and imports in sources, +// and in every file, tests included, triple-slash references and jsdom. +const bans = { + neutral: { + globals: [...browserGlobals, ...nodeGlobals, 'globalThis'], + imports: [...nodeBuiltins, 'jsdom'], + testImports: ['jsdom'], + nodePrefix: true, + tripleSlash: true, + }, + web: { + globals: nodeGlobals, + imports: nodeBuiltins, + testImports: [], + nodePrefix: true, + tripleSlash: false, + }, + server: { + globals: browserGlobals, + imports: [], + testImports: [], + nodePrefix: false, + tripleSlash: true, + }, +}; + +// The only files where a ban is lifted, and the names it lifts. '*' lifts +// every banned global and Node import; jsdom and triple-slash bans still hold. +const exemptions = [ + // The ./node entry of core. + { pkg: 'packages/core', files: ['src/node.ts', 'src/node/**'], allow: '*' }, + // The kit's one replace(globalThis, 'fetch', ...). + { + pkg: 'packages/collector', + files: ['src/dev/boundary.ts'], + allow: ['globalThis'], + }, + // walkeros-design-check, a Node bin. + { + pkg: 'apps/explorer', + files: ['src/design/check/**', 'src/design/generate.ts'], + allow: '*', + }, + // process.env.NODE_ENV, the define every React bundler sets. + { + pkg: 'apps/explorer', + files: ['src/components/atoms/code.tsx'], + allow: ['process'], + }, +]; + +function platformOf(rel, pkg) { + const field = pkg.walkerOS?.platform; + if (!field) return declared[rel]; + const web = field.includes('web'); + const server = field.includes('server'); + if (web && server) return 'neutral'; + if (web) return 'web'; + if (server) return 'server'; +} + +export function readTsconfig(path) { + const { config, error } = ts.readConfigFile(path, ts.sys.readFile); + if (error) + throw new Error(ts.flattenDiagnosticMessageText(error.messageText, '\n')); + const { options } = ts.parseJsonConfigFileContent( + config, + ts.sys, + join(path, '..'), + ); + return { raw: config, options }; +} + +// Without a lib, TypeScript loads the target's default lib, the DOM included. +const hasDom = (options) => + !options.lib || options.lib.some((file) => file.startsWith('lib.dom.')); +const domProblem = (options) => + options.lib ? 'lib has dom' : 'sets no lib, so TypeScript loads the DOM'; + +// The newest ES year the lib names: lib.es2022.d.ts, lib.es2021.string.d.ts. +const esLevel = (options) => + Math.max( + 0, + ...(options.lib ?? []).map((file) => { + const year = /^lib\.es(\d{4}|next)\b/.exec(file)?.[1]; + return year === 'next' ? Infinity : Number(year ?? 0); + }), + ); +const esProblem = `lib goes past es${esFloor}, the floor of walker.js`; + +export function presetProblems(name, { options }) { + const problems = []; + const web = name === presets.web; + if (web !== hasDom(options)) + problems.push(web ? 'lib drops dom' : domProblem(options)); + if (name !== presets.server && esLevel(options) > esFloor) + problems.push(esProblem); + if (String(options.types) !== String(presetTypes)) + problems.push( + `types is ${JSON.stringify(options.types)}, not ${JSON.stringify(presetTypes)}`, + ); + return problems; +} + +export function tsconfigProblems(rel, platform, { raw, options }) { + const problems = []; + const preset = `@walkeros/config/tsconfig/${presets[platform]}`; + if (raw.extends !== preset) + problems.push( + `tsconfig.json extends ${JSON.stringify(raw.extends)}, not ${preset}`, + ); + if (platform === 'web' && !hasDom(options)) + problems.push('tsconfig lib drops dom'); + if (platform !== 'web' && hasDom(options)) + problems.push(`tsconfig ${domProblem(options)}`); + if ( + platform !== 'server' && + !esRaised.includes(rel) && + esLevel(options) > esFloor + ) + problems.push(`tsconfig ${esProblem}`); + if (!options.types) { + problems.push('tsconfig sets no types, so every root @types package loads'); + } else { + const allowed = [...presetTypes, ...(vendorTypes[rel] ?? [])]; + const missing = presetTypes.filter((type) => !options.types.includes(type)); + const extra = options.types.filter((type) => !allowed.includes(type)); + if (missing.length) + problems.push(`tsconfig types misses ${missing.join(', ')}`); + if (extra.length) + problems.push( + `tsconfig types adds ${extra.join(', ')}, which is not allowlisted`, + ); + } + return problems; +} + +const listed = (items) => + items.length > 6 + ? `${items.slice(0, 5).join(', ')} and ${items.length - 5} more` + : items.join(', '); + +const names = (entries) => + entries.map((entry) => (typeof entry === 'string' ? entry : entry.name)); + +// The banned globals, imports and triple-slash setting of one file's config. +function enforced(rules) { + const on = (rule) => rule?.[0] === 2; + const globals = on(rules['no-restricted-globals']) + ? names(rules['no-restricted-globals'].slice(1)) + : []; + const imports = on(rules['no-restricted-imports']) + ? rules['no-restricted-imports'].slice(1) + : []; + // The rule takes either names or one { paths, patterns } object. + const { paths = [], patterns = [] } = + imports[0]?.paths || imports[0]?.patterns ? imports[0] : { paths: imports }; + const tripleSlash = rules['@typescript-eslint/triple-slash-reference']; + return { + globals: new Set(globals), + imports: new Set(names(paths)), + nodePrefix: patterns.some( + (p) => p.regex && new RegExp(p.regex).test('node:fs'), + ), + tripleSlash: + on(tripleSlash) && + tripleSlash[1]?.lib === 'never' && + tripleSlash[1]?.types === 'never', + }; +} + +const isTest = (file) => + /(^|\/)__tests__\//.test(file) || /\.test\.(js|ts|tsx)$/.test(file); + +// Whether an exemption lifts a banned name in this file. +function lifted(rel, file) { + const matches = (pattern) => + pattern.endsWith('/**') + ? file.startsWith(pattern.slice(0, -2)) + : file === pattern; + const exemption = exemptions.find( + (e) => e.pkg === rel && e.files.some(matches), + ); + return (name) => + exemption?.allow === '*' || Boolean(exemption?.allow.includes(name)); +} + +// Every source file's effective eslint config bans what its platform bans. +export async function eslintProblems(rel, platform, eslint, files) { + const ban = bans[platform]; + const byProblem = new Map(); + const add = (problem, file) => + byProblem.set(problem, [...(byProblem.get(problem) ?? []), file]); + for (const file of files) { + // Only jest's __mocks__ may escape lint (the base config ignores them). + if (await eslint.isPathIgnored(file)) { + if (!/(^|\/)__mocks__\//.test(file)) add('ignores the file', file); + continue; + } + const config = await eslint.calculateConfigForFile(file); + const has = enforced(config?.rules ?? {}); + const test = isTest(file); + const allow = lifted(rel, file); + const problems = []; + const globals = test + ? [] + : ban.globals.filter((n) => !has.globals.has(n) && !allow(n)); + if (globals.length) + problems.push(`no-restricted-globals misses ${listed(globals)}`); + const imports = (test ? ban.testImports : ban.imports).filter( + (n) => !has.imports.has(n) && (n === 'jsdom' || !allow(n)), + ); + if (imports.length) + problems.push(`no-restricted-imports misses ${listed(imports)}`); + if (ban.nodePrefix && !test && !has.nodePrefix && !allow('node:*')) + problems.push('no-restricted-imports misses the node: pattern'); + if (ban.tripleSlash && !has.tripleSlash) + problems.push( + "triple-slash-reference is not { lib: 'never', types: 'never' }", + ); + for (const problem of problems) add(problem, file); + } + return [...byProblem].map( + ([problem, list]) => + `eslint ${problem} (${list.length === 1 ? list[0] : `${list.length} files, e.g. ${list[0]}`})`, + ); +} + +const skip = new Set(['node_modules', 'dist', 'coverage', 'src', '__tests__']); + +function findPackages(dir, onlyPublic, found = []) { + const entries = readdirSync(dir, { withFileTypes: true }); + const files = entries.map((entry) => entry.name); + if (files.includes('package.json') && files.includes('tsconfig.json')) { + const pkg = JSON.parse(readFileSync(join(dir, 'package.json'), 'utf8')); + if (!onlyPublic || !pkg.private) found.push({ dir, pkg }); + } + for (const entry of entries) + if ( + entry.isDirectory() && + !skip.has(entry.name) && + !entry.name.startsWith('.') + ) + findPackages(join(dir, entry.name), onlyPublic, found); + return found; +} + +function sourceFiles(dir, found = []) { + if (!existsSync(dir)) return found; + for (const entry of readdirSync(dir, { withFileTypes: true })) { + const path = join(dir, entry.name); + if (entry.isDirectory()) sourceFiles(path, found); + else if (/\.tsx?$/.test(entry.name)) found.push(path); + } + return found; +} + +export async function check(root) { + const problems = []; + for (const name of Object.values(presets)) { + const preset = readTsconfig(join(root, 'packages/config/tsconfig', name)); + for (const problem of presetProblems(name, preset)) + problems.push(`packages/config/tsconfig/${name}: ${problem}`); + } + const packages = [ + ...findPackages(join(root, 'packages'), false), + ...findPackages(join(root, 'apps'), true), + ]; + for (const { dir, pkg } of packages) { + const rel = relative(root, dir); + const platform = platformOf(rel, pkg); + if (!platform) { + problems.push( + `${rel}: no platform; set walkerOS.platform or add it to the table in check-platform.mjs`, + ); + continue; + } + const own = tsconfigProblems( + rel, + platform, + readTsconfig(join(dir, 'tsconfig.json')), + ); + const files = sourceFiles(join(dir, 'src')).map((file) => + relative(dir, file), + ); + own.push( + ...(await eslintProblems(rel, platform, new ESLint({ cwd: dir }), files)), + ); + problems.push(...own.map((problem) => `${rel} (${platform}): ${problem}`)); + } + return { packages: packages.length, problems }; +} + +if (process.argv[1] === fileURLToPath(import.meta.url)) { + const { packages, problems } = await check(process.cwd()); + for (const problem of problems) console.error(problem); + console.log( + problems.length + ? `check:platform: ${problems.length} violations in ${packages} packages` + : `check:platform: ${packages} packages, no violation`, + ); + process.exitCode = problems.length ? 1 : 0; +} diff --git a/packages/config/scripts/check-platform.test.mjs b/packages/config/scripts/check-platform.test.mjs new file mode 100644 index 000000000..0883f28d9 --- /dev/null +++ b/packages/config/scripts/check-platform.test.mjs @@ -0,0 +1,221 @@ +// The drift proof of check-platform.mjs: every kind of drift the check exists +// for is reported. `npm run check:platform` runs it before the check itself. +import assert from 'node:assert/strict'; +import { + copyFileSync, + mkdirSync, + mkdtempSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { after, test } from 'node:test'; +import { fileURLToPath } from 'node:url'; +import { ESLint } from 'eslint'; +import base from '../eslint/index.mjs'; +import neutral from '../eslint/neutral.mjs'; +import { + check, + eslintProblems, + presetProblems, + readTsconfig, + tsconfigProblems, +} from './check-platform.mjs'; + +const root = fileURLToPath(new URL('../../..', import.meta.url)); +const preset = (name) => + readTsconfig(join(root, 'packages/config/tsconfig', name)); +const tsconfig = (rel) => readTsconfig(join(root, rel, 'tsconfig.json')); +const core = tsconfig('packages/core'); +const webCore = tsconfig('packages/web/core'); +const gtag = tsconfig('packages/web/destinations/gtag'); + +const withOptions = (config, options) => ({ + ...config, + options: { ...config.options, ...options }, +}); +const withLib = (config, ...files) => + withOptions(config, { lib: [...config.options.lib, ...files] }); +const lints = (rel, overrideConfig) => + new ESLint({ + cwd: join(root, rel), + overrideConfigFile: true, + overrideConfig, + }); + +const reports = (problems, text) => + assert.ok( + problems.some((problem) => problem.includes(text)), + `expected "${text}" in ${JSON.stringify(problems)}`, + ); + +test('the presets and packages as they are report nothing', () => { + for (const name of ['base.json', 'node.json', 'web.json']) + assert.deepEqual(presetProblems(name, preset(name)), []); + assert.deepEqual(tsconfigProblems('packages/core', 'neutral', core), []); + assert.deepEqual(tsconfigProblems('packages/web/core', 'web', webCore), []); +}); + +test('a preset that loads the DOM, drops types or raises the ES lib', () => { + reports( + presetProblems('base.json', withLib(preset('base.json'), 'lib.dom.d.ts')), + 'lib has dom', + ); + reports( + presetProblems( + 'node.json', + withOptions(preset('node.json'), { types: undefined }), + ), + 'types is undefined', + ); + reports( + presetProblems('web.json', withLib(preset('web.json'), 'lib.es2022.d.ts')), + 'past es2020', + ); +}); + +test('a package on the wrong preset', () => { + const raw = { + ...webCore.raw, + extends: '@walkeros/config/tsconfig/base.json', + }; + reports( + tsconfigProblems('packages/web/core', 'web', { ...webCore, raw }), + 'not @walkeros/config/tsconfig/web.json', + ); +}); + +test('a package lib that gains or drops the DOM or raises the ES lib', () => { + reports( + tsconfigProblems('packages/core', 'neutral', withLib(core, 'lib.dom.d.ts')), + 'lib has dom', + ); + reports( + tsconfigProblems( + 'packages/web/core', + 'web', + withOptions(webCore, { lib: ['lib.es2020.d.ts'] }), + ), + 'lib drops dom', + ); + reports( + tsconfigProblems( + 'packages/core', + 'neutral', + withLib(core, 'lib.es2022.d.ts'), + ), + 'past es2020', + ); + reports( + tsconfigProblems( + 'packages/web/core', + 'web', + withLib(webCore, 'lib.es2021.string.d.ts'), + ), + 'past es2020', + ); + const explorer = tsconfig('apps/explorer'); + assert.deepEqual( + tsconfigProblems( + 'apps/explorer', + 'web', + withLib(explorer, 'lib.es2022.d.ts'), + ), + [], + ); +}); + +test('package types that are unset, not allowlisted or short of the preset', () => { + reports( + tsconfigProblems( + 'packages/core', + 'neutral', + withOptions(core, { types: undefined }), + ), + 'sets no types', + ); + reports( + tsconfigProblems( + 'packages/web/destinations/gtag', + 'web', + withOptions(gtag, { types: [...gtag.options.types, 'jsdom'] }), + ), + 'adds jsdom', + ); + reports( + tsconfigProblems( + 'packages/core', + 'neutral', + withOptions(core, { types: ['node'] }), + ), + 'misses jest', + ); +}); + +test('a package linted without its platform preset', async () => { + const problems = await eslintProblems( + 'packages/core', + 'neutral', + lints('packages/core', base), + ['src/index.ts', 'src/__tests__/flow.test.ts'], + ); + reports(problems, 'no-restricted-globals misses'); + reports(problems, 'no-restricted-imports misses'); + reports(problems, 'triple-slash-reference'); + reports(problems, 'misses jsdom'); +}); + +test('a ban lifted outside the named exemptions, or wider than named', async () => { + const unnamed = [ + ...neutral, + { files: ['src/flow.ts'], rules: { 'no-restricted-globals': 'off' } }, + ]; + reports( + await eslintProblems( + 'packages/collector', + 'neutral', + lints('packages/collector', unnamed), + ['src/flow.ts'], + ), + 'misses window', + ); + const widened = [ + ...neutral, + { + files: ['src/dev/boundary.ts'], + rules: { 'no-restricted-globals': 'off' }, + }, + ]; + reports( + await eslintProblems( + 'packages/collector', + 'neutral', + lints('packages/collector', widened), + ['src/dev/boundary.ts'], + ), + 'misses window', + ); +}); + +test('a package that declares no platform', async () => { + const scratch = mkdtempSync(join(tmpdir(), 'check-platform-')); + after(() => rmSync(scratch, { recursive: true, force: true })); + mkdirSync(join(scratch, 'packages/config/tsconfig'), { recursive: true }); + for (const name of ['base.json', 'node.json', 'web.json']) + copyFileSync( + join(root, 'packages/config/tsconfig', name), + join(scratch, 'packages/config/tsconfig', name), + ); + mkdirSync(join(scratch, 'packages/new-step')); + mkdirSync(join(scratch, 'apps')); + writeFileSync( + join(scratch, 'packages/new-step/package.json'), + '{ "name": "@walkeros/new-step" }', + ); + writeFileSync(join(scratch, 'packages/new-step/tsconfig.json'), '{}'); + const { problems } = await check(scratch); + assert.deepEqual(problems, [ + 'packages/new-step: no platform; set walkerOS.platform or add it to the table in check-platform.mjs', + ]); +}); diff --git a/packages/config/tsconfig/base.json b/packages/config/tsconfig/base.json index 305820041..f9c882866 100644 --- a/packages/config/tsconfig/base.json +++ b/packages/config/tsconfig/base.json @@ -16,6 +16,8 @@ "skipLibCheck": true, "strict": true, "target": "es2022", + "lib": ["es2020"], + "types": ["node", "jest"], "allowJs": true, "resolveJsonModule": true, "incremental": true, diff --git a/packages/config/tsconfig/web.json b/packages/config/tsconfig/web.json index bf28460dc..7735cdbb2 100644 --- a/packages/config/tsconfig/web.json +++ b/packages/config/tsconfig/web.json @@ -3,7 +3,7 @@ "display": "Web", "extends": "./base.json", "compilerOptions": { - "lib": ["dom", "es2018"], + "lib": ["dom", "es2020"], "target": "es2018" } } diff --git a/packages/core/eslint.config.mjs b/packages/core/eslint.config.mjs new file mode 100644 index 000000000..46220b840 --- /dev/null +++ b/packages/core/eslint.config.mjs @@ -0,0 +1,13 @@ +import neutralConfig from '@walkeros/config/eslint/neutral'; + +export default [ + ...neutralConfig, + { + // The ./node entry is Node code; jsdom stays banned there too. + files: ['src/node.ts', 'src/node/**'], + rules: { + 'no-restricted-globals': 'off', + 'no-restricted-imports': ['error', 'jsdom'], + }, + }, +]; diff --git a/packages/core/src/__tests__/cache.test.ts b/packages/core/src/__tests__/cache.test.ts index e8fe1c1e2..a86d69e38 100644 --- a/packages/core/src/__tests__/cache.test.ts +++ b/packages/core/src/__tests__/cache.test.ts @@ -6,7 +6,7 @@ import { createMappingRoot, } from '../cache'; import { getNextSteps } from '../route'; -import { serializeStoreValue, deserializeStoreValue } from '../store/codec'; +import { serializeStoreValue, deserializeStoreValue } from '../node/storeCodec'; import { createMockCollector, createMockStore, diff --git a/packages/core/src/__tests__/clone.test.ts b/packages/core/src/__tests__/clone.test.ts index c073e3382..577f408e3 100644 --- a/packages/core/src/__tests__/clone.test.ts +++ b/packages/core/src/__tests__/clone.test.ts @@ -48,19 +48,15 @@ describe('clone', () => { expect(cloned.regex.flags).toBe(org.regex.flags); }); - test('skip unsupported types', () => { - const element = document.createElement('div'); + test('keeps non-plain objects by reference', () => { + const custom = { [Symbol.toStringTag]: 'Custom' }; const func = jest.mock; - const windowRef = window; - const documentRef = document; - const org = { element, func, windowRef, documentRef }; + const org = { custom, func }; const cloned = clone(org); - expect(cloned.element).toBe(org.element); // Reference remains the same + expect(cloned.custom).toBe(org.custom); // Reference remains the same expect(cloned.func).toBe(org.func); // Reference remains the same - expect(cloned.windowRef).toBe(org.windowRef); // Reference remains the same - expect(cloned.documentRef).toBe(org.documentRef); // Reference remains the same }); test('handle circular references', () => { diff --git a/packages/core/src/__tests__/flow.test.ts b/packages/core/src/__tests__/flow.test.ts index 3e0c9cbdd..02cda1378 100644 --- a/packages/core/src/__tests__/flow.test.ts +++ b/packages/core/src/__tests__/flow.test.ts @@ -17,6 +17,7 @@ import { } from '../schemas/flow'; import { FlowCycleError, + FlowReferenceError, getFlowSettings, getPlatform, packageNameToVariable, @@ -1453,8 +1454,7 @@ describe('Pattern Resolution', () => { process.env = originalEnv; }); - test('resolves $env.NAME from process.env', () => { - process.env.GA4_ID = 'G-ENV123'; + test('resolves $env.NAME from the env map', () => { const setup: Flow.Json = { version: 4, flows: { @@ -1471,14 +1471,15 @@ describe('Pattern Resolution', () => { }, }, }; - const config = getFlowSettings(setup); + const config = getFlowSettings(setup, undefined, { + env: { GA4_ID: 'G-ENV123' }, + }); expect(config.destinations?.gtag.config).toEqual({ id: 'G-ENV123', }); }); test('uses default value when env var not set', () => { - delete process.env.MISSING_VAR; const setup: Flow.Json = { version: 4, flows: { @@ -1495,14 +1496,13 @@ describe('Pattern Resolution', () => { }, }, }; - const config = getFlowSettings(setup); + const config = getFlowSettings(setup, undefined, { env: {} }); expect(config.destinations?.test.config).toEqual({ value: 'default-value', }); }); test('throws when env var missing and no default', () => { - delete process.env.REQUIRED_VAR; const setup: Flow.Json = { version: 4, flows: { @@ -1519,13 +1519,12 @@ describe('Pattern Resolution', () => { }, }, }; - expect(() => getFlowSettings(setup)).toThrow( + expect(() => getFlowSettings(setup, undefined, { env: {} })).toThrow( 'Environment variable "REQUIRED_VAR" not found and no default provided', ); }); test('env var takes precedence over default when set', () => { - process.env.MY_VAR = 'from-env'; const setup: Flow.Json = { version: 4, flows: { @@ -1542,12 +1541,44 @@ describe('Pattern Resolution', () => { }, }, }; - const config = getFlowSettings(setup); + const config = getFlowSettings(setup, undefined, { + env: { MY_VAR: 'from-env' }, + }); expect(config.destinations?.test.config).toEqual({ value: 'from-env', }); }); + test('never reads process.env when no env map is given', () => { + process.env.WALKEROS_PROBE = 'x'; + const setup = (value: string): Flow.Json => ({ + version: 4, + flows: { + default: { + config: { platform: 'web' }, + destinations: { + test: { package: '@walkeros/test', config: { value } }, + }, + }, + }, + }); + let caught: unknown; + try { + getFlowSettings(setup('$env.WALKEROS_PROBE')); + } catch (error) { + caught = error; + } + expect(caught).toBeInstanceOf(FlowReferenceError); + expect(caught).toMatchObject({ + code: 'MISSING_ENV', + reference: 'WALKEROS_PROBE', + }); + expect( + getFlowSettings(setup('$env.WALKEROS_PROBE:d')).destinations?.test + .config, + ).toEqual({ value: 'd' }); + }); + test('an explicit env map is the only source, never process.env', () => { process.env.AMBIENT_SECRET = 'leaked'; const setup: Flow.Json = { @@ -1598,18 +1629,7 @@ describe('Pattern Resolution', () => { }); describe('Combined Patterns', () => { - const originalEnv = process.env; - - beforeEach(() => { - process.env = { ...originalEnv }; - }); - - afterEach(() => { - process.env = originalEnv; - }); - test('uses $var as default value for $env when env not set', () => { - delete process.env.GA4_ID; const setup: Flow.Json = { version: 4, variables: { @@ -1629,14 +1649,13 @@ describe('Pattern Resolution', () => { }, }, }; - const config = getFlowSettings(setup); + const config = getFlowSettings(setup, undefined, { env: {} }); expect(config.destinations?.gtag.config).toEqual({ measurementId: 'G-FALLBACK123', }); }); test('env value takes precedence over $var default', () => { - process.env.GA4_ID = 'G-FROM-ENV'; const setup: Flow.Json = { version: 4, variables: { @@ -1656,7 +1675,9 @@ describe('Pattern Resolution', () => { }, }, }; - const config = getFlowSettings(setup); + const config = getFlowSettings(setup, undefined, { + env: { GA4_ID: 'G-FROM-ENV' }, + }); expect(config.destinations?.gtag.config).toEqual({ measurementId: 'G-FROM-ENV', }); @@ -1905,19 +1926,12 @@ describe('deferred env resolution', () => { }); it('resolves $env normally when deferred is false', () => { - process.env.TEST_SECRET = 'hunter2'; const setup = makeSetup({ key: '$env.TEST_SECRET' }); - const config = getFlowSettings(setup, 'test', { deferred: false }); - expect(config.collector).toEqual({ key: 'hunter2' }); - delete process.env.TEST_SECRET; - }); - - it('resolves $env normally when no options passed', () => { - process.env.TEST_SECRET = 'hunter2'; - const setup = makeSetup({ key: '$env.TEST_SECRET' }); - const config = getFlowSettings(setup, 'test'); + const config = getFlowSettings(setup, 'test', { + deferred: false, + env: { TEST_SECRET: 'hunter2' }, + }); expect(config.collector).toEqual({ key: 'hunter2' }); - delete process.env.TEST_SECRET; }); }); @@ -2537,7 +2551,6 @@ describe('$flow references', () => { }); test('resolves $env inside referenced flow before $flow resolves', () => { - process.env.SERVER_URL = 'https://prod.example.com/collect'; const config: Flow.Json = { version: 4, flows: { @@ -2550,14 +2563,12 @@ describe('$flow references', () => { }, }, }; - try { - const web = getFlowSettings(config, 'web'); - expect(web.destinations?.api.config).toEqual({ - settings: { url: 'https://prod.example.com/collect' }, - }); - } finally { - delete process.env.SERVER_URL; - } + const web = getFlowSettings(config, 'web', { + env: { SERVER_URL: 'https://prod.example.com/collect' }, + }); + expect(web.destinations?.api.config).toEqual({ + settings: { url: 'https://prod.example.com/collect' }, + }); }); test('resolves a flow name with a hyphen', () => { @@ -2949,7 +2960,6 @@ describe('$var unified resolver', () => { }); it('variable containing $env resolves at use', () => { - process.env.TEST_VAR = 'envvalue'; const config: Flow.Json = { version: 4, variables: { token: '$env.TEST_VAR' }, @@ -2962,9 +2972,10 @@ describe('$var unified resolver', () => { }, }, }; - const flow = getFlowSettings(config); + const flow = getFlowSettings(config, undefined, { + env: { TEST_VAR: 'envvalue' }, + }); expect(flow.config?.settings?.t).toBe('envvalue'); - delete process.env.TEST_VAR; }); it('step-level object variable shadows root-level scalar', () => { @@ -2996,21 +3007,10 @@ describe('$var unified resolver', () => { // ======================================== describe('getFlowSettings resolves $env inside config.credentials', () => { - const ENV_KEYS = ['SA', 'PK'] as const; - const saved: Record = {}; - - beforeEach(() => { - for (const key of ENV_KEYS) saved[key] = process.env[key]; - process.env.SA = '{"client_email":"svc@example.com"}'; - process.env.PK = 'private-key-value'; - }); - - afterEach(() => { - for (const key of ENV_KEYS) { - if (saved[key] === undefined) delete process.env[key]; - else process.env[key] = saved[key]; - } - }); + const env = { + SA: '{"client_email":"svc@example.com"}', + PK: 'private-key-value', + }; it('resolves a string config.credentials on a destination', () => { const config: Flow.Json = { @@ -3024,7 +3024,7 @@ describe('getFlowSettings resolves $env inside config.credentials', () => { }, }, }; - const flow = getFlowSettings(config); + const flow = getFlowSettings(config, undefined, { env }); expect(flow.destinations?.d.config).toEqual({ credentials: '{"client_email":"svc@example.com"}', }); @@ -3042,7 +3042,7 @@ describe('getFlowSettings resolves $env inside config.credentials', () => { }, }, }; - const flow = getFlowSettings(config); + const flow = getFlowSettings(config, undefined, { env }); expect(flow.stores?.s.config).toEqual({ credentials: '{"client_email":"svc@example.com"}', }); @@ -3062,7 +3062,7 @@ describe('getFlowSettings resolves $env inside config.credentials', () => { }, }, }; - const flow = getFlowSettings(config); + const flow = getFlowSettings(config, undefined, { env }); expect(flow.sources?.src.config).toEqual({ credentials: { private_key: 'private-key-value' }, }); @@ -3087,7 +3087,7 @@ describe('getFlowSettings resolves $env inside config.credentials', () => { }, }, }; - const flow = getFlowSettings(config); + const flow = getFlowSettings(config, undefined, { env }); expect(flow.destinations?.d.config).toEqual({ credentials: { client_email: '{"client_email":"svc@example.com"}', diff --git a/packages/core/src/__tests__/preview.test.ts b/packages/core/src/__tests__/preview.test.ts index eb24b36be..9f73e45cb 100644 --- a/packages/core/src/__tests__/preview.test.ts +++ b/packages/core/src/__tests__/preview.test.ts @@ -79,6 +79,10 @@ describe('parseGrant', () => { ])('returns null for %s', (_label, raw) => { expect(parseGrant(raw)).toBeNull(); }); + + it('rejects a grant whose art escapes the /preview/ path prefix (path traversal)', () => { + expect(parseGrant(makeRaw({ ...payload, art: '../../evil' }))).toBeNull(); + }); }); describe('verifyActivation', () => { diff --git a/packages/core/src/__tests__/state.test.ts b/packages/core/src/__tests__/state.test.ts index 294609655..bcd29ac4f 100644 --- a/packages/core/src/__tests__/state.test.ts +++ b/packages/core/src/__tests__/state.test.ts @@ -3,7 +3,7 @@ import { applyState, compileState } from '../state'; import type { GetStore } from '../state'; import { getByPath } from '../byPath'; import { FatalError } from '../fatalError'; -import { serializeStoreValue, deserializeStoreValue } from '../store/codec'; +import { serializeStoreValue, deserializeStoreValue } from '../node/storeCodec'; import { createAsyncMockStore, createMockCollector, diff --git a/packages/core/src/__tests__/stepError.test.ts b/packages/core/src/__tests__/stepError.test.ts index a4095179a..98d0ee189 100644 --- a/packages/core/src/__tests__/stepError.test.ts +++ b/packages/core/src/__tests__/stepError.test.ts @@ -1,3 +1,5 @@ +import type * as FatalErrorModule from '../fatalError'; +import type * as StepErrorModule from '../stepError'; import { networkErrorMessage, StepError, @@ -5,6 +7,56 @@ import { toFailureReason, } from '../stepError'; +/** + * Loads a module under an `Error` that drops the ES2022 options argument, as + * a browser without `ErrorOptions` does. + */ +function underLegacyError(path: string): T { + const Native = globalThis.Error; + class LegacyError extends Native { + constructor(message?: string) { + super(message); + } + } + const setError = (value: ErrorConstructor | typeof LegacyError) => + Object.defineProperty(globalThis, 'Error', { + value, + configurable: true, + writable: true, + }); + setError(LegacyError); + try { + let loaded: T | undefined; + jest.isolateModules(() => { + loaded = jest.requireActual(path); + }); + if (loaded === undefined) throw new Error(`${path} did not load`); + return loaded; + } finally { + setError(Native); + } +} + +describe('cause without ES2022 ErrorOptions', () => { + const original = new Error('original'); + + it('StepError keeps its cause as an own property', () => { + const { StepError: Legacy } = + underLegacyError('../stepError'); + const error = new Legacy('m', { code: 'transport', cause: original }); + expect(error.cause).toBe(original); + expect(Object.prototype.hasOwnProperty.call(error, 'cause')).toBe(true); + }); + + it('FatalError keeps its cause as an own property', () => { + const { FatalError: Legacy } = + underLegacyError('../fatalError'); + const error = new Legacy('m', { cause: original }); + expect(error.cause).toBe(original); + expect(Object.prototype.hasOwnProperty.call(error, 'cause')).toBe(true); + }); +}); + describe('StepError', () => { it('carries code, status, retryable and the response body', () => { const err = new StepError('Config settings apiKey missing', { @@ -97,7 +149,7 @@ describe('StepError', () => { describe('networkErrorMessage', () => { const withCause = (cause: unknown) => - new TypeError('fetch failed', { cause }); + Object.assign(new TypeError('fetch failed'), { cause }); it.each<[string, unknown, string]>([ [ @@ -164,7 +216,7 @@ describe('stepCall', () => { message: 'Vendor call failed', retryable: true, }); - expect(error instanceof Error && error.cause).toBeInstanceOf(Error); + expect(error instanceof StepError && error.cause).toBeInstanceOf(Error); }, ); diff --git a/packages/core/src/batchedPoster.ts b/packages/core/src/batchedPoster.ts index 29917097a..9a66f54b8 100644 --- a/packages/core/src/batchedPoster.ts +++ b/packages/core/src/batchedPoster.ts @@ -156,10 +156,9 @@ export function createBatchedPoster( ? Math.floor(rawMaxBodyBytes) : 60000; // Lazy lookup of the global `fetch` so the helper imports cleanly even in - // environments without one (it only fails when actually used). The cast - // is to the narrowed PosterFetch surface, not to a broader type. + // environments without one (it only fails when actually used). const fetchImpl: PosterFetch = - opts.fetch ?? ((url, init) => (globalThis.fetch as PosterFetch)(url, init)); + opts.fetch ?? ((url, init) => fetch(url, init)); const onError = opts.onError ?? (() => { diff --git a/packages/core/src/errorCause.ts b/packages/core/src/errorCause.ts new file mode 100644 index 000000000..2da52bcca --- /dev/null +++ b/packages/core/src/errorCause.ts @@ -0,0 +1,12 @@ +/** + * Sets `cause` the way the ES2022 `ErrorOptions` argument does (an own, + * non-enumerable property), so a cause survives in a browser that drops that + * argument. + */ +export function setCause(error: Error, cause: unknown): void { + Object.defineProperty(error, 'cause', { + value: cause, + writable: true, + configurable: true, + }); +} diff --git a/packages/core/src/fatalError.ts b/packages/core/src/fatalError.ts index fae2c172e..7c75e6549 100644 --- a/packages/core/src/fatalError.ts +++ b/packages/core/src/fatalError.ts @@ -1,3 +1,5 @@ +import { setCause } from './errorCause'; + /** * Error subclass for invariant violations or operator-initiated aborts * that must escape the top-level boundary catches in `collector.push` @@ -13,9 +15,12 @@ * violations or explicit fail-stop signals. */ export class FatalError extends Error { - constructor(message: string, options?: ErrorOptions) { - super(message, options); + declare cause?: unknown; + + constructor(message: string, options?: { cause?: unknown }) { + super(message); this.name = 'FatalError'; + if (options && 'cause' in options) setCause(this, options.cause); // Preserve prototype chain for `instanceof` checks across realms. Object.setPrototypeOf(this, FatalError.prototype); diff --git a/packages/core/src/flow.ts b/packages/core/src/flow.ts index 07425cb9b..a578bd5c1 100644 --- a/packages/core/src/flow.ts +++ b/packages/core/src/flow.ts @@ -88,11 +88,11 @@ export const SECRET_MARKER_PREFIX = '__WALKEROS_SECRET:'; export interface ResolveOptions { deferred?: boolean; /** - * The only source for non-deferred `$env` references. When set, `$env.NAME` - * reads this map (then the inline `:default`) and never `process.env`. - * Build tooling passes it so a web bundle cannot pick up whatever happens - * to be in the building process's environment. When omitted, `process.env` - * is used. + * The only source for non-deferred `$env` references: `$env.NAME` reads + * this map, then the inline `:default`, and never the ambient environment + * (`process.env`). Without it, a non-deferred `$env.NAME` takes its default + * or throws `FlowReferenceError`. A runtime that means its own environment + * passes it explicitly (`{ env: process.env }`). */ env?: Readonly>; /** @@ -180,8 +180,8 @@ export type FlowConfigResolver = (flowName: string) => unknown; * (object, array, number, boolean, string). Inline interpolation requires a * scalar (string/number/boolean) and throws on objects/arrays. Resolution is * recursive (variables may reference other variables) with cycle detection. - * - $env.NAME or $env.NAME:default → Look up `options.env` (or process.env when - * no env map is given) for non-secret config + * - $env.NAME or $env.NAME:default → Look up `options.env` for non-secret + * config, never the ambient environment * - $secret.NAME → Managed secret. In deferred (server) mode emits a marker the * bundler turns into a runtime process.env read; the deploy pipeline injects * the secret value. Throws in non-deferred (web) mode. Use for credentials. @@ -409,10 +409,7 @@ function resolvePatterns( ? `${ENV_MARKER_PREFIX}${name}:${defaultValue}` : `${ENV_MARKER_PREFIX}${name}`; } - const source = - options?.env ?? - (typeof process !== 'undefined' ? process.env : undefined); - const found = source?.[name]; + const found = options?.env?.[name]; if (found !== undefined) return found; if (defaultValue !== undefined) { return defaultValue; diff --git a/packages/core/src/getId.ts b/packages/core/src/getId.ts index 1d26ee5c0..1e70e167d 100644 --- a/packages/core/src/getId.ts +++ b/packages/core/src/getId.ts @@ -23,7 +23,9 @@ export function getId(length = 6, charset: string = defaultCharset): string { const limit = 256 - (256 % n); while (str.length < length) { const need = length - str.length; - const buffer = new Uint8Array(Math.ceil(need * 1.3) + 4); + const buffer: Uint8Array = new Uint8Array( + Math.ceil(need * 1.3) + 4, + ); if (!secureBytes(buffer)) break; // No secure source, drop to fallback below for (let i = 0; i < buffer.length && str.length < length; i++) { if (buffer[i] < limit) str += charset[buffer[i] % n]; @@ -37,11 +39,13 @@ export function getId(length = 6, charset: string = defaultCharset): string { return str; } -function secureBytes(buffer: Uint8Array): boolean { +function secureBytes(buffer: Uint8Array): boolean { try { - const cryptoObj: Crypto | undefined = globalThis.crypto; - if (cryptoObj && typeof cryptoObj.getRandomValues === 'function') { - cryptoObj.getRandomValues(buffer); + if ( + typeof crypto !== 'undefined' && + typeof crypto.getRandomValues === 'function' + ) { + crypto.getRandomValues(buffer); return true; } } catch { diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index b4cbc2599..e8d86b111 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -63,7 +63,7 @@ export * from './useHooks'; export * from './telemetry'; export * from './telemetryResolver'; export * from './preview'; -export { MOIN_PARAM, MOIN_PRODUCTION, moin } from './moin'; +export { MOIN_PARAM, MOIN_PRODUCTION } from './moin'; export type { MoinTarget } from './moin'; export * from './journey'; export { getTraceUntil, setTraceUntil } from './traceState'; @@ -115,12 +115,7 @@ export type { } from './chain'; export * from './cache'; export * from './cache-envelope'; -export { - serializeStoreValue, - deserializeStoreValue, - StoreCodecError, - isStoreValue, -} from './store/codec'; +export { isStoreValue } from './store/codec'; export * from './state'; export * from './step-entry'; export * from './transformer-mapping'; diff --git a/packages/core/src/is.ts b/packages/core/src/is.ts index 88bbf4805..ffcd22ead 100644 --- a/packages/core/src/is.ts +++ b/packages/core/src/is.ts @@ -50,18 +50,6 @@ export function isDefined(val: T | undefined): val is T { return typeof val !== 'undefined'; } -/** - * Checks if a value is an element or the document. - * - * @param elem The value to check. - * @returns True if the value is an element or the document, false otherwise. - */ -export function isElementOrDocument(elem: unknown): elem is Element { - if (!elem || typeof elem !== 'object') return false; - // Property check: Document has 'body', Element has 'tagName' - return 'body' in elem || 'tagName' in elem; -} - /** * Checks if a value is a function. * diff --git a/packages/core/src/logger.ts b/packages/core/src/logger.ts index 3381e6e16..5aad0a9a4 100644 --- a/packages/core/src/logger.ts +++ b/packages/core/src/logger.ts @@ -38,7 +38,7 @@ function normalizeError(err: Error): ErrorContext { name: err.name, stack: err.stack, }; - const cause = normalizeCause(err.cause); + const cause = 'cause' in err ? normalizeCause(err.cause) : undefined; if (cause) context.cause = cause; return context; } diff --git a/packages/core/src/moin.ts b/packages/core/src/moin.ts index 9067e1e73..bf086d4e7 100644 --- a/packages/core/src/moin.ts +++ b/packages/core/src/moin.ts @@ -1,5 +1,3 @@ -import { injectScript } from './inject-script'; - /** Where a Tag Mode script comes from and whose message may request it. */ export interface MoinTarget { /** The app origin whose message is trusted, compared with `===`. */ @@ -15,92 +13,3 @@ export const MOIN_PRODUCTION: MoinTarget = { app: 'https://app.walkeros.io', base: 'https://cdn.walkeros.io/tag-mode/', }; - -// The base64url sha384 of the script's bytes: names the file and is its -// integrity value. -const FILE = /^[A-Za-z0-9_-]{64}$/; - -// Shared by every copy on the page, so the first loader is the only one. -const GUARD = Symbol.for('walkeros.moin'); - -interface MoinMessage { - origin: string; - data: unknown; -} - -// Structural slices of the browser globals the loader touches. Core compiles -// without the DOM lib; see preview.ts for the same pattern. -export interface MoinGlobals { - window?: { - top: unknown; - location: { search: string }; - sessionStorage: { - getItem(key: string): string | null; - setItem(key: string, value: string): void; - }; - addEventListener( - type: 'message', - listener: (event: MoinMessage) => void, - capture?: boolean, - ): void; - removeEventListener( - type: 'message', - listener: (event: MoinMessage) => void, - capture?: boolean, - ): void; - }; - URLSearchParams?: new (init: string) => { has(name: string): boolean }; -} - -/** - * The Tag Mode loader. Armed only by `?elbMoin` or the tab flag it leaves in - * sessionStorage, and only in the top frame; it then waits for one message - * from `target.app` naming a script by its hash, and loads that script from - * `target.base` with the same hash as its integrity. Not armed, it adds no - * listener, makes no request and writes nothing. - */ -export function moin(target: MoinTarget = MOIN_PRODUCTION): void { - runMoin(globalThis as MoinGlobals, target); -} - -/** `moin` against explicit globals. */ -export function runMoin(g: MoinGlobals, target: MoinTarget): void { - // The loader runs first in every bundle: nothing here may stop it. - try { - const win = g.window; - const USP = g.URLSearchParams; - if (!win || !USP || win.top !== win || Reflect.get(win, GUARD)) return; - - let armed = new USP(win.location.search).has(MOIN_PARAM); - try { - const storage = win.sessionStorage; - armed = armed || storage.getItem(MOIN_PARAM) === '1'; - if (armed) storage.setItem(MOIN_PARAM, '1'); - } catch { - // Storage denied: the parameter alone arms. - } - if (!armed) return; - Reflect.set(win, GUARD, true); - - const onMessage = (event: MoinMessage): void => { - if (event.origin !== target.app) return; - const data = event.data; - if (typeof data !== 'object' || data === null) return; - if (Reflect.get(data, 'elbMoin') !== 1) return; - const file: unknown = Reflect.get(data, 'file'); - if (typeof file !== 'string' || !FILE.test(file)) return; - win.removeEventListener('message', onMessage, true); - void injectScript( - `${target.base}${file}.js`, - `sha384-${file.replace(/-/g, '+').replace(/_/g, '/')}`, - ); - }; - // In the capture phase: a site that screens messages by origin and stops - // the ones it does not know would otherwise keep the app's ping from the - // loader. On the window, a capture listener runs before every listener of - // the bubble phase. - win.addEventListener('message', onMessage, true); - } catch { - // A broken page global leaves the loader off. - } -} diff --git a/packages/core/src/node.ts b/packages/core/src/node.ts index b1d9094ed..4c8f9dfb2 100644 --- a/packages/core/src/node.ts +++ b/packages/core/src/node.ts @@ -1,13 +1,13 @@ /** * Node entry point (`@walkeros/core/node`). * - * Helpers shared by `@walkeros/cli` and `@walkeros/runner` that do not belong - * in the browser `.` entry: the temp-path helpers import node builtins, the - * terminal logger config writes to the console, and the secret redactor and - * the simulate output formatter (`toPrintable`, it reads `Buffer`) have - * no browser consumer (in `.` it was not tree-shaken out of walker.js). Core gains no dependency - * for them: the colouriser is injected by the caller, so chalk stays in the - * packages that already depend on it. + * Helpers shared by `@walkeros/cli`, `@walkeros/runner` and the server stores + * that do not belong in the browser `.` entry: the temp-path helpers import + * node builtins, the terminal logger config writes to the console, and the + * secret redactor, the simulate output formatter (`toPrintable`) and the store + * byte codec (both read `Buffer`) have no browser consumer. Core gains no + * dependency for them: the colouriser is injected by the caller, so chalk + * stays in the packages that already depend on it. */ /* eslint-disable no-console */ @@ -15,12 +15,17 @@ import os from 'os'; import path from 'path'; import { createLogger } from './logger'; import { maskKnownValues, scrubSecrets } from './redactLine'; -import { maskKnownNumbers } from './scrubJson'; +import { maskKnownNumbers } from './node/scrubJson'; export { scrubSecrets, redactLine } from './redactLine'; export type { ScrubOptions } from './redactLine'; -export { scrubJson } from './scrubJson'; -export { toPrintable } from './toPrintable'; +export { scrubJson } from './node/scrubJson'; +export { toPrintable } from './node/toPrintable'; +export { + serializeStoreValue, + deserializeStoreValue, + StoreCodecError, +} from './node/storeCodec'; import type { Config, Format, Instance, LogContext } from './types/logger'; import { Level } from './types/logger'; diff --git a/packages/core/src/__tests__/scrubJson.test.ts b/packages/core/src/node/__tests__/scrubJson.test.ts similarity index 97% rename from packages/core/src/__tests__/scrubJson.test.ts rename to packages/core/src/node/__tests__/scrubJson.test.ts index 78101484f..cfa4c28dd 100644 --- a/packages/core/src/__tests__/scrubJson.test.ts +++ b/packages/core/src/node/__tests__/scrubJson.test.ts @@ -1,5 +1,5 @@ import { maskKnownNumbers, scrubJson } from '../scrubJson'; -import { scrubSecrets } from '../redactLine'; +import { scrubSecrets } from '../../redactLine'; describe('maskKnownNumbers', () => { it('masks numbers that print a digits-only known secret', () => { diff --git a/packages/core/src/store/__tests__/codec.test.ts b/packages/core/src/node/__tests__/storeCodec.test.ts similarity index 98% rename from packages/core/src/store/__tests__/codec.test.ts rename to packages/core/src/node/__tests__/storeCodec.test.ts index cac237d09..9050b1ab8 100644 --- a/packages/core/src/store/__tests__/codec.test.ts +++ b/packages/core/src/node/__tests__/storeCodec.test.ts @@ -2,7 +2,7 @@ import { serializeStoreValue, deserializeStoreValue, StoreCodecError, -} from '../codec'; +} from '../storeCodec'; import type { StoreValue } from '../../types/store'; describe('store codec', () => { diff --git a/packages/core/src/__tests__/toPrintable.test.ts b/packages/core/src/node/__tests__/toPrintable.test.ts similarity index 100% rename from packages/core/src/__tests__/toPrintable.test.ts rename to packages/core/src/node/__tests__/toPrintable.test.ts diff --git a/packages/core/src/scrubJson.ts b/packages/core/src/node/scrubJson.ts similarity index 99% rename from packages/core/src/scrubJson.ts rename to packages/core/src/node/scrubJson.ts index 3830b1fc0..4aeace907 100644 --- a/packages/core/src/scrubJson.ts +++ b/packages/core/src/node/scrubJson.ts @@ -1,4 +1,4 @@ -import { MIN_KNOWN_LEN, scrubSecrets, type ScrubOptions } from './redactLine'; +import { MIN_KNOWN_LEN, scrubSecrets, type ScrubOptions } from '../redactLine'; import { toPrintable } from './toPrintable'; /** diff --git a/packages/core/src/node/storeCodec.ts b/packages/core/src/node/storeCodec.ts new file mode 100644 index 000000000..78b3598ad --- /dev/null +++ b/packages/core/src/node/storeCodec.ts @@ -0,0 +1,142 @@ +import type { StoreValue } from '../types/store'; +import { setCause } from '../errorCause'; +import { isRecord, isStoreValue } from '../store/codec'; + +/** + * Shared structured codec for store values. + * + * Stateless, format-only: serializes a `StoreValue` to a UTF-8 JSON byte + * payload and restores it, with no store-type branches. Binary leaves are + * tagged base64 behind a reserved marker; user objects that happen to carry + * the marker are escaped, so NO payload can be corrupted. On the store path + * binary always decodes back to a platform-neutral `Uint8Array`, never a Node + * `Buffer`. + */ + +const CACHE_MARKER = '__walkeros_cache__'; + +/** + * Thrown when a value cannot be serialized (e.g. a cyclic structure that + * `JSON.stringify` rejects). Catchable and distinguishable from a raw + * `TypeError` leaking out of the platform JSON implementation. + */ +export class StoreCodecError extends Error { + declare cause?: unknown; + + constructor(message: string, options?: { cause?: unknown }) { + super(message); + this.name = 'StoreCodecError'; + if (options && 'cause' in options) setCause(this, options.cause); + } +} + +// Precondition: the input is acyclic JSON-shaped data; binary leaves are the +// only non-JSON values handled. A Node `Buffer`, any `Uint8Array` (e.g. a +// Fetch-sourced body), and a raw `ArrayBuffer` all normalize to the same +// base64 'buffer' tag and decode back through the caller-supplied binary +// factory (see fromSerializableWith). A `Buffer` IS a `Uint8Array`, but under +// split realms (jsdom test env) a Node `Buffer` fails the `instanceof +// Uint8Array` check against the realm's `Uint8Array`, so the realm-independent +// `Buffer.isBuffer` check stays FIRST. The `Uint8Array` branch then handles +// typed-array views by slicing on byteOffset/byteLength so a subarray view +// contributes only its own bytes. +function toSerializable(value: unknown): unknown { + if (Buffer.isBuffer(value)) + return { [CACHE_MARKER]: 'buffer', d: value.toString('base64') }; + if (value instanceof Uint8Array) + return { + [CACHE_MARKER]: 'buffer', + d: Buffer.from(value.buffer, value.byteOffset, value.byteLength).toString( + 'base64', + ), + }; + if (value instanceof ArrayBuffer) + return { + [CACHE_MARKER]: 'buffer', + d: Buffer.from(value).toString('base64'), + }; + if (Array.isArray(value)) return value.map(toSerializable); + if (isRecord(value)) { + if (CACHE_MARKER in value) { + const inner: Record = {}; + for (const [k, v] of Object.entries(value)) inner[k] = toSerializable(v); + return { [CACHE_MARKER]: 'escape', d: inner }; + } + const out: Record = {}; + for (const [k, v] of Object.entries(value)) out[k] = toSerializable(v); + return out; + } + return value; +} + +/** + * Single traversal that restores the serialized form. The binary-leaf type is + * supplied by the caller via `toBinary`; the store path constructs a plain + * `Uint8Array`. Parametrizing the leaf factory keeps one implementation of the + * walk while leaving the binary representation to the caller. + */ +function fromSerializableWith( + value: unknown, + toBinary: (buf: Buffer) => Uint8Array, +): unknown { + if (isRecord(value)) { + if (CACHE_MARKER in value) { + const tag = value[CACHE_MARKER]; + const data = value.d; + if (tag === 'buffer' && typeof data === 'string') + return toBinary(Buffer.from(data, 'base64')); + if (tag === 'escape' && isRecord(data)) { + const out: Record = {}; + for (const [k, v] of Object.entries(data)) + out[k] = fromSerializableWith(v, toBinary); + return out; + } + // Unrecognized tag (not 'buffer'/'escape'): fall through to default + // object traversal so a user object that merely carries the marker key, + // or a value from a future tag version, is preserved verbatim rather + // than coerced. + } + const out: Record = {}; + for (const [k, v] of Object.entries(value)) + out[k] = fromSerializableWith(v, toBinary); + return out; + } + if (Array.isArray(value)) + return value.map((v) => fromSerializableWith(v, toBinary)); + return value; +} + +// Store path: binary leaves become a plain, platform-neutral Uint8Array (a +// standalone view over the exact decoded bytes), never a Node Buffer. +function toStoreBinary(buf: Buffer): Uint8Array { + return new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength); +} + +/** + * Serialize a `StoreValue` to its UTF-8 JSON byte payload. Throws + * `StoreCodecError` if the value cannot be encoded (e.g. a cyclic structure). + */ +export function serializeStoreValue(value: StoreValue): Uint8Array { + let json: string; + try { + json = JSON.stringify(toSerializable(value)); + } catch (cause) { + throw new StoreCodecError( + 'Failed to serialize store value (likely a cyclic structure).', + { cause }, + ); + } + return new Uint8Array(Buffer.from(json, 'utf8')); +} + +/** + * Restore a `StoreValue` from its UTF-8 JSON byte payload (or a JSON string). + */ +export function deserializeStoreValue(raw: Uint8Array | string): StoreValue { + const text = + typeof raw === 'string' ? raw : Buffer.from(raw).toString('utf8'); + const restored = fromSerializableWith(JSON.parse(text), toStoreBinary); + if (!isStoreValue(restored)) + throw new StoreCodecError('Restored value is not a valid StoreValue.'); + return restored; +} diff --git a/packages/core/src/toPrintable.ts b/packages/core/src/node/toPrintable.ts similarity index 100% rename from packages/core/src/toPrintable.ts rename to packages/core/src/node/toPrintable.ts diff --git a/packages/core/src/observeConnect.ts b/packages/core/src/observeConnect.ts index 8f36bdee4..98e7dbe50 100644 --- a/packages/core/src/observeConnect.ts +++ b/packages/core/src/observeConnect.ts @@ -38,6 +38,16 @@ export interface ObserveServer { export type Observe = ObserveWeb | ObserveServer; +/** Where a web arm finds its credential: the page, from `@walkeros/web-core`. */ +export interface ObserveSlot { + /** The URL credential if present, else the stored one; null when absent. */ + read(): { raw: string; fromUrl: boolean } | null; + /** Persist `raw`; when it came from the URL, strip it from the address bar. */ + keep(raw: string, fromUrl: boolean): void; + /** Clear the stored credential only while it still equals `raw`. */ + clear(raw: string): void; +} + /** Claims carried by a parsed `obsw_` web credential. */ export interface ObserveCredential { /** Project binding. Must match the value baked into the host bundle. */ diff --git a/packages/core/src/preview.ts b/packages/core/src/preview.ts index 15017c55b..ee809332a 100644 --- a/packages/core/src/preview.ts +++ b/packages/core/src/preview.ts @@ -1,5 +1,3 @@ -import { injectScript } from './inject-script'; - /** Claims carried by an activation grant. See the preview-sessions design spec. */ export interface ActivationGrant { /** Issuing environment, e.g. 'app:stage'. Verifier rejects a foreign issuer. */ @@ -66,12 +64,8 @@ function decodeSegment(segment: string): Uint8Array | null { const pad = segment.length % 4 === 0 ? '' : '='.repeat(4 - (segment.length % 4)); const base64 = segment.replace(/-/g, '+').replace(/_/g, '/') + pad; - // atob is untyped in core (no DOM lib, no own @types/node) but present in - // every runtime we target (browsers, Node >=16, jsdom): structural access. - const atobFn = (globalThis as { atob?: (data: string) => string }).atob; - if (!atobFn) return null; try { - const binary = atobFn(base64); + const binary = atob(base64); const bytes = new Uint8Array(binary.length); for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i); return bytes; @@ -242,7 +236,7 @@ export type VerifyParams = VerifyAudience & { source: 'url' | 'storage'; /** Epoch milliseconds. Injected so tests and the server control the clock. */ now: number; - /** Defaults to globalThis.crypto. Injectable for jsdom/node tests. */ + /** Defaults to the global crypto. Injectable for jsdom/node tests. */ crypto?: PreviewCrypto; }; @@ -360,10 +354,10 @@ export async function verifyActivation( const key = params.keyring.find((k) => k.kid === kid); if (!key) return { ok: false, reason: 'kid-mismatch' }; - // globalThis.crypto is untyped without the DOM lib: structural access is - // the batchedPoster precedent (`globalThis.fetch as PosterFetch`). - const cryptoImpl = - params.crypto ?? (globalThis as { crypto?: PreviewCrypto }).crypto; + // A runtime without WebCrypto answers `no-subtle`: verifyActivation never + // throws. + const cryptoImpl: PreviewCrypto | undefined = + params.crypto ?? (typeof crypto === 'undefined' ? undefined : crypto); const subtle = cryptoImpl?.subtle; if (!subtle) return { ok: false, reason: 'no-subtle' }; @@ -392,31 +386,6 @@ export async function verifyActivation( return { ok: true, grant }; } -export interface SwapActivatorConfig { - keyring: PreviewKey[]; - iss: string; - /** This bundle's project binding. Absent on demo hosts (see acceptForeign). */ - pb?: string; - acceptForeign?: boolean; - demoAllowlist?: string[]; - /** Bare CDN hostname, e.g. 'cdn.walkeros.io'. */ - previewOrigin: string; - /** Injectable for tests. */ - now?: () => number; - crypto?: PreviewCrypto; -} - -const STORAGE_KEY = 'elbPreview'; -/** - * Storage slot for the session-forwarding companion grant. The web arm never - * verifies it (it is session-bound by design; the session container is its - * verifier) — the activator only ferries it from the activation URL into - * storage, where the seamed preview artifact reads it to stamp the - * `X-Walkeros-Preview` header. It lives and dies with the activation state: - * set (or cleared) whenever a URL activation succeeds, cleared whenever the - * stored activation clears. - */ -export const SESSION_STORAGE_KEY = 'elbPreviewSession'; /** * Storage slot for the observe ingest credential companion (`obsw_...`), a * plaintext bearer credential riding the activation URL. The web arm never @@ -427,228 +396,3 @@ export const SESSION_STORAGE_KEY = 'elbPreviewSession'; * stored activation clears. */ export const OBSERVE_STORAGE_KEY = 'elbObserve'; -const LOG = '[walkerOS:preview]'; -const SWAP_TIMEOUT_MS = 5000; - -// Structural slices of the browser globals the activator touches. Core -// compiles without the DOM lib, so window/document/URLSearchParams/console -// are all untyped here; reading them off globalThis behind guards is the -// batchedPoster precedent. Runtime behavior is identical. -interface BrowserGlobals { - window?: { - location: { href: string; origin: string; search: string }; - history: { replaceState(data: unknown, title: string, url: string): void }; - localStorage: { - getItem(key: string): string | null; - setItem(key: string, value: string): void; - removeItem(key: string): void; - }; - }; - document?: unknown; - URLSearchParams?: new (init: string) => { - getAll(name: string): string[]; - delete(name: string): void; - toString(): string; - }; - console?: { warn(msg: string): void; info(msg: string): void }; -} - -const G = globalThis as BrowserGlobals; - -interface PreviewStore { - get(): string | null; - set(value: string): void; - clear(): void; -} - -/** localStorage under `key`, with an in-memory fallback for Safari private mode. */ -function createStore(key: string): PreviewStore { - let memory: string | null = null; - return { - get() { - try { - return G.window?.localStorage.getItem(key) ?? memory; - } catch { - return memory; - } - }, - set(value) { - try { - G.window?.localStorage.setItem(key, value); - } catch { - memory = value; - } - }, - clear() { - memory = null; - try { - G.window?.localStorage.removeItem(key); - } catch { - // in-memory copy already cleared - } - }, - }; -} - -function stripParam(): void { - const win = G.window; - const USP = G.URLSearchParams; - if (!win || !USP) return; - try { - const href = win.location.href; - const hashIdx = href.indexOf('#'); - const hash = hashIdx >= 0 ? href.slice(hashIdx) : ''; - const withoutHash = hashIdx >= 0 ? href.slice(0, hashIdx) : href; - const qIdx = withoutHash.indexOf('?'); - const path = qIdx >= 0 ? withoutHash.slice(0, qIdx) : withoutHash; - const params = new USP(qIdx >= 0 ? withoutHash.slice(qIdx + 1) : ''); - params.delete('elbPreview'); - params.delete('elbPreviewSession'); - params.delete('elbObserve'); - const query = params.toString(); - win.history.replaceState( - {}, - '', - `${path}${query ? `?${query}` : ''}${hash}`, - ); - } catch { - // A page that forbids history writes still previews; the param just stays. - } -} - -/** - * Decide whether a preview bundle should boot in place of this bundle's own flow. - * - * Returns true iff a preview took over, in which case the caller must not boot - * the production flow. Every failure path deterministically returns false: the - * loader never throws, never retries, and never leaves a page without a walker. - * - * Anti-griefing invariant: a rejected URL grant never touches stored state, - * the activator falls back to the stored grant instead. Only a failing stored - * grant clears storage (self-heal), plus `off` and `swap-failed`. - */ -export async function browserSwapActivator( - cfg: SwapActivatorConfig, -): Promise { - const win = G.window; - if (!win || !G.document) return false; - - const store = createStore(STORAGE_KEY); - const sessionStore = createStore(SESSION_STORAGE_KEY); - const observeStore = createStore(OBSERVE_STORAGE_KEY); - const now = cfg.now ?? (() => Date.now()); - const warn = (reason: string) => G.console?.warn(`${LOG} ${reason}`); - - let param: string | null = null; - let companion: string | null = null; - let observeCompanion: string | null = null; - const USP = G.URLSearchParams; - if (USP) { - try { - const search = new USP(win.location.search.replace(/^\?/, '')); - const values = search.getAll('elbPreview'); - param = values.length > 0 ? (values[values.length - 1] ?? null) : null; - const companions = search.getAll('elbPreviewSession'); - companion = - companions.length > 0 - ? (companions[companions.length - 1] ?? null) - : null; - const observes = search.getAll('elbObserve'); - observeCompanion = - observes.length > 0 ? (observes[observes.length - 1] ?? null) : null; - } catch { - param = null; - companion = null; - observeCompanion = null; - } - } - - if (param === 'off') { - store.clear(); - sessionStore.clear(); - observeStore.clear(); - stripParam(); - return false; - } - - const stored = store.get(); - - // Zero-work fast path: the overwhelmingly common case on a production page. - // No crypto, no storage writes, no DOM work. - if (!param && !stored) return false; - - const verify = (raw: string, source: 'url' | 'storage') => - verifyActivation(raw, { - keyring: cfg.keyring, - iss: cfg.iss, - origin: win.location.origin, - pb: cfg.pb, - acceptForeign: cfg.acceptForeign, - demoAllowlist: cfg.demoAllowlist, - source, - now: now(), - crypto: cfg.crypto, - }); - - let grant: ActivationGrant | null = null; - - // 1) URL grant first. Rejection for any reason (malformed, unsigned, bad - // signature, expired, wrong aud) is warned and ignored, and must never - // clear stored state: verification runs cheap checks before crypto, so - // at rejection time we cannot know whether the value was even one of - // ours, and a crafted link must not end someone's active session. - if (param) { - const result = await verify(param, 'url'); - if (result.ok) { - grant = result.grant; - store.set(param); - // The URL activation defines the session context: carry its companion - // forwarding grant into storage, or clear a stale one from an earlier - // session. Never verified here — the session container is its verifier. - if (companion) sessionStore.set(companion); - else sessionStore.clear(); - // Same rule for the observe ingest credential: ferried raw, never - // verified here (the observer is its verifier). - if (observeCompanion) observeStore.set(observeCompanion); - else observeStore.clear(); - } else { - warn(result.reason); - } - stripParam(); - } - - // 2) Fall back to the stored grant. This one clears on failure: it came - // from our own storage, so clearing is self-healing, not griefable. - // (Except `no-subtle`: that is the environment's fault, not the grant's.) - if (!grant && stored) { - const result = await verify(stored, 'storage'); - if (result.ok) { - grant = result.grant; - } else { - warn(result.reason); - if (result.reason !== 'no-subtle') { - store.clear(); - sessionStore.clear(); - observeStore.clear(); - } - } - } - - if (!grant) return false; - - const src = `https://${cfg.previewOrigin}/preview/${grant.art}.js`; - const loaded = await injectScript(src, grant.sri, SWAP_TIMEOUT_MS); - - if (!loaded) { - warn('swap-failed'); - store.clear(); - sessionStore.clear(); - observeStore.clear(); - return false; - } - - G.console?.info( - `${LOG} active ${grant.art} ends ${new Date(grant.sxp * 1000).toISOString()}`, - ); - return true; -} diff --git a/packages/core/src/scope.ts b/packages/core/src/scope.ts index b694e0f9b..b4ddce61a 100644 --- a/packages/core/src/scope.ts +++ b/packages/core/src/scope.ts @@ -84,12 +84,8 @@ export function normalizeBody(body: unknown, base64?: boolean): unknown { let text = body; if (base64) { - // atob is untyped in core (no DOM lib, no own @types/node) but present in - // every runtime we target: structural access, as in preview.ts. - const atobFn = (globalThis as { atob?: (data: string) => string }).atob; - if (!atobFn) return body; try { - const binary = atobFn(body); + const binary = atob(body); // UTF-8 decode without TextDecoder, which core cannot rely on. const bytes = new Uint8Array(binary.length); for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i); diff --git a/packages/core/src/stepError.ts b/packages/core/src/stepError.ts index d73fdc4ca..ce3680a79 100644 --- a/packages/core/src/stepError.ts +++ b/packages/core/src/stepError.ts @@ -1,3 +1,5 @@ +import { setCause } from './errorCause'; + /** * Options of a {@link StepError}: the failure code, and what decides whether * a retry can help (`status`, an explicit `retryable`). @@ -64,12 +66,11 @@ export class StepError extends Error { status?: number; retryable: boolean; body?: unknown; + declare cause?: unknown; constructor(message: string, options: StepErrorOptions) { - super( - message, - options.cause !== undefined ? { cause: options.cause } : undefined, - ); + super(message); + if (options.cause !== undefined) setCause(this, options.cause); this.name = 'StepError'; this.code = options.code; if (options.status !== undefined) this.status = options.status; @@ -137,7 +138,8 @@ export function toFailureReason(err: unknown): FailureReason { */ export function networkErrorMessage(error: unknown): string { const message = error instanceof Error ? error.message : 'Network error'; - const cause = error instanceof Error ? error.cause : undefined; + const cause = + error instanceof Error && 'cause' in error ? error.cause : undefined; const code = typeof cause === 'object' && cause !== null && diff --git a/packages/core/src/store/codec.ts b/packages/core/src/store/codec.ts index 6a34ca8c3..53cbc0c7c 100644 --- a/packages/core/src/store/codec.ts +++ b/packages/core/src/store/codec.ts @@ -1,116 +1,15 @@ import type { StoreValue } from '../types/store'; /** - * Shared structured codec for store values. - * - * Stateless, format-only: serializes a `StoreValue` to a UTF-8 JSON byte - * payload and restores it, with no store-type branches. Binary leaves are - * tagged base64 behind a reserved marker; user objects that happen to carry - * the marker are escaped, so NO payload can be corrupted. On the store path - * binary always decodes back to a platform-neutral `Uint8Array`, never a Node - * `Buffer`. + * The neutral half of the store codec: the guard a cache uses to narrow a + * wide value to `StoreValue`. The byte codec itself reads `Buffer`, so it + * lives in the node entry (`@walkeros/core/node`, `../node/storeCodec`). */ -const CACHE_MARKER = '__walkeros_cache__'; - -/** - * Thrown when a value cannot be serialized (e.g. a cyclic structure that - * `JSON.stringify` rejects). Catchable and distinguishable from a raw - * `TypeError` leaking out of the platform JSON implementation. - */ -export class StoreCodecError extends Error { - constructor(message: string, options?: { cause?: unknown }) { - super(message, options); - this.name = 'StoreCodecError'; - } -} - -function isRecord(value: unknown): value is Record { +export function isRecord(value: unknown): value is Record { return value !== null && typeof value === 'object' && !Array.isArray(value); } -// Precondition: the input is acyclic JSON-shaped data; binary leaves are the -// only non-JSON values handled. A Node `Buffer`, any `Uint8Array` (e.g. a -// Fetch-sourced body), and a raw `ArrayBuffer` all normalize to the same -// base64 'buffer' tag and decode back through the caller-supplied binary -// factory (see fromSerializableWith). A `Buffer` IS a `Uint8Array`, but under -// split realms (jsdom test env) a Node `Buffer` fails the `instanceof -// Uint8Array` check against the realm's `Uint8Array`, so the realm-independent -// `Buffer.isBuffer` check stays FIRST. The `Uint8Array` branch then handles -// typed-array views by slicing on byteOffset/byteLength so a subarray view -// contributes only its own bytes. -function toSerializable(value: unknown): unknown { - if (Buffer.isBuffer(value)) - return { [CACHE_MARKER]: 'buffer', d: value.toString('base64') }; - if (value instanceof Uint8Array) - return { - [CACHE_MARKER]: 'buffer', - d: Buffer.from(value.buffer, value.byteOffset, value.byteLength).toString( - 'base64', - ), - }; - if (value instanceof ArrayBuffer) - return { - [CACHE_MARKER]: 'buffer', - d: Buffer.from(value).toString('base64'), - }; - if (Array.isArray(value)) return value.map(toSerializable); - if (isRecord(value)) { - if (CACHE_MARKER in value) { - const inner: Record = {}; - for (const [k, v] of Object.entries(value)) inner[k] = toSerializable(v); - return { [CACHE_MARKER]: 'escape', d: inner }; - } - const out: Record = {}; - for (const [k, v] of Object.entries(value)) out[k] = toSerializable(v); - return out; - } - return value; -} - -/** - * Single traversal that restores the serialized form. The binary-leaf type is - * supplied by the caller via `toBinary`; the store path constructs a plain - * `Uint8Array`. Parametrizing the leaf factory keeps one implementation of the - * walk while leaving the binary representation to the caller. - */ -function fromSerializableWith( - value: unknown, - toBinary: (buf: Buffer) => Uint8Array, -): unknown { - if (isRecord(value)) { - if (CACHE_MARKER in value) { - const tag = value[CACHE_MARKER]; - const data = value.d; - if (tag === 'buffer' && typeof data === 'string') - return toBinary(Buffer.from(data, 'base64')); - if (tag === 'escape' && isRecord(data)) { - const out: Record = {}; - for (const [k, v] of Object.entries(data)) - out[k] = fromSerializableWith(v, toBinary); - return out; - } - // Unrecognized tag (not 'buffer'/'escape'): fall through to default - // object traversal so a user object that merely carries the marker key, - // or a value from a future tag version, is preserved verbatim rather - // than coerced. - } - const out: Record = {}; - for (const [k, v] of Object.entries(value)) - out[k] = fromSerializableWith(v, toBinary); - return out; - } - if (Array.isArray(value)) - return value.map((v) => fromSerializableWith(v, toBinary)); - return value; -} - -// Store path: binary leaves become a plain, platform-neutral Uint8Array (a -// standalone view over the exact decoded bytes), never a Node Buffer. -function toStoreBinary(buf: Buffer): Uint8Array { - return new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength); -} - /** * Recursive type guard: the restored value is structurally a `StoreValue`. * The single restore walk (`fromSerializableWith`) is JSON-derived plus @@ -124,7 +23,7 @@ function toStoreBinary(buf: Buffer): Uint8Array { * `unknown -> StoreValue` boundary). The `StoreValue` TYPE still excludes * `undefined`; this only mirrors what survives serialization. */ -function isStoreValue(value: unknown): value is StoreValue { +export function isStoreValue(value: unknown): value is StoreValue { if ( value === null || typeof value === 'string' || @@ -141,44 +40,3 @@ function isStoreValue(value: unknown): value is StoreValue { ); return false; } - -/** - * Serialize a `StoreValue` to its UTF-8 JSON byte payload. Throws - * `StoreCodecError` if the value cannot be encoded (e.g. a cyclic structure). - */ -export function serializeStoreValue(value: StoreValue): Uint8Array { - let json: string; - try { - json = JSON.stringify(toSerializable(value)); - } catch (cause) { - throw new StoreCodecError( - 'Failed to serialize store value (likely a cyclic structure).', - { cause }, - ); - } - return new Uint8Array(Buffer.from(json, 'utf8')); -} - -/** - * Restore a `StoreValue` from its UTF-8 JSON byte payload (or a JSON string). - */ -export function deserializeStoreValue(raw: Uint8Array | string): StoreValue { - const text = - typeof raw === 'string' ? raw : Buffer.from(raw).toString('utf8'); - const restored = fromSerializableWith(JSON.parse(text), toStoreBinary); - if (!isStoreValue(restored)) - throw new StoreCodecError('Restored value is not a valid StoreValue.'); - return restored; -} - -// Internal format helpers shared with the cache envelope codec (cache.ts) so -// there is exactly one implementation of the structured transform. -// `isStoreValue` lets boundary callers (the event cache) narrow a wide -// `unknown` payload to `StoreValue` without a cast before wrapping it. -export { - toSerializable, - fromSerializableWith, - CACHE_MARKER, - isRecord, - isStoreValue, -}; diff --git a/packages/core/src/types/collector.ts b/packages/core/src/types/collector.ts index 219866ce4..262867c44 100644 --- a/packages/core/src/types/collector.ts +++ b/packages/core/src/types/collector.ts @@ -11,7 +11,7 @@ import type { Mapping, } from '.'; import type { Ingest } from './ingest'; -import type { Observe } from '../observeConnect'; +import type { ObserveServer, ObserveSlot, ObserveWeb } from '../observeConnect'; import type { ObserverFn } from './observer'; import type { CauseLog } from '../causeLog'; @@ -145,11 +145,12 @@ export interface InitConfig extends Partial { /** * Observation session connect config, wired by `startFlow` before the run * command. The server form attaches a telemetry poster directly from its - * config trio; the web form attaches only when an `elbObserve` credential - * is present (URL param or storage slot) and its binding matches. Purely - * advisory: nothing on this path can affect event processing. + * config trio; the web form reads its credential through `slot` + * (`pageObserveSlot()` from `@walkeros/web-core`) and attaches only when + * one is present and its binding matches. Purely advisory: nothing on this + * path can affect event processing. */ - observe?: Observe; + observe?: (ObserveWeb & { slot: ObserveSlot }) | ObserveServer; /** * Caller-supplied observers, added verbatim to `collector.observers` by * `startFlow`. Escape hatch for custom emit targets; same advisory diff --git a/packages/core/src/types/flow.ts b/packages/core/src/types/flow.ts index 50b379835..87768e47d 100644 --- a/packages/core/src/types/flow.ts +++ b/packages/core/src/types/flow.ts @@ -786,6 +786,44 @@ export namespace Flow { /** The effects a step produces, in execution order. Empty = no observable effect (filtered, passthrough). */ export type StepOut = readonly StepEffect[]; + /** How a kit boundary answers one request. */ + export interface StepAnswer { + status?: number; + body?: unknown; + /** The status text, which failure answers use to echo a canary. */ + statusText?: string; + /** A network error, or no answer at all. */ + error?: 'network' | 'timeout'; + /** Answer one macrotask late (the floating-send check). */ + late?: boolean; + } + + /** Records one request at the kit boundary; returns its answer and marks it settled. */ + export type StepRecord = (effect: StepEffect) => { + answer: StepAnswer; + settle: () => void; + }; + + /** + * The browser page a web step runs on in the conformance kit and the + * examples runner. `@walkeros/web-core/dev` implements it as `page`. + */ + export interface StepPage { + open(record: StepRecord): StepPageSession; + } + + /** One run on the page, from `open` to `close`. */ + export interface StepPageSession { + /** Page globals merged over an examples env's `push` (the document). */ + globals: Record; + /** Scripts added since `open`, with and without the loadVendor marker. */ + scripts(): { marked: number; unmarked: number }; + /** Fire `error` on every marked script (kit item 2, vendor-global). */ + failScripts(): void; + /** Restore the stubs and remove the scripts added since `open`. */ + close(): void; + } + /** * Named example pair for a step. * `in` is the input to the step, `out` is the expected output. diff --git a/packages/destinations/eslint.config.mjs b/packages/destinations/eslint.config.mjs new file mode 100644 index 000000000..0bbff757c --- /dev/null +++ b/packages/destinations/eslint.config.mjs @@ -0,0 +1,3 @@ +import neutralConfig from '@walkeros/config/eslint/neutral'; + +export default [...neutralConfig]; diff --git a/packages/mcps/eslint.config.mjs b/packages/mcps/eslint.config.mjs new file mode 100644 index 000000000..a9ecf85f8 --- /dev/null +++ b/packages/mcps/eslint.config.mjs @@ -0,0 +1,4 @@ +import baseConfig from '@walkeros/config/eslint'; +import { nodePlatform } from '@walkeros/config/eslint/node'; + +export default [...baseConfig, ...nodePlatform]; diff --git a/packages/mcps/mcp/eslint.config.js b/packages/mcps/mcp/eslint.config.js index e26755d4f..aa3799dea 100644 --- a/packages/mcps/mcp/eslint.config.js +++ b/packages/mcps/mcp/eslint.config.js @@ -1,4 +1,5 @@ import baseConfig from '@walkeros/config/eslint'; +import { nodePlatform } from '@walkeros/config/eslint/node'; const message = 'Files, processes and cli loaders or runners belong to src/runtime. Accept a FlowRuntime instead.'; @@ -31,6 +32,7 @@ const devZones = ['src/dev.ts', 'src/examples/**', 'src/schemas/**']; export default [ ...baseConfig, + ...nodePlatform, { files: ['src/**/*.ts'], ignores: [...tests, ...devZones], diff --git a/packages/runner/eslint.config.js b/packages/runner/eslint.config.js index 3e9847f37..25def3968 100644 --- a/packages/runner/eslint.config.js +++ b/packages/runner/eslint.config.js @@ -1,4 +1,5 @@ import baseConfig from '@walkeros/config/eslint'; +import { nodePlatform } from '@walkeros/config/eslint/node'; // The base config's own no-restricted-syntax selectors. A flat config replaces // a rule set twice for one file rather than merging it, so the block below @@ -26,6 +27,7 @@ const apiPathSyntax = [ export default [ ...baseConfig, + ...nodePlatform, { files: ['src/**/*.ts'], ignores: ['src/api-paths.ts', 'src/**/__tests__/**', 'src/**/*.test.ts'], diff --git a/packages/runner/src/auth-headers.ts b/packages/runner/src/auth-headers.ts index aadde3d0d..03026c033 100644 --- a/packages/runner/src/auth-headers.ts +++ b/packages/runner/src/auth-headers.ts @@ -1,11 +1,17 @@ +// What a Headers takes; the DOM lib's HeadersInit is not in a server program. +type HeadersInput = ConstructorParameters[0]; + /** * Normalize headers from any RequestInit format to a plain object. */ -function normalizeHeaders(headers?: HeadersInit): Record { +function normalizeHeaders(headers?: HeadersInput): Record { if (!headers) return {}; if (headers instanceof Headers) return Object.fromEntries(headers.entries()); if (Array.isArray(headers)) return Object.fromEntries(headers); - return headers; + const normalized: Record = {}; + for (const [name, value] of Object.entries(headers)) + if (value !== undefined) normalized[name] = value; + return normalized; } /** @@ -15,7 +21,7 @@ function normalizeHeaders(headers?: HeadersInit): Record { */ export function mergeAuthHeaders( token: string | null | undefined, - headers?: HeadersInit, + headers?: HeadersInput, ): Record { const normalized = normalizeHeaders(headers); if (token) normalized.Authorization = `Bearer ${token}`; diff --git a/packages/server/core/tsconfig.json b/packages/server/core/tsconfig.json index b1ac14360..b1c75c589 100644 --- a/packages/server/core/tsconfig.json +++ b/packages/server/core/tsconfig.json @@ -1,5 +1,5 @@ { - "extends": "@walkeros/config/tsconfig/base.json", + "extends": "@walkeros/config/tsconfig/node.json", "compilerOptions": { "rootDir": "src" }, diff --git a/packages/server/destinations/datamanager/src/__tests__/index.test.ts b/packages/server/destinations/datamanager/src/__tests__/index.test.ts index 276541ddf..67d625b8a 100644 --- a/packages/server/destinations/datamanager/src/__tests__/index.test.ts +++ b/packages/server/destinations/datamanager/src/__tests__/index.test.ts @@ -536,6 +536,45 @@ describe('Server Destination Data Manager', () => { ).rejects.toThrow('Data Manager rejected the event'); }); + test('delivers a 2xx answer without a request id', async () => { + mockFetch.mockResolvedValue({ ok: true, json: async () => ({}) }); + + await expect( + destination.push( + getEvent('page view'), + createMockContext({ + config: defaultConfig, + env: { authClient: mockAuthClient, fetch: mockFetch }, + logger: mockLogger, + id: 'test-dm', + }), + ), + ).resolves.toBeUndefined(); + }); + + test.each([null, 'accepted'])( + 'throws on an unreadable answer (%p)', + async (answer) => { + mockFetch.mockResolvedValue({ ok: true, json: async () => answer }); + + await expect( + destination.push( + getEvent('page view'), + createMockContext({ + config: defaultConfig, + env: { authClient: mockAuthClient, fetch: mockFetch }, + logger: mockLogger, + id: 'test-dm', + }), + ), + ).rejects.toMatchObject({ + message: 'Data Manager answered with an unreadable body', + code: 'transport', + retryable: false, + }); + }, + ); + test('uses custom URL when provided', async () => { const event = getEvent('page view'); diff --git a/packages/server/destinations/datamanager/src/push.ts b/packages/server/destinations/datamanager/src/push.ts index 2a991e4ef..fa3699703 100644 --- a/packages/server/destinations/datamanager/src/push.ts +++ b/packages/server/destinations/datamanager/src/push.ts @@ -1,5 +1,5 @@ import type { PushFn } from './types'; -import type { IngestEventsRequest, IngestEventsResponse } from './types'; +import type { IngestEventsRequest } from './types'; import type { Collector, Mapping as WalkerOSMapping } from '@walkeros/core'; import { createMappingRoot, @@ -223,11 +223,20 @@ const send: PushFn = async function ( }); } - const result: IngestEventsResponse = await response.json(); + const result: unknown = await response.json(); + if (!isIngestAnswer(result)) { + // Google answered 2xx: a retry would send events it already accepted. + throw new StepError('Data Manager answered with an unreadable body', { + code: 'transport', + retryable: false, + body: result, + }); + } logger.debug('API response', { status: response.status, - requestId: result.requestId, + requestId: + typeof result.requestId === 'string' ? result.requestId : undefined, }); // If validation errors exist, throw them @@ -257,3 +266,20 @@ async function resolveClient( const value = await getMappingValue(root, setting ?? fallback, { collector }); return typeof value === 'string' && value !== '' ? value : undefined; } + +/** + * What push reads from an events.ingest answer: an object, with validation + * errors only under validateOnly. The request id is only logged. + */ +interface IngestAnswer { + requestId?: unknown; + validationErrors?: unknown[]; +} + +function isIngestAnswer(value: unknown): value is IngestAnswer { + return ( + isObject(value) && + (value.validationErrors === undefined || + Array.isArray(value.validationErrors)) + ); +} diff --git a/packages/server/sources/express/src/__tests__/input-hygiene.test.ts b/packages/server/sources/express/src/__tests__/input-hygiene.test.ts index 7317298bd..7e3b5dc10 100644 --- a/packages/server/sources/express/src/__tests__/input-hygiene.test.ts +++ b/packages/server/sources/express/src/__tests__/input-hygiene.test.ts @@ -146,8 +146,8 @@ describe('express input hygiene', () => { expect(response.headers.get('content-type')).toContain( 'application/json', ); - const body: { success: boolean } = await response.json(); - expect(body.success).toBe(false); + const body: unknown = await response.json(); + expect(body).toMatchObject({ success: false }); expect(mockPush).not.toHaveBeenCalled(); } finally { await close(); @@ -164,8 +164,8 @@ describe('express input hygiene', () => { body: `{"pad":"${'x'.repeat(1024 * 1024 + 64)}"}`, }); expect(response.status).toBe(413); - const body: { success: boolean } = await response.json(); - expect(body.success).toBe(false); + const body: unknown = await response.json(); + expect(body).toMatchObject({ success: false }); } finally { await close(); } @@ -271,7 +271,7 @@ describe('express input hygiene', () => { expect(response.headers.get('content-type')).toContain( 'application/json', ); - const body: { success: boolean; error: string } = await response.json(); + const body: unknown = await response.json(); expect(body).toEqual({ success: false, error: 'Not found' }); expect(mockPush).not.toHaveBeenCalled(); } finally { @@ -416,7 +416,7 @@ describe('express input hygiene', () => { try { const response = await postEvent(url); expect(response.status).toBe(500); - const body: { success: boolean; error: string } = await response.json(); + const body: unknown = await response.json(); expect(body).toEqual({ success: false, error: 'Internal server error', @@ -441,8 +441,8 @@ describe('express input hygiene', () => { expect(response.headers.get('content-type')).toContain( 'application/json', ); - const body: { success: boolean } = await response.json(); - expect(body.success).toBe(false); + const body: unknown = await response.json(); + expect(body).toMatchObject({ success: false }); expect(logger.error).toHaveBeenCalledWith(fault); expect(logger.debug).not.toHaveBeenCalled(); } finally { diff --git a/packages/server/sources/express/src/__tests__/text-body.test.ts b/packages/server/sources/express/src/__tests__/text-body.test.ts index 9fa006a4d..ddcafd287 100644 --- a/packages/server/sources/express/src/__tests__/text-body.test.ts +++ b/packages/server/sources/express/src/__tests__/text-body.test.ts @@ -176,8 +176,8 @@ describe('express text/plain bodies', () => { }); expect(response.status).toBe(400); - const body: { success: boolean } = await response.json(); - expect(body.success).toBe(false); + const body: unknown = await response.json(); + expect(body).toMatchObject({ success: false }); expect(harness.received).toEqual([]); expect(harness.collector.status.sources.express?.rejected).toBe(1); } finally { diff --git a/packages/server/sources/fetch/tsconfig.json b/packages/server/sources/fetch/tsconfig.json index d548a0926..639d7d4bc 100644 --- a/packages/server/sources/fetch/tsconfig.json +++ b/packages/server/sources/fetch/tsconfig.json @@ -1,5 +1,5 @@ { - "extends": "@walkeros/config/tsconfig/base.json", + "extends": "@walkeros/config/tsconfig/node.json", "include": ["src/**/*.ts"], "exclude": ["node_modules", "dist"] } diff --git a/packages/server/stores/fs/src/__tests__/conformance.test.ts b/packages/server/stores/fs/src/__tests__/conformance.test.ts index 182d3a2d7..70b4c1b47 100644 --- a/packages/server/stores/fs/src/__tests__/conformance.test.ts +++ b/packages/server/stores/fs/src/__tests__/conformance.test.ts @@ -1,6 +1,6 @@ import path from 'path'; import { conformance } from '@walkeros/collector/dev'; -import { serializeStoreValue } from '@walkeros/core'; +import { serializeStoreValue } from '@walkeros/core/node'; import { storeFsInit } from '..'; import { examples, schemas } from '../dev'; import type { FsStoreEnv } from '../types'; diff --git a/packages/server/stores/fs/src/store.ts b/packages/server/stores/fs/src/store.ts index 9cf91ad86..102ad1694 100644 --- a/packages/server/stores/fs/src/store.ts +++ b/packages/server/stores/fs/src/store.ts @@ -1,11 +1,11 @@ import * as fs from 'fs/promises'; import * as path from 'path'; import type { Store } from '@walkeros/core'; +import { StepError } from '@walkeros/core'; import { serializeStoreValue, deserializeStoreValue, - StepError, -} from '@walkeros/core'; +} from '@walkeros/core/node'; import type { FsStoreEnv, FsStoreFs, FsStoreSettings } from './types'; function isValidKey(key: string): boolean { diff --git a/packages/server/stores/gcs/src/store.ts b/packages/server/stores/gcs/src/store.ts index 116d0e530..8bc8de3f8 100644 --- a/packages/server/stores/gcs/src/store.ts +++ b/packages/server/stores/gcs/src/store.ts @@ -1,9 +1,9 @@ import type { Logger, Store } from '@walkeros/core'; +import { StepError } from '@walkeros/core'; import { serializeStoreValue, deserializeStoreValue, - StepError, -} from '@walkeros/core'; +} from '@walkeros/core/node'; import type { GcsStoreSettings, ServiceAccountCredentials, diff --git a/packages/server/stores/s3/src/store.ts b/packages/server/stores/s3/src/store.ts index 5c9a19f11..3401fa21c 100644 --- a/packages/server/stores/s3/src/store.ts +++ b/packages/server/stores/s3/src/store.ts @@ -1,9 +1,9 @@ import { S3mini } from 's3mini'; +import { StepError } from '@walkeros/core'; import { serializeStoreValue, deserializeStoreValue, - StepError, -} from '@walkeros/core'; +} from '@walkeros/core/node'; import type { Store } from '@walkeros/core'; import type { S3StoreSettings, Types } from './types'; diff --git a/packages/transformers/eslint.config.mjs b/packages/transformers/eslint.config.mjs new file mode 100644 index 000000000..0bbff757c --- /dev/null +++ b/packages/transformers/eslint.config.mjs @@ -0,0 +1,3 @@ +import neutralConfig from '@walkeros/config/eslint/neutral'; + +export default [...neutralConfig]; diff --git a/packages/transformers/ga4/tsconfig.json b/packages/transformers/ga4/tsconfig.json index b1c75c589..b1ac14360 100644 --- a/packages/transformers/ga4/tsconfig.json +++ b/packages/transformers/ga4/tsconfig.json @@ -1,5 +1,5 @@ { - "extends": "@walkeros/config/tsconfig/node.json", + "extends": "@walkeros/config/tsconfig/base.json", "compilerOptions": { "rootDir": "src" }, diff --git a/packages/transformers/validate/tsconfig.json b/packages/transformers/validate/tsconfig.json index b1c75c589..b1ac14360 100644 --- a/packages/transformers/validate/tsconfig.json +++ b/packages/transformers/validate/tsconfig.json @@ -1,5 +1,5 @@ { - "extends": "@walkeros/config/tsconfig/node.json", + "extends": "@walkeros/config/tsconfig/base.json", "compilerOptions": { "rootDir": "src" }, diff --git a/packages/web/core/package.json b/packages/web/core/package.json index 33b7645c0..5100ff818 100644 --- a/packages/web/core/package.json +++ b/packages/web/core/package.json @@ -32,6 +32,7 @@ "update": "npx npm-check-updates -u && npm update" }, "devDependencies": { + "@walkeros/collector": "4.8.1", "@walkeros/core": "4.8.1" }, "repository": { diff --git a/packages/web/core/src/__tests__/fixtures/src/sender.ts b/packages/web/core/src/__tests__/fixtures/src/sender.ts new file mode 100644 index 000000000..1bab27e56 --- /dev/null +++ b/packages/web/core/src/__tests__/fixtures/src/sender.ts @@ -0,0 +1,27 @@ +import { StepError } from '@walkeros/core'; +import type { Destination } from '@walkeros/core'; +import { sendWeb } from '../../../sendWeb'; + +type Types = Destination.Types<{ url?: string }>; + +function requireUrl(config: Destination.Config): string { + const url = config.settings?.url; + if (!url) + throw new StepError('Config settings url missing', { code: 'config' }); + return url; +} + +/** Sends every event with sendWeb's synchronous XHR (the http profile). */ +export const xhrSender: Destination.Instance = { + type: 'xhr-sender', + config: {}, + init: ({ config }) => { + requireUrl(config); + }, + push: (event, { config, rule }) => { + if (rule?.silent) return; + sendWeb(requireUrl(config), JSON.stringify({ name: event.name }), { + transport: 'xhr', + }); + }, +}; diff --git a/packages/web/core/src/__tests__/fixtures/src/vendor.ts b/packages/web/core/src/__tests__/fixtures/src/vendor.ts new file mode 100644 index 000000000..e45c4a9c0 --- /dev/null +++ b/packages/web/core/src/__tests__/fixtures/src/vendor.ts @@ -0,0 +1,147 @@ +import { isObject, StepError } from '@walkeros/core'; +import type { Collector, Destination, On } from '@walkeros/core'; +import { VENDOR_SCRIPT_MARKER } from '../../../loadVendor'; + +/** + * Vendor destinations that load a script into the page, for the kit's + * vendor-global checks on `page` (item 6 scans this folder). + */ + +export interface Vendor { + track(name: string, site?: string): void | Promise; + consent(granted: boolean): void; +} + +type Types = Destination.Types< + { site?: string }, + unknown, + { vendor?: Vendor; document?: Document } +>; + +function isVendor(value: unknown): value is Vendor { + return ( + isObject(value) && + typeof value.track === 'function' && + typeof value.consent === 'function' + ); +} + +/** Passes a consent change to its vendor (item 8). */ +const passConsent: On.OnFn = (type, { env, data }) => { + const vendor = env.vendor; + if (type !== 'consent' || !isVendor(vendor) || !isObject(data)) return; + vendor.consent(data.marketing === true); +}; + +/** Calls the vendor global and reports its failure (`transport`). */ +async function track( + name: string, + { config, env }: Destination.PushContext, +): Promise { + if (!env.vendor) + throw new StepError('Vendor not ready', { code: 'vendor-not-ready' }); + try { + await env.vendor.track(name, config.settings?.site); + } catch (error) { + throw new StepError('Vendor call failed', { + code: 'transport', + cause: error, + }); + } +} + +/** Whether a page's vendor script fired `error`, per collector (one page). */ +const scriptFailed = new WeakMap(); + +/** + * Loads its vendor script as loadVendor does (marked, awaited): a push + * waits for the script and fails `vendor-timeout` at a short deadline (jsdom + * never loads it), and once the script fired `error` every push fails + * `vendor-not-loaded`: two causes, one line each. + */ +export const scriptVendor: Destination.Instance = { + type: 'script-vendor', + config: {}, + init: ({ collector }) => { + const script = document.createElement('script'); + script.src = 'https://vendor.example/sdk.js'; + script.setAttribute(VENDOR_SCRIPT_MARKER, ''); + script.addEventListener('error', () => scriptFailed.set(collector, true)); + document.head.appendChild(script); + }, + push: async (_, { collector, rule }) => { + if (rule?.silent) return; + if (scriptFailed.get(collector)) + throw new StepError('Vendor script failed to load', { + code: 'vendor-not-loaded', + }); + await new Promise((__, reject) => + setTimeout( + () => + reject( + new StepError('Vendor script did not load within 50ms', { + code: 'vendor-timeout', + }), + ), + 50, + ), + ); + }, + on: passConsent, +}; + +/** A script's error, kept with its element (the page is the registry). */ +const failedScripts = new WeakSet(); + +/** + * Injects its marked script once per page, as loadVendor does (a marked + * script with its src already on the page is the injected one), into the + * env's document where it has one, with `loadScript` on by default. A push + * calls the vendor and, with `reports`, fails `vendor-not-loaded` once the + * script fired `error`. + */ +function pageScript(reports: boolean): Destination.Instance { + const src = 'https://vendor.example/page.js'; + const scriptOnPage = (doc: Document) => + Array.from(doc.querySelectorAll(`script[${VENDOR_SCRIPT_MARKER}]`)).find( + (script) => script.getAttribute('src') === src, + ); + return { + type: reports ? 'page-script' : 'page-script-ignoring', + config: { loadScript: true }, + init: ({ config, env }) => { + const doc = env.document ?? document; + if (!config.loadScript || scriptOnPage(doc)) return; + const script = doc.createElement('script'); + script.src = src; + script.setAttribute(VENDOR_SCRIPT_MARKER, ''); + script.addEventListener('error', () => failedScripts.add(script)); + doc.head.appendChild(script); + }, + push: async (event, context) => { + const script = scriptOnPage(context.env.document ?? document); + if (reports && script && failedScripts.has(script)) + throw new StepError('Vendor script failed to load', { + code: 'vendor-not-loaded', + }); + if (context.rule?.silent) return; + await track(event.name, context); + }, + on: passConsent, + }; +} + +export const pageScriptVendor = pageScript(true); +export const pageScriptIgnoring = pageScript(false); + +/** Loads its marked script plus a helper script without the marker (item 1). */ +export const unmarkedScriptVendor: Destination.Instance = { + ...pageScriptVendor, + type: 'unmarked-script', + init: (context) => { + pageScriptVendor.init?.(context); + const helper = document.createElement('script'); + helper.src = 'https://vendor.example/helper.js'; + document.head.appendChild(helper); + }, +}; diff --git a/packages/core/src/__tests__/inject-script.test.ts b/packages/web/core/src/__tests__/injectScript.test.ts similarity index 93% rename from packages/core/src/__tests__/inject-script.test.ts rename to packages/web/core/src/__tests__/injectScript.test.ts index 96f153702..2a5f50ff9 100644 --- a/packages/core/src/__tests__/inject-script.test.ts +++ b/packages/web/core/src/__tests__/injectScript.test.ts @@ -1,7 +1,7 @@ /** * @jest-environment jsdom */ -import { injectScript } from '../inject-script'; +import { injectScript } from '../injectScript'; beforeEach(() => { document.head.innerHTML = ''; diff --git a/packages/core/src/__tests__/moin.test.ts b/packages/web/core/src/__tests__/moin.test.ts similarity index 82% rename from packages/core/src/__tests__/moin.test.ts rename to packages/web/core/src/__tests__/moin.test.ts index e0cd3540e..e74a335b2 100644 --- a/packages/core/src/__tests__/moin.test.ts +++ b/packages/web/core/src/__tests__/moin.test.ts @@ -2,14 +2,9 @@ * @jest-environment jsdom * @jest-environment-options {"url": "https://shop.example.com/page"} */ -import { - MOIN_PARAM, - MOIN_PRODUCTION, - moin, - runMoin, - type MoinGlobals, - type MoinTarget, -} from '../moin'; +import type { MoinTarget } from '@walkeros/core'; +import { MOIN_PARAM, MOIN_PRODUCTION } from '@walkeros/core'; +import { moin, runMoin } from '../moin'; const T: MoinTarget = { app: 'https://app.test', @@ -135,27 +130,23 @@ describe('arming', () => { }); describe('frames', () => { - // A frame window: the parameter is present, so only the top check decides. - function frame(top: 'self' | 'other') { - const addEventListener = jest.fn(); - const frameWindow: MoinGlobals['window'] = { - top: null, - location: { search: '?elbMoin' }, - sessionStorage: window.sessionStorage, - addEventListener, - removeEventListener: jest.fn(), - }; - frameWindow.top = top === 'self' ? frameWindow : window; - runMoin({ window: frameWindow, URLSearchParams }, T); - return addEventListener; - } - it('stays off in a sub-frame', () => { - expect(frame('other')).not.toHaveBeenCalled(); + const iframe = document.createElement('iframe'); + document.body.appendChild(iframe); + const frame = iframe.contentWindow; + if (!frame) throw new Error('the iframe has no window'); + // The tab flag arms the frame, so only the top check decides. + frame.sessionStorage.setItem(MOIN_PARAM, '1'); + const frameListen = jest.spyOn(frame, 'addEventListener'); + runMoin(frame, T); + expect(frameListen).not.toHaveBeenCalled(); + expect(Reflect.get(frame, GUARD)).toBeUndefined(); }); it('arms the same window when it is the top frame', () => { - expect(frame('self')).toHaveBeenCalledWith( + setSearch('?elbMoin'); + runMoin(window, T); + expect(listen).toHaveBeenCalledWith( 'message', expect.any(Function), true, @@ -164,42 +155,31 @@ describe('arming', () => { }); describe('broken globals', () => { - class BrokenParams { - constructor() { - throw new Error('broken polyfill'); - } - has(): boolean { - return false; - } - } - const fail = (): never => { throw new Error('broken page'); }; - function broken(part: 'params' | 'top' | 'listener'): MoinGlobals { - const frameWindow: MoinGlobals['window'] = { - top: null, - location: { search: '?elbMoin' }, - sessionStorage: window.sessionStorage, - addEventListener: part === 'listener' ? fail : jest.fn(), - removeEventListener: jest.fn(), - }; - if (part === 'top') - Object.defineProperty(frameWindow, 'top', { get: fail }); - else frameWindow.top = frameWindow; - return { - window: frameWindow, - URLSearchParams: part === 'params' ? BrokenParams : URLSearchParams, - }; - } + beforeEach(() => setSearch('?elbMoin')); - it.each(['params', 'top', 'listener'] as const)( - 'never throws when %s throws', - (part) => { - expect(() => runMoin(broken(part), T)).not.toThrow(); - }, - ); + it('never throws when URLSearchParams throws', () => { + jest.spyOn(window, 'URLSearchParams').mockImplementation(fail); + expect(() => moin(T)).not.toThrow(); + expect(messageListeners()).toHaveLength(0); + }); + + it('never throws when the top check throws', () => { + const page = new Proxy(window, { + get: (target, key) => + key === 'top' ? fail() : Reflect.get(target, key), + }); + expect(() => runMoin(page, T)).not.toThrow(); + expect(messageListeners()).toHaveLength(0); + }); + + it('never throws when adding the listener throws', () => { + listen.mockImplementation(fail); + expect(() => moin(T)).not.toThrow(); + }); }); it('lets only the first loader on a page arm', () => { diff --git a/packages/web/core/src/__tests__/observeSlot.test.ts b/packages/web/core/src/__tests__/observeSlot.test.ts new file mode 100644 index 000000000..c87197f5e --- /dev/null +++ b/packages/web/core/src/__tests__/observeSlot.test.ts @@ -0,0 +1,300 @@ +/** + * The page's Observe credential slot, driven through a real collector: the + * web arm reads the credential from the real jsdom URL (history.replaceState) + * or the localStorage slot, persists it, strips a URL-borne copy and + * self-heals the slot. + * + * Env-injection only: the poster's global fetch is swapped via + * Object.defineProperty (and restored), warns are captured via an injected + * logger handler. No jest.mock anywhere. + */ +import { startFlow } from '@walkeros/collector'; +import { Level } from '@walkeros/core'; +import type { Logger, PosterFetch } from '@walkeros/core'; +import { pageObserveSlot } from '../observeSlot'; + +const OBSERVER_URL = 'https://obs.example'; + +interface CapturedPost { + url: string; + body: string; + headers: Record; +} + +/** Swap the global fetch for a capturing stub; returns a restore fn. */ +function stubFetch( + posts: CapturedPost[], + response: { ok: boolean; status: number } = { ok: true, status: 200 }, +): () => void { + const stub: PosterFetch = async (url, init) => { + posts.push({ url, body: init.body, headers: init.headers }); + return response; + }; + const original = Object.getOwnPropertyDescriptor(globalThis, 'fetch'); + Object.defineProperty(globalThis, 'fetch', { + value: stub, + configurable: true, + writable: true, + }); + return () => { + if (original) Object.defineProperty(globalThis, 'fetch', original); + else Reflect.deleteProperty(globalThis, 'fetch'); + }; +} + +/** Capture WARN-level messages via an injected logger handler. */ +function warnCapture(): { warns: string[]; logger: Logger.Config } { + const warns: string[] = []; + const handler: Logger.Handler = (level, message) => { + if (level === Level.WARN) warns.push(message); + }; + return { warns, logger: { level: 'WARN', handler } }; +} + +function setUrl(search: string): void { + window.history.replaceState({}, '', `https://example.com/${search}`); +} + +describe('pageObserveSlot', () => { + beforeEach(() => { + localStorage.clear(); + setUrl(''); + }); + + it('installs nothing with no URL param and no stored slot (zero-work)', async () => { + const setItem = jest.spyOn(Storage.prototype, 'setItem'); + const { collector } = await startFlow({ + observe: { url: OBSERVER_URL, binding: 'pb_x', slot: pageObserveSlot() }, + }); + expect(collector.observers.size).toBe(0); + expect(setItem).not.toHaveBeenCalled(); + setItem.mockRestore(); + }); + + it('reads the URL param, persists the slot, and posts with binding header + credential secret', async () => { + jest.useRealTimers(); + setUrl('?elbObserve=obsw_pb_x.ses_9.sec_1'); + const posts: CapturedPost[] = []; + const restore = stubFetch(posts); + try { + const { collector, elb } = await startFlow({ + observe: { + url: OBSERVER_URL, + binding: 'pb_x', + slot: pageObserveSlot(), + }, + }); + expect(collector.observers.size).toBe(1); + expect(localStorage.getItem('elbObserve')).toBe('obsw_pb_x.ses_9.sec_1'); + + await elb('page view'); + await new Promise((resolve) => setTimeout(resolve, 120)); + + expect(posts.length).toBeGreaterThan(0); + const first = posts[0]; + expect(first?.url).toBe(`${OBSERVER_URL}/ingest/ses_9`); + expect(first?.headers.Authorization).toBe('Bearer sec_1'); + expect(first?.headers['X-Walkeros-Binding']).toBe('pb_x'); + } finally { + restore(); + } + }); + + it('attaches from the stored slot when no URL param is present', async () => { + setUrl('?keep=1'); + localStorage.setItem('elbObserve', 'obsw_pb_x.ses_9.sec_1'); + const { collector } = await startFlow({ + observe: { url: OBSERVER_URL, binding: 'pb_x', slot: pageObserveSlot() }, + }); + expect(collector.observers.size).toBe(1); + // Slot-sourced attach never touches the address bar. + expect(window.location.search).toBe('?keep=1'); + }); + + it('strips elbObserve from the address bar after a URL-param attach, keeping innocent params', async () => { + setUrl('?foo=1&elbObserve=obsw_pb_x.ses_9.sec_1&bar=2'); + const { collector } = await startFlow({ + observe: { url: OBSERVER_URL, binding: 'pb_x', slot: pageObserveSlot() }, + }); + expect(collector.observers.size).toBe(1); + expect(window.location.search).not.toContain('elbObserve'); + expect(window.location.search).toContain('foo=1'); + expect(window.location.search).toContain('bar=2'); + }); + + it('clears the slot and stops posting when the observer answers 401', async () => { + jest.useRealTimers(); + localStorage.setItem('elbObserve', 'obsw_pb_x.ses_9.sec_1'); + const posts: CapturedPost[] = []; + const restore = stubFetch(posts, { ok: false, status: 401 }); + try { + const { collector, elb } = await startFlow({ + observe: { + url: OBSERVER_URL, + binding: 'pb_x', + slot: pageObserveSlot(), + }, + }); + expect(collector.observers.size).toBe(1); + + await elb('page view'); + await new Promise((resolve) => setTimeout(resolve, 120)); + + expect(posts.length).toBeGreaterThan(0); + expect(localStorage.getItem('elbObserve')).toBeNull(); + expect(collector.observers.size).toBe(0); + + // Detached: further events never reach the poster. + const before = posts.length; + await elb('page next'); + await new Promise((resolve) => setTimeout(resolve, 120)); + expect(posts.length).toBe(before); + } finally { + restore(); + } + }); + + it('keeps the slot and keeps observing on a non-401 status', async () => { + jest.useRealTimers(); + localStorage.setItem('elbObserve', 'obsw_pb_x.ses_9.sec_1'); + const posts: CapturedPost[] = []; + const restore = stubFetch(posts, { ok: false, status: 500 }); + try { + const { collector, elb } = await startFlow({ + observe: { + url: OBSERVER_URL, + binding: 'pb_x', + slot: pageObserveSlot(), + }, + }); + + await elb('page view'); + await new Promise((resolve) => setTimeout(resolve, 120)); + + expect(posts.length).toBeGreaterThan(0); + expect(localStorage.getItem('elbObserve')).toBe('obsw_pb_x.ses_9.sec_1'); + expect(collector.observers.size).toBe(1); + } finally { + restore(); + } + }); + + it('warns and installs nothing when the credential pb mismatches the binding', async () => { + setUrl('?elbObserve=obsw_pb_other.ses_9.sec_1'); + const { warns, logger } = warnCapture(); + const { collector } = await startFlow({ + logger, + observe: { url: OBSERVER_URL, binding: 'pb_x', slot: pageObserveSlot() }, + }); + expect(collector.observers.size).toBe(0); + expect(warns.some((m) => m.includes('binding'))).toBe(true); + expect(localStorage.getItem('elbObserve')).toBeNull(); + }); + + it('warns and installs nothing on a malformed credential', async () => { + setUrl('?elbObserve=garbage'); + const { warns, logger } = warnCapture(); + const { collector } = await startFlow({ + logger, + observe: { url: OBSERVER_URL, binding: 'pb_x', slot: pageObserveSlot() }, + }); + expect(collector.observers.size).toBe(0); + expect(warns.length).toBeGreaterThan(0); + }); + + it('never touches a stored slot for a malformed URL credential', async () => { + // URL-sourced garbage never touches stored state (anti-griefing): a + // crafted link cannot destroy a valid stored credential. + setUrl('?elbObserve=garbage'); + localStorage.setItem('elbObserve', 'obsw_pb_x.ses_9.sec_1'); + const setItem = jest.spyOn(Storage.prototype, 'setItem'); + const removeItem = jest.spyOn(Storage.prototype, 'removeItem'); + const { warns, logger } = warnCapture(); + const { collector } = await startFlow({ + logger, + observe: { url: OBSERVER_URL, binding: 'pb_x', slot: pageObserveSlot() }, + }); + expect(collector.observers.size).toBe(0); + expect(warns).toContain('observe: malformed credential ignored'); + expect(setItem).not.toHaveBeenCalled(); + expect(removeItem).not.toHaveBeenCalled(); + expect(localStorage.getItem('elbObserve')).toBe('obsw_pb_x.ses_9.sec_1'); + setItem.mockRestore(); + removeItem.mockRestore(); + }); + + it('honors a baked flowId and trace level from the connect config', async () => { + jest.useRealTimers(); + localStorage.setItem('elbObserve', 'obsw_pb_x.ses_9.sec_1'); + const posts: CapturedPost[] = []; + const restore = stubFetch(posts); + try { + const { collector, elb } = await startFlow({ + name: 'localname', + observe: { + url: OBSERVER_URL, + binding: 'pb_x', + flowId: 'flow_db_1', + level: 'trace', + slot: pageObserveSlot(), + }, + }); + // The baked level reaches the collector-wide supplier so destination + // call capture runs at trace. + expect(collector.observeLevel?.()).toBe('trace'); + + await elb('page view'); + await new Promise((resolve) => setTimeout(resolve, 120)); + + expect(posts.length).toBeGreaterThan(0); + // Records carry the BAKED flowId (not the local config name), and at + // trace the projection keeps the inbound event payload. + expect(JSON.parse(posts[0]?.body ?? '')).toEqual( + expect.objectContaining({ + v: 1, + records: expect.arrayContaining([ + expect.objectContaining({ + flowId: 'flow_db_1', + inEvent: expect.objectContaining({ name: 'page view' }), + }), + ]), + }), + ); + } finally { + restore(); + } + }); + + it('defaults to the flow name, standard level, and no observeLevel supplier', async () => { + localStorage.setItem('elbObserve', 'obsw_pb_x.ses_9.sec_1'); + const { collector } = await startFlow({ + name: 'myflow', + observe: { url: OBSERVER_URL, binding: 'pb_x', slot: pageObserveSlot() }, + }); + expect(collector.observers.size).toBe(1); + // No baked level: the collector-wide capture supplier stays unset. + expect(collector.observeLevel).toBeUndefined(); + }); + + it('warns and self-heals a malformed STORED slot value', async () => { + localStorage.setItem('elbObserve', 'garbage'); + const { warns, logger } = warnCapture(); + const { collector } = await startFlow({ + logger, + observe: { url: OBSERVER_URL, binding: 'pb_x', slot: pageObserveSlot() }, + }); + expect(collector.observers.size).toBe(0); + expect(warns.length).toBeGreaterThan(0); + // Stored state clears on its own failure -- the next pageview is + // back on the silent zero-work path. + expect(localStorage.getItem('elbObserve')).toBeNull(); + }); + + it('clears the stored slot only while it still holds the same credential', () => { + // CAS: a credential re-minted by another tab is never destroyed by a + // stale poster's self-heal. + localStorage.setItem('elbObserve', 'obsw_pb_x.ses_new.sec_2'); + pageObserveSlot().clear('obsw_pb_x.ses_9.sec_1'); + expect(localStorage.getItem('elbObserve')).toBe('obsw_pb_x.ses_new.sec_2'); + }); +}); diff --git a/packages/web/core/src/__tests__/page.test.ts b/packages/web/core/src/__tests__/page.test.ts new file mode 100644 index 000000000..a768cbd39 --- /dev/null +++ b/packages/web/core/src/__tests__/page.test.ts @@ -0,0 +1,287 @@ +import path from 'path'; +import { conformance, runExamples } from '@walkeros/collector/dev'; +import type { Destination, Flow } from '@walkeros/core'; +import { page } from '../dev'; +import { VENDOR_SCRIPT_MARKER } from '../loadVendor'; +import { sendWeb } from '../sendWeb'; +import { xhrSender } from './fixtures/src/sender'; +import type { Vendor } from './fixtures/src/vendor'; +import { + pageScriptIgnoring, + pageScriptVendor, + scriptVendor, + unmarkedScriptVendor, +} from './fixtures/src/vendor'; + +describe('page', () => { + const effects: Flow.StepEffect[] = []; + let settled = 0; + const recordAnswering = + (answer: Flow.StepAnswer): Flow.StepRecord => + (effect) => { + effects.push(effect); + return { + answer, + settle: () => { + settled++; + }, + }; + }; + + beforeEach(() => { + jest.useRealTimers(); + effects.length = 0; + settled = 0; + }); + + it('answers sendBeacon at the boundary while open, and removes it at close', () => { + const session = page.open(recordAnswering({ status: 500 })); + try { + expect(navigator.sendBeacon('https://e.example/b', 'body')).toBe(false); + } finally { + session.close(); + } + expect(effects).toEqual([['sendBeacon', 'https://e.example/b', 'body']]); + expect(settled).toBe(1); + expect('sendBeacon' in navigator).toBe(false); + }); + + it('answers a synchronous XMLHttpRequest at the boundary, and restores the original at close', () => { + const original = window.XMLHttpRequest; + const session = page.open( + recordAnswering({ status: 201, body: { ok: true } }), + ); + const xhr = new XMLHttpRequest(); + try { + xhr.open('POST', 'https://e.example/x', false); + xhr.send('body'); + } finally { + session.close(); + } + expect(effects).toEqual([ + ['XMLHttpRequest', 'POST', 'https://e.example/x', 'body'], + ]); + expect(settled).toBe(1); + expect(xhr.status).toBe(201); + expect(xhr.responseText).toBe('{"ok":true}'); + expect(window.XMLHttpRequest).toBe(original); + }); + + it('fails a synchronous XMLHttpRequest on a network error, and answers the async form late', async () => { + const failing = page.open(recordAnswering({ error: 'network' })); + try { + const xhr = new XMLHttpRequest(); + xhr.open('POST', 'https://e.example/x', false); + expect(() => xhr.send()).toThrow('NetworkError'); + } finally { + failing.close(); + } + + const late = page.open(recordAnswering({ status: 200 })); + try { + const xhr = new XMLHttpRequest(); + xhr.open('GET', 'https://e.example/x'); + const loaded = new Promise((resolve) => + xhr.addEventListener('load', () => resolve()), + ); + xhr.send(); + expect(settled).toBe(1); + expect(xhr.readyState).toBe(1); + await loaded; + expect(settled).toBe(2); + expect(xhr.status).toBe(200); + } finally { + late.close(); + } + }); + + it('counts the scripts added since open, fails the marked ones, and removes them at close', () => { + const kept = document.createElement('script'); + document.head.appendChild(kept); + const session = page.open(recordAnswering({})); + const marked = document.createElement('script'); + marked.setAttribute(VENDOR_SCRIPT_MARKER, ''); + document.head.appendChild(marked); + const unmarked = document.createElement('script'); + document.body.appendChild(unmarked); + const failed = jest.fn(); + marked.addEventListener('error', failed); + + expect(session.globals).toEqual({ document }); + expect(session.scripts()).toEqual({ marked: 1, unmarked: 1 }); + session.failScripts(); + expect(failed).toHaveBeenCalledTimes(1); + session.close(); + + expect(marked.isConnected).toBe(false); + expect(unmarked.isConnected).toBe(false); + expect(kept.isConnected).toBe(true); + }); + + it("records sendWeb's beacon and XHR as a run's effects in the examples runner", async () => { + const url = 'https://e.example/collect'; + const sending: Destination.Instance = { + type: 'sending', + config: {}, + push: (event) => { + const body = JSON.stringify({ name: event.name }); + sendWeb(url, body, { transport: 'beacon' }); + sendWeb(url, body, { transport: 'xhr' }); + }, + }; + await expect( + runExamples( + 'destination', + sending, + { + pageView: { + in: { name: 'page view' }, + out: [ + ['sendBeacon', url, '{"name":"page view"}'], + ['XMLHttpRequest', 'POST', url, '{"name":"page view"}'], + ], + }, + }, + { page }, + ), + ).resolves.toBeUndefined(); + }); +}); + +describe('conformance kit on the page', () => { + beforeEach(() => { + jest.useRealTimers(); + }); + + const vendor: Vendor = { + track: () => undefined, + consent: () => undefined, + }; + const event = { name: 'page view', data: { title: 'Home' } }; + const examples: Flow.StepExamples = { + pageView: { in: event, out: [['vendor.track', 'page view', 'docs']] }, + silent: { + public: false, + description: 'A silent rule calls nothing.', + in: event, + mapping: { silent: true }, + out: [], + }, + revoke: { + command: 'consent', + in: { marketing: false }, + out: [['vendor.consent', false]], + }, + }; + const root = path.join(__dirname, 'fixtures'); + const options = { + profile: 'vendor-global' as const, + root, + schema: { type: 'object', properties: { site: { type: 'string' } } }, + settings: { site: 'docs' }, + env: { + push: { vendor }, + simulation: ['call:vendor.track', 'call:vendor.consent'], + }, + examples, + page, + }; + + it('counts item 2 per cause: a vendor-timeout line before the script error does not count', async () => { + await expect( + conformance('destination', scriptVendor, { + ...options, + schema: { type: 'object', properties: {} }, + settings: {}, + examples: { + pageView: { in: event }, + silent: examples.silent, + revoke: examples.revoke, + }, + }), + ).resolves.toBeUndefined(); + }); + + // No declared failure example: item 2 must reach the script error itself. + describe('a script injected once per page, loadScript on by default (item 2)', () => { + it('reaches the script error although item 1 injected the src first', async () => { + await expect( + conformance('destination', pageScriptVendor, options), + ).resolves.toBeUndefined(); + }); + + it('reaches the script error when the examples env carries its own document', async () => { + const own = document.implementation.createHTMLDocument('env'); + await expect( + conformance('destination', pageScriptVendor, { + ...options, + env: { ...options.env, push: { vendor, document: own } }, + }), + ).resolves.toBeUndefined(); + }); + + it('fails item 2 when the push ignores the script error', async () => { + await expect( + conformance('destination', pageScriptIgnoring, options), + ).rejects.toMatchObject({ + items: [2], + message: expect.stringContaining('is not reported vendor-not-loaded'), + }); + }); + }); + + it('fails item 1 only for a script added without the loadVendor marker', async () => { + await expect( + conformance('destination', unmarkedScriptVendor, options), + ).rejects.toMatchObject({ + items: [1], + message: expect.stringContaining( + 'example pageView inserts a script without loadVendor', + ), + }); + }); + + it('passes the web-sdk profile for a destination that loads its script into the page', async () => { + await expect( + conformance('destination', pageScriptVendor, { + ...options, + profile: 'web-sdk', + }), + ).resolves.toBeUndefined(); + }); + + it("passes the http profile for a destination on sendWeb's XHR, answered at the page's boundary", async () => { + const url = 'https://e.example/collect'; + const event = { + name: 'page view', + data: { title: 'Home' }, + user: { id: 'user-1' }, + }; + await expect( + conformance('destination', xhrSender, { + profile: 'http', + root, + page, + schema: { + type: 'object', + properties: { url: { type: 'string' } }, + required: ['url'], + }, + settings: { url }, + examples: { + pageView: { + in: event, + out: [['XMLHttpRequest', 'POST', url, '{"name":"page view"}']], + }, + silent: { + public: false, + description: 'A silent rule sends nothing.', + in: event, + mapping: { silent: true }, + out: [], + }, + }, + }), + ).resolves.toBeUndefined(); + }); +}); diff --git a/packages/core/src/__tests__/preview-activator.test.ts b/packages/web/core/src/__tests__/previewActivator.test.ts similarity index 97% rename from packages/core/src/__tests__/preview-activator.test.ts rename to packages/web/core/src/__tests__/previewActivator.test.ts index 822b88846..732a710c3 100644 --- a/packages/core/src/__tests__/preview-activator.test.ts +++ b/packages/web/core/src/__tests__/previewActivator.test.ts @@ -2,15 +2,14 @@ * @jest-environment jsdom * @jest-environment-options {"url": "https://shop.example.com/page"} */ -// Core's jest defaults to jsdom at https://example.com (web.config); the +// web-core's jest defaults to jsdom at https://example.com (web.config); the // options docblock pins this suite's origin so aud checks are realistic. // jsdom has no crypto.subtle and no TextEncoder: inject webcrypto, use Buffer. import { webcrypto } from 'node:crypto'; import { browserSwapActivator, - parseGrant, type SwapActivatorConfig, -} from '../preview'; +} from '../previewActivator'; const crypto = webcrypto; const NOW = 1_700_000_000_000; @@ -165,7 +164,7 @@ describe('browserSwapActivator', () => { it('self-heals when the preview script neither loads nor errors (hung CDN), via the swap timeout', async () => { // No autoResolveScripts here: the injected